diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..232f1ddcb03520892e211b3ff40ca6e7fa61edb7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +wheel/*.whl filter=lfs diff=lfs merge=lfs -text +*.whl filter=lfs diff=lfs merge=lfs -text +checkpoints/* filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100755 index 0000000000000000000000000000000000000000..8d6f7f144e338a38eed1f93bec7483a41ed03791 --- /dev/null +++ b/.gitignore @@ -0,0 +1,157 @@ +# *.pth +*.pt +submodules/diff-gaussian-rasterization +submodules/simple-knn +# checkpoints/ +output* +.gradio/ + +core.* +logs/* +/data/ +# checkpoints/ +video* +train_images* +test_images_save* +/pl_main +/to_be_test +/test_lsm +/test_img +/figure3 + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +video/ +scannet_processed_scenes_1.tar.gz +test_results/* +output/* +test_images +colmap_scannet +/test_lseg \ No newline at end of file diff --git a/app.py b/app.py index 2aceec851e6065888da1e4ee81145dd7d75e949c..a29e061a829766b5fbd79fccb7196573e8d0880c 100644 --- a/app.py +++ b/app.py @@ -1,12 +1,152 @@ -import os -import shlex +import os, subprocess, shlex, sys, gc +import time +import torch +import numpy as np +import shutil +import argparse import gradio as gr -import subprocess -from huggingface_hub import HfApi +import uuid +import spaces +# -hf_token = os.getenv("LSM_token") +subprocess.run(shlex.split("pip install wheel/torch_scatter-2.1.2+pt21cu121-cp310-cp310-linux_x86_64.whl")) +subprocess.run(shlex.split("pip install wheel/flash_attn-2.6.3+cu123torch2.1cxx11abiFALSE-cp310-cp310-linux_x86_64.whl")) +subprocess.run(shlex.split("pip install wheel/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl")) +subprocess.run(shlex.split("pip install wheel/simple_knn-0.0.0-cp310-cp310-linux_x86_64.whl")) +subprocess.run(shlex.split("pip install wheel/curope-0.0.0-cp310-cp310-linux_x86_64.whl")) +subprocess.run(shlex.split("pip install wheel/pointops-1.0-cp310-cp310-linux_x86_64.whl")) -api = HfApi() -api.snapshot_download(repo_id="kairunwen/LSM_private_mast3r", repo_type="space", local_dir=".", token=hf_token) -subprocess.run(shlex.split("pip install -r requirements.txt")) -subprocess.run(shlex.split("python app.py")) \ No newline at end of file +from src.utils.visualization_utils import render_video_from_file +from src.model import LSM_MASt3R + +model = LSM_MASt3R.from_pretrained("checkpoints/pretrained_model/checkpoint-40.pth") +model = model.eval() + + +@spaces.GPU(duration=80) +def process(inputfiles, input_path=None): + # 创建唯一的缓存目录 + cache_dir = os.path.join('outputs', str(uuid.uuid4())) + os.makedirs(cache_dir, exist_ok=True) + + if input_path is not None: + imgs_path = './assets/examples/' + input_path + imgs_names = sorted(os.listdir(imgs_path)) + + inputfiles = [] + for imgs_name in imgs_names: + file_path = os.path.join(imgs_path, imgs_name) + print(file_path) + inputfiles.append(file_path) + print(inputfiles) + + filelist = inputfiles + if len(filelist) != 2: + gr.Warning("Please select 2 images") + shutil.rmtree(cache_dir) # 清理缓存目录 + return None, None, None, None, None, None + + ply_path = os.path.join(cache_dir, 'gaussians.ply') + # render_video_from_file(filelist, model, output_path=cache_dir, resolution=224) + render_video_from_file(filelist, model, output_path=cache_dir, resolution=512) + + rgb_video_path = os.path.join(cache_dir, 'moved', 'output_images_video.mp4') + depth_video_path = os.path.join(cache_dir, 'moved', 'output_depth_video.mp4') + feature_video_path = os.path.join(cache_dir, 'moved', 'output_fmap_video.mp4') + + return filelist, rgb_video_path, depth_video_path, feature_video_path, ply_path, ply_path + + +_TITLE = 'LargeSpatialModel' +_DESCRIPTION = ''' +
+
+ Large Spatial Model: End-to-end Unposed Images to Semantic 3D +
+
+

+ +
+ arxiv  +   + + social + + +
+

+ +* Official demo of: [LargeSpatialModel: End-to-end Unposed Images to Semantic 3D](https://largespatialmodel.github.io/). +* Examples for direct viewing: you can simply click the examples (in the bottom of the page), to quickly view the results on representative data. +''' + +block = gr.Blocks().queue() +with block: + gr.Markdown(_DESCRIPTION) + + with gr.Column(variant="panel"): + with gr.Tab("Input"): + with gr.Row(): + with gr.Column(scale=1): + inputfiles = gr.File(file_count="multiple", label="Load Images") + input_path = gr.Textbox(visible=False, label="example_path") + with gr.Column(scale=1): + image_gallery = gr.Gallery( + label="Gallery", + show_label=False, + elem_id="gallery", + columns=[2], + height=300, # 固定高度 + object_fit="cover" # 确保图片填满空间 + ) + + button_gen = gr.Button("Start Reconstruction", elem_id="button_gen") + processing_msg = gr.Markdown("Processing...", visible=False, elem_id="processing_msg") + + + with gr.Column(variant="panel"): + with gr.Tab("Output"): + with gr.Row(): + with gr.Column(scale=1): + rgb_video = gr.Video(label="RGB Video", autoplay=True) + with gr.Column(scale=1): + feature_video = gr.Video(label="Feature Video", autoplay=True) + with gr.Column(scale=1): + depth_video = gr.Video(label="Depth Video", autoplay=True) + with gr.Row(): + with gr.Group(): + output_model = gr.Model3D( + label="3D Dense Model under Gaussian Splats Formats, need more time to visualize", + interactive=False, + camera_position=[0.5, 0.5, 1], # 稍微偏移一点，以便更好地查看模型 + height=600, + ) + gr.Markdown( + """ +
+   Use the left mouse button to rotate, the scroll wheel to zoom, and the right mouse button to move. +
+ """ + ) + with gr.Row(): + output_file = gr.File(label="PLY File") + + examples = gr.Examples( + examples=[ + "sofa", + ], + inputs=[input_path], + outputs=[image_gallery, rgb_video, depth_video, feature_video, output_model, output_file], + fn=lambda x: process(inputfiles=None, input_path=x), + cache_examples=True, + label="Examples" + ) + + + button_gen.click( + process, + inputs=[inputfiles], + outputs=[image_gallery, rgb_video, depth_video, feature_video, output_model, output_file], + ) + +block.launch(server_name="0.0.0.0", share=False) diff --git a/assets/examples/bicycle/_DSC8679.JPG b/assets/examples/bicycle/_DSC8679.JPG new file mode 100644 index 0000000000000000000000000000000000000000..6a35307868ddd21c46b6742b6aae3154700ea0eb Binary files /dev/null and b/assets/examples/bicycle/_DSC8679.JPG differ diff --git a/assets/examples/bicycle/_DSC8689.JPG b/assets/examples/bicycle/_DSC8689.JPG new file mode 100644 index 0000000000000000000000000000000000000000..190675bb8a81761c08d39ac0bb3dbbf1ba56f0b3 Binary files /dev/null and b/assets/examples/bicycle/_DSC8689.JPG differ diff --git a/assets/examples/bonsai/DSCF5565.JPG b/assets/examples/bonsai/DSCF5565.JPG new file mode 100644 index 0000000000000000000000000000000000000000..02d0f56801b70e98081a03d14c755bbd22a1d883 Binary files /dev/null and b/assets/examples/bonsai/DSCF5565.JPG differ diff --git a/assets/examples/bonsai/DSCF5575.JPG b/assets/examples/bonsai/DSCF5575.JPG new file mode 100644 index 0000000000000000000000000000000000000000..59be6593d106c6b69971f548e630b1413e4619ac Binary files /dev/null and b/assets/examples/bonsai/DSCF5575.JPG differ diff --git a/assets/examples/garden/DSC07956.JPG b/assets/examples/garden/DSC07956.JPG new file mode 100644 index 0000000000000000000000000000000000000000..a21b785a2d7d4ead7787dfed65d0a4e9212053db Binary files /dev/null and b/assets/examples/garden/DSC07956.JPG differ diff --git a/assets/examples/garden/DSC07960.JPG b/assets/examples/garden/DSC07960.JPG new file mode 100644 index 0000000000000000000000000000000000000000..c7f94a4676f6ade86780bbd1b2d63d2dce88e24a Binary files /dev/null and b/assets/examples/garden/DSC07960.JPG differ diff --git a/assets/examples/kitchen/0.jpg b/assets/examples/kitchen/0.jpg new file mode 100644 index 0000000000000000000000000000000000000000..83268805b6f9373672a961bedae9eb267c0715b1 Binary files /dev/null and b/assets/examples/kitchen/0.jpg differ diff --git a/assets/examples/kitchen/64.jpg b/assets/examples/kitchen/64.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3e18c08aab7d0db34651a59f2601db433a0791d4 Binary files /dev/null and b/assets/examples/kitchen/64.jpg differ diff --git a/assets/examples/sofa/000000.jpg b/assets/examples/sofa/000000.jpg new file mode 100644 index 0000000000000000000000000000000000000000..473ed4a1df7496c980d6ea53a1ba18d093c8fa52 Binary files /dev/null and b/assets/examples/sofa/000000.jpg differ diff --git a/assets/examples/sofa/000008.jpg b/assets/examples/sofa/000008.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e6981e4ad3addd4223cd3b29865863c777a2295d Binary files /dev/null and b/assets/examples/sofa/000008.jpg differ diff --git a/configs/model_config.yaml b/configs/model_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..961382de06ff651ca2522470d989629ed7c02499 --- /dev/null +++ b/configs/model_config.yaml @@ -0,0 +1,20 @@ +mast3r_config: + pretrained_model_name_or_path: "checkpoints/pretrained_model/MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric.pth" + +point_transformer_config: + enc_depths: [1, 1, 1, 3, 1] + enc_channels: [32, 64, 128, 256, 512] + enc_num_head: [2, 4, 8, 16, 32] + enc_patch_size: [1024, 1024, 1024, 1024, 1024] + dec_depths: [1, 1, 1, 1] + dec_channels: [64, 64, 128, 256] + dec_num_head: [4, 4, 8, 16] + dec_patch_size: [1024, 1024, 1024, 1024] + +gaussian_head_config: + rgb_residual: true + d_gs_feats: 32 + +lseg_config: + pretrained_model_name_or_path: "checkpoints/pretrained_model/lang_seg.ckpt" + half_res: true diff --git a/requirements.txt b/requirements.txt new file mode 100755 index 0000000000000000000000000000000000000000..432b6f9f4670c3490dc13fd3d9bd30275ee2fcdf --- /dev/null +++ b/requirements.txt @@ -0,0 +1,41 @@ +torch==2.1.2 +torchvision==0.16.2 +pytorch-lightning==2.1.2 +open3d +roma +gradio +matplotlib +tqdm +opencv-python +scipy +einops +trimesh +tensorboard +pyglet<2 +numpy<2.0 +huggingface-hub[torch]>=0.22 +ninja +scikit-learn + + +arrow +pandas +torch-tb-profiler +jaxtyping +ninja +h5py +pyyaml +moviepy==1.0.3 +jupyter +lpips +torch-geometric +spconv-cu120 +git+https://github.com/openai/CLIP.git +sharedarray +tensorboardx +yapf +addict +plyfile +termcolor +timm + diff --git a/scannetv2-labels.combined.tsv b/scannetv2-labels.combined.tsv new file mode 100644 index 0000000000000000000000000000000000000000..b2ad275bb6a80a52d438b5c4d8652cba15dad82b --- /dev/null +++ b/scannetv2-labels.combined.tsv @@ -0,0 +1,608 @@ +id raw_category category count nyu40id eigen13id nyuClass nyu40class eigen13class ModelNet40 ModelNet10 ShapeNetCore55 synsetoffset wnsynsetid wnsynsetkey mpcat40 mpcat40index +1 wall wall 8277 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +2 chair chair 4646 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +22 books book 1678 23 2 book books Books n02870526 book.n.11 objects 39 +3 floor floor 1553 2 5 floor floor Floor n03365592 floor.n.01 floor 2 +5 door door 1483 8 12 door door Wall door n03221720 door.n.01 door 4 +1163 object object 1313 40 7 otherprop Objects objects 39 +16 window window 1209 9 13 window window Window n04587648 window.n.01 window 9 +4 table table 1170 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +56 trash can trash can 1090 39 6 garbage bin otherfurniture Furniture trash_bin 2747177 n02747177 ashcan.n.01 objects 39 +13 pillow pillow 937 18 7 pillow pillow Objects pillow 3938244 n03938244 pillow.n.01 cushion 8 +15 picture picture 862 11 8 picture picture Picture n03931044 picture.n.01 picture 6 +41 ceiling ceiling 806 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 ceiling 17 +26 box box 775 29 7 box box Objects n02883344 box.n.01 objects 39 +161 doorframe doorframe 768 8 12 door door Wall door doorframe.n.01 door 4 +19 monitor monitor 765 40 7 monitor otherprop Objects monitor monitor tv or monitor 3211117 n03782190 monitor.n.04 objects 39 +7 cabinet cabinet 731 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +9 desk desk 680 14 10 desk desk Table desk desk table 4379243 n03179701 desk.n.01 table 5 +8 shelf shelf 641 15 6 shelves shelves Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +10 office chair office chair 595 5 4 chair chair Chair chair chair chair 3001627 n04373704 swivel_chair.n.01 chair 3 +31 towel towel 570 27 7 towel towel Objects n04459362 towel.n.01 towel 20 +6 couch couch 502 6 9 sofa sofa Sofa sofa sofa sofa 4256520 n04256520 sofa.n.01 sofa 10 +14 sink sink 488 34 7 sink sink Objects sink n04223580 sink.n.01 sink 15 +48 backpack backpack 479 40 7 backpack otherprop Objects n02769748 backpack.n.01 objects 39 +28 lamp lamp 419 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +11 bed bed 370 4 1 bed bed Bed bed bed bed 2818832 n02818832 bed.n.01 bed 11 +18 bookshelf bookshelf 360 10 6 bookshelf bookshelf Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +71 mirror mirror 349 19 7 mirror mirror Objects n03773035 mirror.n.01 mirror 21 +21 curtain curtain 347 16 13 curtain curtain Window curtain n03151077 curtain.n.01 curtain 12 +40 plant plant 331 40 7 plant otherprop Objects plant n00017222 plant.n.02 plant 14 +52 whiteboard whiteboard 327 30 7 whiteboard whiteboard Objects n03211616 display_panel.n.01 board_panel 35 +96 radiator radiator 322 39 6 radiator otherfurniture Furniture n04041069 radiator.n.02 misc 40 +22 book book 318 23 2 book books Books n02870526 book.n.11 objects 39 +29 kitchen cabinet kitchen cabinet 310 3 6 cabinet cabinet Furniture n02933112 cabinet.n.01 cabinet 7 +49 toilet paper toilet paper 291 40 7 toilet paper otherprop Objects n15075141 toilet_tissue.n.01 objects 39 +29 kitchen cabinets kitchen cabinet 289 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +23 armchair armchair 281 5 4 chair chair Chair chair chair chair 3001627 n02738535 armchair.n.01 chair 3 +63 shoes shoe 272 40 7 shoe otherprop Objects n04199027 shoe.n.01 clothes 38 +24 coffee table coffee table 258 7 10 coffee table table Table table table table 4379243 n03063968 coffee_table.n.01 table 5 +17 toilet toilet 256 33 7 toilet toilet Objects toilet toilet n04446276 toilet.n.01 toilet 18 +47 bag bag 252 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +32 clothes clothes 248 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +46 keyboard keyboard 246 40 7 keyboard otherprop Objects keyboard computer keyboard 3085013 n03085013 computer_keyboard.n.01 objects 39 +65 bottle bottle 226 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +97 recycling bin recycling bin 225 39 6 garbage bin otherfurniture Furniture trash_bin 2747177 n02747177 ashcan.n.01 objects 39 +34 nightstand nightstand 224 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 chest_of_drawers 13 +38 stool stool 221 40 7 stool otherprop Objects stool n04326896 stool.n.01 stool 19 +33 tv tv 219 25 11 television television TV tv or monitor 3211117 n03211117 display.n.06 tv_monitor 22 +75 file cabinet file cabinet 217 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +36 dresser dresser 213 17 6 dresser dresser Furniture dresser dresser n03015254 chest_of_drawers.n.01 chest_of_drawers 13 +64 computer tower computer tower 203 40 7 computer otherprop Objects n03082979 computer.n.01 objects 39 +32 clothing clothes 165 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +101 telephone telephone 164 40 7 telephone otherprop Objects telephone 4401088 n04401088 telephone.n.01 objects 39 +130 cup cup 157 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 +27 refrigerator refrigerator 154 24 6 refridgerator refridgerator Furniture n04070727 refrigerator.n.01 appliances 37 +44 end table end table 147 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +131 jacket jacket 146 40 7 jacket otherprop Objects n03589791 jacket.n.01 clothes 38 +55 shower curtain shower curtain 144 28 7 shower curtain shower curtain Objects curtain n04209239 shower_curtain.n.01 curtain 12 +42 bathtub bathtub 144 36 7 bathtub bathtub Objects bathtub bathtub tub 2808440 n02808440 bathtub.n.01 bathtub 25 +59 microwave microwave 141 40 7 microwave otherprop Objects microwave 3761084 n03761084 microwave.n.02 appliances 37 +159 kitchen counter kitchen counter 140 12 6 counter counter Furniture table table table 4379243 n03116530 counter.n.01 counter 26 +74 sofa chair sofa chair 129 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +82 paper towel dispenser paper towel dispenser 129 40 7 paper towel dispenser otherprop Objects objects 39 +1164 bathroom vanity bathroom vanity 126 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 table 5 +93 suitcase suitcase 118 40 7 luggage otherprop Objects n02773838 bag.n.06 objects 39 +77 laptop laptop 111 40 7 laptop otherprop Objects laptop laptop 3642806 n03642806 laptop.n.01 objects 39 +67 ottoman ottoman 111 39 6 ottoman otherfurniture Furniture stool n03380724 footstool.n.01 stool 19 +128 shower walls shower wall 109 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +50 printer printer 106 40 7 printer otherprop Objects printer 4004475 n04004475 printer.n.03 appliances 37 +35 counter counter 104 12 6 counter counter Furniture table table table 4379243 n03116530 counter.n.01 counter 26 +69 board board 100 38 7 board otherstructure Objects board_panel 35 +100 soap dispenser soap dispenser 99 40 7 otherprop Objects n04254120 soap_dispenser.n.01 objects 39 +62 stove stove 95 38 7 stove otherstructure Objects stove 4330267 n04330267 stove.n.02 appliances 37 +105 light light 93 38 7 light otherstructure Objects n03665366 light.n.02 lighting 28 +1165 closet wall closet wall 90 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +165 mini fridge mini fridge 87 24 6 refridgerator refridgerator Furniture n03273913 electric_refrigerator.n.01 appliances 37 +7 cabinets cabinet 79 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +5 doors door 76 8 12 door door Wall door n03221720 door.n.01 door 4 +76 fan fan 75 40 7 fan otherprop Objects n03320046 fan.n.01 misc 40 +230 tissue box tissue box 73 40 7 tissue box otherprop Objects n02883344 box.n.01 objects 39 +54 blanket blanket 72 40 7 blanket otherprop Objects n02849154 blanket.n.01 objects 39 +125 bathroom stall bathroom stall 71 38 7 otherstructure Objects n02873839 booth.n.02 misc 40 +72 copier copier 70 40 7 otherprop Objects n03257586 duplicator.n.01 appliances 37 +68 bench bench 66 39 6 bench otherfurniture Furniture bench bench 2828884 n02828884 bench.n.01 seating 34 +145 bar bar 66 38 7 bar otherstructure Objects n02788689 bar.n.03 misc 40 +157 soap dish soap dish 65 40 7 soap dish otherprop Objects n04254009 soap_dish.n.01 objects 39 +1166 laundry hamper laundry hamper 65 40 7 laundry basket otherprop Objects objects 39 +132 storage bin storage bin 63 40 7 storage bin otherprop Objects objects 39 +1167 bathroom stall door bathroom stall door 62 8 12 door door Wall door n03221720 door.n.01 door 4 +232 light switch light switch 61 38 7 light switch otherstructure Objects n04372370 switch.n.01 misc 40 +134 coffee maker coffee maker 61 40 7 otherprop Objects n03063338 coffee_maker.n.01 appliances 37 +51 tv stand tv stand 61 39 6 tv stand otherfurniture Furniture tv_stand n03290653 entertainment_center.n.01 furniture 36 +250 decoration decoration 60 40 7 otherprop Objects n03169390 decoration.n.01 misc 40 +1168 ceiling light ceiling light 59 38 7 light otherstructure Objects n03665366 light.n.02 lighting 28 +342 range hood range hood 59 38 7 range hood otherstructure Objects range_hood n04053677 range_hood.n.01 misc 40 +89 blackboard blackboard 58 38 7 blackboard otherstructure Objects n02846511 blackboard.n.01 board_panel 35 +103 clock clock 58 40 7 clock otherprop Objects clock 3046257 n03046257 clock.n.01 objects 39 +99 wardrobe closet wardrobe 54 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 furniture 36 +95 rail rail 53 38 7 railing otherstructure Objects n04047401 railing.n.01 railing 30 +154 bulletin board bulletin board 53 38 7 board otherstructure Objects n03211616 display_panel.n.01 board_panel 35 +140 mat mat 52 20 5 floor mat floor mat Floor n03727837 mat.n.01 floor 2 +1169 trash bin trash bin 52 39 6 garbage bin otherfurniture Furniture trash_bin 2747177 n02747177 ashcan.n.01 objects 39 +193 ledge ledge 51 38 7 otherstructure Objects n09337253 ledge.n.01 misc 40 +116 seat seat 49 39 6 furniture otherfurniture Furniture n04161981 seat.n.03 furniture 36 +202 mouse mouse 49 40 7 mouse otherprop Objects n03793489 mouse.n.04 objects 39 +73 basket basket 48 40 7 basket otherprop Objects basket 2801938 n02801938 basket.n.01 objects 39 +78 shower shower 48 38 7 otherstructure Objects n04208936 shower.n.01 shower 23 +1170 dumbbell dumbbell 48 40 7 otherprop Objects n03255030 dumbbell.n.01 objects 39 +79 paper paper 46 26 7 paper paper Objects n14974264 paper.n.01 objects 39 +80 person person 46 31 7 person person Objects person n05217688 person.n.02 misc 40 +141 windowsill windowsill 45 38 7 otherstructure Objects n04590263 windowsill.n.01 window 9 +57 closet closet 45 39 6 wardrobe otherfurniture Furniture wardrobe misc 40 +102 bucket bucket 45 40 7 bucket otherprop Objects n02909870 bucket.n.01 misc 40 +261 sign sign 44 40 7 sign otherprop Objects n04217882 signboard.n.01 objects 39 +118 speaker speaker 43 40 7 speaker otherprop Objects speaker 3691459 n03691459 loudspeaker.n.01 objects 39 +136 dishwasher dishwasher 43 38 7 dishwasher otherstructure Objects dishwasher 3207941 n03207941 dishwasher.n.01 appliances 37 +98 container container 43 40 7 container otherprop Objects n03094503 container.n.01 objects 39 +1171 stair rail stair rail 42 38 7 banister otherstructure Objects n02788148 bannister.n.02 railing 30 +170 shower curtain rod shower curtain rod 42 40 7 otherprop Objects curtain 12 +1172 tube tube 41 40 7 otherprop Objects misc 40 +1173 bathroom cabinet bathroom cabinet 39 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +79 papers paper 39 26 7 paper paper Objects n14974264 paper.n.01 objects 39 +221 storage container storage container 39 40 7 container otherprop Objects objects 39 +570 paper bag paper bag 39 37 7 bag bag Objects n04122825 sack.n.01 objects 39 +138 paper towel roll paper towel roll 39 40 7 paper towel otherprop Objects n03887697 paper_towel.n.01 towel 20 +168 ball ball 39 40 7 ball otherprop Objects objects 39 +276 closet doors closet door 38 8 12 door door Wall door n03221720 door.n.01 door 4 +106 laundry basket laundry basket 37 40 7 laundry basket otherprop Objects basket 2801938 n03050864 clothes_hamper.n.01 objects 39 +214 cart cart 37 40 7 cart otherprop Objects n03484083 handcart.n.01 shelving 31 +276 closet door closet door 35 8 12 door door Wall door n03221720 door.n.01 door 4 +323 dish rack dish rack 35 40 7 dish rack otherprop Objects n03207630 dish_rack.n.01 objects 39 +58 stairs stairs 35 38 7 stairs otherstructure Objects n04298308 stairway.n.01 stairs 16 +86 blinds blinds 35 13 13 blinds blinds Window n02851099 blind.n.03 blinds 32 +2 stack of chairs chair 35 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +399 purse purse 34 40 7 purse otherprop Objects n02774152 bag.n.04 objects 39 +121 bicycle bicycle 33 40 7 bicycle otherprop Objects bicycle 2834778 n02834778 bicycle.n.01 objects 39 +185 tray tray 32 40 7 tray otherprop Objects n04476259 tray.n.01 objects 39 +300 plunger plunger 30 40 7 otherprop Objects n03970156 plunger.n.03 objects 39 +180 paper cutter paper cutter 30 40 7 paper cutter otherprop Objects n03886940 paper_cutter.n.01 objects 39 +163 toilet paper dispenser toilet paper dispenser 29 40 7 otherprop Objects objects 39 +26 boxes box 29 29 7 box box Objects n02883344 box.n.01 objects 39 +66 bin bin 28 40 7 bin otherprop Objects n02839910 bin.n.01 objects 39 +208 toilet seat cover dispenser toilet seat cover dispenser 28 40 7 otherprop Objects objects 39 +112 guitar guitar 28 40 7 guitar otherprop Objects guitar guitar 3467517 n03467517 guitar.n.01 objects 39 +540 mailboxes mailbox 28 29 7 box box Objects mailbox 3710193 n03710193 mailbox.n.01 misc 40 +395 handicap bar handicap bar 27 38 7 bar otherstructure Objects misc 40 +166 fire extinguisher fire extinguisher 27 40 7 fire extinguisher otherprop Objects n03345837 fire_extinguisher.n.01 misc 40 +122 ladder ladder 27 39 6 ladder otherfurniture Furniture stairs n03632277 ladder.n.01 stairs 16 +120 column column 26 38 7 column otherstructure Objects n03074380 column.n.06 column 24 +107 pipe pipe 25 40 7 pipe otherprop Objects n03944672 pipe.n.02 misc 40 +283 vacuum cleaner vacuum cleaner 25 40 7 otherprop Objects n04517823 vacuum.n.04 objects 39 +88 plate plate 24 40 7 plate otherprop Objects n03959485 plate.n.04 objects 39 +90 piano piano 24 39 6 piano otherfurniture Furniture piano piano 3928116 n03928116 piano.n.01 furniture 36 +177 water cooler water cooler 24 39 6 water cooler otherfurniture Furniture n04559166 water_cooler.n.01 misc 40 +1174 cd case cd case 24 40 7 otherprop Objects objects 39 +562 bowl bowl 24 40 7 bowl otherprop Objects bowl bowl 2880940 n02880940 bowl.n.03 objects 39 +1175 closet rod closet rod 24 40 7 otherprop Objects n04100174 rod.n.01 misc 40 +1156 bathroom counter bathroom counter 24 12 6 counter counter Furniture table table table 4379243 n03116530 counter.n.01 counter 26 +84 oven oven 23 38 7 oven otherstructure Objects n03862676 oven.n.01 appliances 37 +104 stand stand 23 39 6 stand otherfurniture Furniture table table table 4379243 n04301000 stand.n.04 table 5 +229 scale scale 23 40 7 scale otherprop Objects n04141975 scale.n.07 objects 39 +70 washing machine washing machine 23 39 6 washing machine otherfurniture Furniture washing_machine 4554684 n04554684 washer.n.03 appliances 37 +325 broom broom 22 40 7 broom otherprop Objects n02906734 broom.n.01 objects 39 +169 hat hat 22 40 7 hat otherprop Objects n03497657 hat.n.01 clothes 38 +128 shower wall shower wall 22 1 12 wall wall Wall n04208936 shower.n.01 wall 1 +331 guitar case guitar case 21 40 7 guitar case otherprop Objects objects 39 +87 rack rack 21 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +488 water pitcher water pitcher 21 40 7 pitcher otherprop Objects n03950228 pitcher.n.02 objects 39 +776 laundry detergent laundry detergent 21 40 7 otherprop Objects objects 39 +370 hair dryer hair dryer 21 40 7 hair dryer otherprop Objects n03483316 hand_blower.n.01 objects 39 +191 pillar pillar 21 38 7 column otherstructure Objects n03073977 column.n.07 column 24 +748 divider divider 20 40 7 otherprop Objects wall 1 +242 power outlet power outlet 19 40 7 otherprop Objects misc 40 +45 dining table dining table 19 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +417 shower floor shower floor 19 2 5 floor floor Floor n04208936 shower.n.01 floor 2 +70 washing machines washing machine 19 39 6 washing machine otherfurniture Furniture washing_machine 4554684 n04554684 washer.n.03 appliances 37 +188 shower door shower door 19 8 12 door door Wall door n04208936 shower.n.01 door 4 +1176 coffee kettle coffee kettle 18 40 7 pot otherprop Objects n03612814 kettle.n.01 objects 39 +1177 wardrobe cabinet wardrobe 18 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 furniture 36 +1178 structure structure 18 38 7 otherstructure Objects misc 40 +18 bookshelves bookshelf 17 10 6 bookshelf bookshelf Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +110 clothes dryer clothes dryer 17 39 6 otherfurniture Furniture n03251766 dryer.n.01 appliances 37 +148 toaster toaster 17 40 7 toaster otherprop Objects n04442312 toaster.n.02 appliances 37 +63 shoe shoe 17 40 7 shoe otherprop Objects n04199027 shoe.n.01 clothes 38 +155 ironing board ironing board 16 39 6 ironing board otherfurniture Furniture n03586090 ironing_board.n.01 objects 39 +572 alarm clock alarm clock 16 40 7 alarm clock otherprop Objects clock 3046257 n02694662 alarm_clock.n.01 objects 39 +1179 shower head shower head 15 38 7 otherstructure Objects shower 23 +28 lamp base lamp 15 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +392 water bottle water bottle 15 40 7 bottle otherprop Objects bottle bottle 2876657 n04557648 water_bottle.n.01 objects 39 +1180 keyboard piano keyboard piano 15 39 6 piano otherfurniture Furniture piano piano 3928116 n03928116 piano.n.01 furniture 36 +609 projector screen projector screen 15 38 7 projector screen otherstructure Objects misc 40 +1181 case of water bottles case of water bottles 15 40 7 otherprop Objects objects 39 +195 toaster oven toaster oven 14 40 7 toaster oven otherprop Objects n04442441 toaster_oven.n.01 appliances 37 +581 music stand music stand 14 39 6 music stand otherfurniture Furniture n03801760 music_stand.n.01 furniture 36 +58 staircase stairs 14 38 7 stairs otherstructure Objects n04298308 stairway.n.01 stairs 16 +1182 coat rack coat rack 14 40 7 otherprop Objects n03059103 coatrack.n.01 shelving 3 +1183 storage organizer storage organizer 14 40 7 otherprop Objects shelving 3 +139 machine machine 14 40 7 machine otherprop Objects n03699975 machine.n.01 appliances 37 +1184 folded chair folded chair 14 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +1185 fire alarm fire alarm 14 40 7 otherprop Objects n03343737 fire_alarm.n.02 misc 40 +156 fireplace fireplace 13 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 fireplace 27 +408 vent vent 13 40 7 otherprop Objects n04526241 vent.n.01 misc 40 +213 furniture furniture 13 39 6 furniture otherfurniture Furniture n03405725 furniture.n.01 furniture 36 +1186 power strip power strip 13 40 7 otherprop Objects objects 39 +1187 calendar calendar 13 40 7 otherprop Objects objects 39 +1188 poster poster 13 11 8 picture picture Picture n03931044 picture.n.01 picture 6 +115 toilet paper holder toilet paper holder 13 40 7 toilet paper holder otherprop Objects objects 39 +1189 potted plant potted plant 12 40 7 plant otherprop Objects plant n00017222 plant.n.02 plant 14 +304 stuffed animal stuffed animal 12 40 7 stuffed animal otherprop Objects n04399382 teddy.n.01 objects 39 +1190 luggage luggage 12 40 7 luggage otherprop Objects n02774630 baggage.n.01 objects 39 +21 curtains curtain 12 16 13 curtain curtain Window curtain n03151077 curtain.n.01 curtain 12 +312 headphones headphones 12 40 7 otherprop Objects n03261776 earphone.n.01 objects 39 +233 crate crate 12 39 6 crate otherfurniture Furniture n03127925 crate.n.01 objects 39 +286 candle candle 12 40 7 candle otherprop Objects lamp n02948072 candle.n.01 objects 39 +264 projector projector 12 40 7 projector otherprop Objects n04009552 projector.n.02 objects 39 +110 clothes dryers clothes dryer 12 39 6 otherfurniture Furniture n03251766 dryer.n.01 appliances 37 +1191 mattress mattress 12 4 1 bed bed Bed bed bed bed 2818832 n02818832 bed.n.01 bed 11 +356 dustpan dustpan 12 40 7 otherprop Objects n03259009 dustpan.n.02 objects 39 +25 drawer drawer 11 39 6 drawer otherfurniture Furniture n03233905 drawer.n.01 furniture 36 +750 rod rod 11 40 7 otherprop Objects pistol 3948459 n03427202 gat.n.01 misc 40 +269 globe globe 11 40 7 globe otherprop Objects objects 39 +307 footrest footrest 11 39 6 foot rest otherfurniture Furniture stool n03380724 footstool.n.01 stool 19 +410 piano bench piano bench 11 39 6 piano bench otherfurniture Furniture bench bench 2828884 n02828884 bench.n.01 seating 34 +730 breakfast bar breakfast bar 11 38 7 bar otherstructure Objects counter 26 +216 step stool step stool 11 40 7 step stool otherprop Objects stool n04315713 step_stool.n.01 stool 19 +1192 hand rail hand rail 11 38 7 railing otherstructure Objects railing 30 +119 vending machine vending machine 11 40 7 machine otherprop Objects n04525305 vending_machine.n.01 appliances 37 +682 ceiling fan ceiling fan 11 40 7 fan otherprop Objects n03320046 fan.n.01 misc 40 +434 swiffer swiffer 11 40 7 otherprop Objects objects 39 +126 foosball table foosball table 11 39 6 foosball table otherfurniture Furniture table table table 4379243 n04379243 table.n.02 table 5 +919 jar jar 11 40 7 jar otherprop Objects jar 3593526 n03593526 jar.n.01 objects 39 +85 footstool footstool 11 39 6 ottoman otherfurniture Furniture stool n03380724 footstool.n.01 stool 19 +1193 folded table folded table 10 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +108 round table round table 10 7 10 table table Table table table table 4379243 n04114554 round_table.n.02 table 5 +135 hamper hamper 10 40 7 basket otherprop Objects basket 2801938 n03482405 hamper.n.02 objects 39 +1194 poster tube poster tube 10 40 7 otherprop Objects objects 39 +432 case case 10 40 7 case otherprop Objects objects 39 +53 carpet carpet 10 40 7 rug otherprop Objects n04118021 rug.n.01 floor 2 +1195 thermostat thermostat 10 40 7 otherprop Objects n04422875 thermostat.n.01 misc 40 +111 coat coat 10 40 7 jacket otherprop Objects n03057021 coat.n.01 clothes 38 +305 water fountain water fountain 10 38 7 water fountain otherstructure Objects n03241335 drinking_fountain.n.01 misc 40 +1125 smoke detector smoke detector 10 40 7 otherprop Objects misc 40 +13 pillows pillow 9 18 7 pillow pillow Objects pillow 3938244 n03938244 pillow.n.01 cushion 8 +1196 flip flops flip flops 9 40 7 shoe otherprop Objects n04199027 shoe.n.01 clothes 38 +1197 cloth cloth 9 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +1198 banner banner 9 40 7 otherprop Objects n02788021 banner.n.01 misc 40 +1199 clothes hanger clothes hanger 9 40 7 otherprop Objects n03057920 coat_hanger.n.01 objects 39 +1200 whiteboard eraser whiteboard eraser 9 40 7 otherprop Objects objects 39 +378 iron iron 9 40 7 otherprop Objects n03584829 iron.n.04 objects 39 +591 instrument case instrument case 9 40 7 case otherprop Objects objects 39 +49 toilet paper rolls toilet paper 9 40 7 toilet paper otherprop Objects n15075141 toilet_tissue.n.01 objects 39 +92 soap soap 9 40 7 soap otherprop Objects n04253437 soap.n.01 objects 39 +1098 block block 9 40 7 otherprop Objects misc 40 +291 wall hanging wall hanging 8 40 7 otherprop Objects n03491178 hanging.n.01 picture 6 +1063 kitchen island kitchen island 8 38 7 kitchen island otherstructure Objects n03620600 kitchen_island.n.01 counter 26 +107 pipes pipe 8 38 7 otherstructure Objects misc 40 +1135 toothbrush toothbrush 8 40 7 toothbrush otherprop Objects n04453156 toothbrush.n.01 objects 39 +189 shirt shirt 8 40 7 otherprop Objects n04197391 shirt.n.01 clothes 38 +245 cutting board cutting board 8 40 7 cutting board otherprop Objects n03025513 chopping_board.n.01 objects 39 +194 vase vase 8 40 7 vase otherprop Objects vase jar 3593526 n04522168 vase.n.01 objects 39 +1201 shower control valve shower control valve 8 38 7 otherstructure Objects n04208936 shower.n.01 shower 23 +386 exercise machine exercise machine 8 40 7 machine otherprop Objects gym_equipment 33 +1202 compost bin compost bin 8 39 6 garbage bin otherfurniture Furniture trash_bin 2747177 n02747177 ashcan.n.01 objects 39 +857 shorts shorts 8 40 7 shorts otherprop Objects clothes 38 +452 tire tire 8 40 7 otherprop Objects n04440749 tire.n.01 objects 39 +1203 teddy bear teddy bear 7 40 7 stuffed animal otherprop Objects n04399382 teddy.n.01 objects 39 +346 bathrobe bathrobe 7 40 7 otherprop Objects n02807616 bathrobe.n.01 clothes 38 +152 handrail handrail 7 38 7 railing otherstructure Objects n02788148 bannister.n.02 railing 30 +83 faucet faucet 7 40 7 faucet otherprop Objects faucet 3325088 n03325088 faucet.n.01 misc 40 +1204 pantry wall pantry wall 7 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +726 thermos thermos 7 40 7 flask otherprop Objects bottle bottle 2876657 n04422727 thermos.n.01 objects 39 +61 rug rug 7 40 7 rug otherprop Objects n04118021 rug.n.01 floor 2 +39 couch cushions cushion 7 18 7 pillow pillow Objects n03151500 cushion.n.03 cushion 8 +1117 tripod tripod 7 39 6 stand otherfurniture Furniture n04485082 tripod.n.01 objects 39 +540 mailbox mailbox 7 29 7 box box Objects mailbox 3710193 n03710193 mailbox.n.01 misc 40 +1205 tupperware tupperware 7 40 7 otherprop Objects objects 39 +415 shoe rack shoe rack 7 40 7 shoe rack otherprop Objects shelving 31 +31 towels towel 6 27 7 towel towel Objects n04459362 towel.n.01 towel 20 +1206 beer bottles beer bottle 6 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +153 treadmill treadmill 6 39 6 treadmill otherfurniture Furniture n04477387 treadmill.n.01 gym_equipment 33 +1207 salt salt 6 40 7 otherprop Objects objects 39 +129 chest chest 6 39 6 chest otherfurniture Furniture dresser dresser chest_of_drawers 13 +220 dispenser dispenser 6 40 7 otherprop Objects n03210683 dispenser.n.01 objects 39 +1208 mirror doors mirror door 6 8 12 door door Wall door n03221720 door.n.01 door 4 +231 remote remote 6 40 7 otherprop Objects remote_control 4074963 n04074963 remote_control.n.01 objects 39 +1209 folded ladder folded ladder 6 39 6 ladder otherfurniture Furniture stairs n03632277 ladder.n.01 misc 40 +39 cushion cushion 6 18 7 pillow pillow Objects n03151500 cushion.n.03 cushion 8 +1210 carton carton 6 40 7 otherprop Objects objects 39 +117 step step 6 38 7 otherstructure Objects n04314914 step.n.04 misc 40 +822 drying rack drying rack 6 39 6 drying rack otherfurniture Furniture shelving 31 +238 slippers slipper 6 40 7 shoe otherprop Objects n04241394 slipper.n.01 clothes 38 +143 pool table pool table 6 39 6 pool table otherfurniture Furniture table table table 4379243 n03982430 pool_table.n.01 table 5 +1211 soda stream soda stream 6 40 7 otherprop Objects objects 39 +228 toilet brush toilet brush 6 40 7 toilet brush otherprop Objects objects 39 +494 loft bed loft bed 6 4 1 bed bed Bed bed bed bed 2818832 n02818832 bed.n.01 bed 11 +226 cooking pot cooking pot 6 40 7 pot otherprop Objects objects 39 +91 heater heater 6 39 6 heater otherfurniture Furniture n03508101 heater.n.01 misc 40 +1072 messenger bag messenger bag 6 37 7 bag bag Objects objects 39 +435 stapler stapler 6 40 7 stapler otherprop Objects n04303497 stapler.n.01 objects 39 +1165 closet walls closet wall 5 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +345 scanner scanner 5 40 7 otherprop Objects appliances 37 +893 elliptical machine elliptical machine 5 40 7 machine otherprop Objects gym_equipment 33 +621 kettle kettle 5 40 7 pot otherprop Objects n03612814 kettle.n.01 objects 39 +1212 metronome metronome 5 40 7 otherprop Objects n03757604 metronome.n.01 objects 39 +297 dumbell dumbell 5 40 7 otherprop Objects objects 39 +1213 music book music book 5 23 2 book books Books n02870526 book.n.11 objects 39 +1214 rice cooker rice cooker 5 40 7 otherprop Objects objects 39 +1215 dart board dart board 5 38 7 board otherstructure Objects n03162940 dartboard.n.01 objects 39 +529 sewing machine sewing machine 5 40 7 sewing machine otherprop Objects n04179913 sewing_machine.n.01 objects 39 +1216 grab bar grab bar 5 38 7 railing otherstructure Objects railing 30 +1217 flowerpot flowerpot 5 40 7 vase otherprop Objects vase jar 3593526 n04522168 vase.n.01 objects 39 +1218 painting painting 5 11 8 picture picture Picture n03931044 picture.n.01 picture 6 +1219 railing railing 5 38 7 railing otherstructure Objects n04047401 railing.n.01 railing 30 +1220 stair stair 5 38 7 stairs otherstructure Objects stairs n04314914 step.n.04 stairs 16 +525 toolbox toolbox 5 39 6 chest otherfurniture Furniture n04452615 toolbox.n.01 objects 39 +204 nerf gun nerf gun 5 40 7 otherprop Objects objects 39 +693 binders binder 5 40 7 binder otherprop Objects objects 39 +179 desk lamp desk lamp 5 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +1221 quadcopter quadcopter 5 40 7 otherprop Objects objects 39 +1222 pitcher pitcher 5 40 7 pitcher otherprop Objects n03950228 pitcher.n.02 objects 39 +1223 hanging hanging 5 40 7 otherprop Objects misc 40 +1224 mail mail 5 40 7 otherprop Objects misc 40 +1225 closet ceiling closet ceiling 5 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 ceiling 17 +1226 hoverboard hoverboard 5 40 7 otherprop Objects objects 39 +1227 beanbag chair beanbag chair 5 39 6 bean bag otherfurniture Furniture n02816656 beanbag.n.01 chair 3 +571 water heater water heater 5 40 7 water heater otherprop Objects n04560113 water_heater.n.01 misc 40 +1228 spray bottle spray bottle 5 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +556 rope rope 5 40 7 rope otherprop Objects n04108268 rope.n.01 objects 39 +280 plastic container plastic container 5 40 7 container otherprop Objects objects 39 +1229 soap bottle soap bottle 5 40 7 soap otherprop Objects objects 39 +1230 ikea bag ikea bag 4 37 7 bag bag Objects 2773838 n02773838 bag.n.06 objects 39 +1231 sleeping bag sleeping bag 4 40 7 otherprop Objects n04235860 sleeping_bag.n.01 objects 39 +1232 duffel bag duffel bag 4 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +746 frying pan frying pan 4 40 7 frying pan otherprop Objects n03400231 frying_pan.n.01 objects 39 +1233 oven mitt oven mitt 4 40 7 otherprop Objects objects 39 +1234 pot pot 4 40 7 pot otherprop Objects n04235860 sleeping_bag.n.01 objects 39 +144 hand dryer hand dryer 4 40 7 otherprop Objects objects 39 +282 dollhouse dollhouse 4 39 6 doll house otherfurniture Furniture n03219483 dollhouse.n.01 objects 39 +167 shampoo bottle shampoo bottle 4 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +1235 hair brush hair brush 4 40 7 otherprop Objects n02908217 brush.n.02 objects 39 +1236 tennis racket tennis racket 4 40 7 otherprop Objects n04409806 tennis_racket.n.01 objects 39 +1237 display case display case 4 40 7 case otherprop Objects objects 39 +234 ping pong table ping pong table 4 39 6 ping pong table otherfurniture Furniture table table table 4379243 n04379243 table.n.02 table 5 +563 boiler boiler 4 40 7 otherprop Objects misc 40 +1238 bag of coffee beans bag of coffee beans 4 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +1239 bananas banana 4 40 7 otherprop Objects n00021265 food.n.01 objects 39 +1240 carseat carseat 4 40 7 otherprop Objects misc 40 +366 helmet helmet 4 40 7 otherprop Objects helmet 3513137 n03513137 helmet.n.02 clothes 38 +816 umbrella umbrella 4 40 7 umbrella otherprop Objects n04507155 umbrella.n.01 objects 39 +1241 coffee box coffee box 4 40 7 otherprop Objects objects 39 +719 envelope envelope 4 40 7 envelope otherprop Objects n03291819 envelope.n.01 objects 39 +284 wet floor sign wet floor sign 4 40 7 sign otherprop Objects misc 40 +1242 clothing rack clothing rack 4 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +247 controller controller 4 40 7 otherprop Objects n03096960 control.n.09 objects 39 +1243 bath walls bathroom wall 4 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +1244 podium podium 4 39 6 otherfurniture Furniture n03159640 dais.n.01 furniture 36 +1245 storage box storage box 4 29 7 box box Objects n02883344 box.n.01 objects 39 +1246 dolly dolly 4 40 7 otherprop Objects misc 40 +1247 shampoo shampoo 3 40 7 otherprop Objects n04183516 shampoo.n.01 objects 39 +592 paper tray paper tray 3 40 7 paper tray otherprop Objects objects 39 +385 cabinet door cabinet door 3 8 12 door door Wall door door 4 +1248 changing station changing station 3 40 7 otherprop Objects misc 40 +1249 poster printer poster printer 3 40 7 printer otherprop Objects printer 4004475 n04004475 printer.n.03 appliances 37 +133 screen screen 3 40 7 otherprop Objects n03151077 curtain.n.01 curtain 12 +301 soap bar soap bar 3 38 7 bar otherstructure Objects objects 39 +1250 crutches crutches 3 40 7 otherprop Objects n03141823 crutch.n.01 objects 39 +379 studio light studio light 3 38 7 light otherstructure Objects lighting 28 +130 stack of cups cup 3 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 +1251 toilet flush button toilet flush button 3 40 7 otherprop Objects objects 39 +450 trunk trunk 3 40 7 otherprop Objects misc 40 +1252 grocery bag grocery bag 3 37 7 bag bag Objects suitcase 2773838 n03461288 grocery_bag.n.01 objects 39 +316 plastic bin plastic bin 3 40 7 bin otherprop Objects objects 39 +1253 pizza box pizza box 3 29 7 box box Objects objects 39 +385 cabinet doors cabinet door 3 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 door 4 +1254 legs legs 3 31 7 person person Objects person n05217688 person.n.02 misc 40 +461 car car 3 40 7 car otherprop Objects car car 2958343 n02958343 car.n.01 misc 40 +1255 shaving cream shaving cream 3 40 7 otherprop Objects n04186051 shaving_cream.n.01 objects 39 +1256 luggage stand luggage stand 3 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +599 shredder shredder 3 40 7 otherprop Objects n04210120 shredder.n.01 objects 39 +281 statue statue 3 40 7 sculpture otherprop Objects n04306847 statue.n.01 misc 40 +1257 urinal urinal 3 33 7 toilet toilet Objects toilet toilet n04515991 urinal.n.01 toilet 18 +1258 hose hose 3 40 7 otherprop Objects n03539875 hose.n.03 misc 40 +1259 bike pump bike pump 3 40 7 otherprop Objects objects 39 +319 coatrack coatrack 3 40 7 otherprop Objects n03059103 coatrack.n.01 shelving 31 +1260 bear bear 3 40 7 otherprop Objects objects 39 +28 wall lamp lamp 3 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +1261 humidifier humidifier 3 40 7 otherprop Objects objects 39 +546 toothpaste toothpaste 3 40 7 toothpaste otherprop Objects objects 39 +1262 mouthwash bottle mouthwash bottle 3 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +1263 poster cutter poster cutter 3 40 7 otherprop Objects objects 39 +1264 golf bag golf bag 3 37 7 bag bag Objects suitcase 2773838 n03445617 golf_bag.n.01 objects 39 +1265 food container food container 3 40 7 container otherprop Objects n03094503 container.n.01 objects 39 +1266 camera camera 3 40 7 otherprop Objects objects 39 +28 table lamp lamp 3 35 7 lamp lamp Objects lamp lamp 3636649 n04380533 table_lamp.n.01 lighting 28 +1267 yoga mat yoga mat 3 20 5 floor mat floor mat Floor n03727837 mat.n.01 floor 2 +1268 card card 3 40 7 otherprop Objects objects 39 +1269 mug mug 3 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 +188 shower doors shower door 3 38 7 otherstructure Objects n04208936 shower.n.01 door 4 +689 cardboard cardboard 3 40 7 otherprop Objects objects 39 +1270 rack stand rack stand 3 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +1271 boxes of paper boxes of paper 3 29 7 box box Objects n02883344 box.n.01 objects 39 +1272 flag flag 3 40 7 otherprop Objects misc 40 +354 futon futon 3 39 6 mattress otherfurniture Furniture n03408444 futon.n.01 sofa 10 +339 magazine magazine 3 40 7 magazine otherprop Objects n06595351 magazine.n.01 objects 39 +1009 exit sign exit sign 3 40 7 exit sign otherprop Objects misc 40 +1273 rolled poster rolled poster 3 40 7 otherprop Objects objects 39 +1274 wheel wheel 3 40 7 otherprop Objects objects 39 +15 pictures picture 3 11 8 picture picture Picture n03931044 picture.n.01 picture 6 +1275 blackboard eraser blackboard eraser 3 40 7 eraser otherprop Objects n03294833 eraser.n.01 objects 39 +361 organizer organizer 3 40 7 otherprop Objects n03918737 personal_digital_assistant.n.01 objects 39 +1276 doll doll 3 40 7 toy otherprop Objects n03219135 doll.n.01 objects 39 +326 book rack book rack 3 39 6 bookrack otherfurniture Furniture objects 39 +1277 laundry bag laundry bag 3 40 7 laundry basket otherprop Objects basket 2801938 n03050864 clothes_hamper.n.01 objects 39 +1278 sponge sponge 3 40 7 otherprop Objects n01906749 sponge.n.04 objects 39 +116 seating seat 3 39 6 furniture otherfurniture Furniture n04161981 seat.n.03 furniture 36 +1184 folded chairs folded chair 2 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +1279 lotion bottle lotion bottle 2 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +212 can can 2 40 7 can otherprop Objects can 2946921 n02946921 can.n.01 objects 39 +1280 lunch box lunch box 2 40 7 otherprop Objects objects 39 +1281 food display food display 2 40 7 otherprop Objects misc 40 +794 storage shelf storage shelf 2 40 7 otherprop Objects shelving 31 +1282 sliding wood door sliding wood door 2 40 7 otherprop Objects door 4 +955 pants pants 2 40 7 otherprop Objects n04489008 trouser.n.01 clothes 38 +387 wood wood 2 40 7 otherprop Objects misc 40 +69 boards board 2 38 7 board otherstructure Objects board_panel 35 +65 bottles bottle 2 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +523 washcloth washcloth 2 40 7 otherprop Objects n04554523 washcloth.n.01 towel 20 +389 workbench workbench 2 39 6 bench otherfurniture Furniture bench table 4379243 n04600486 workbench.n.01 table 5 +29 open kitchen cabinet kitchen cabinet 2 3 6 cabinet cabinet Furniture n02933112 cabinet.n.01 cabinet 7 +1283 organizer shelf organizer shelf 2 15 6 shelves shelves Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +146 frame frame 2 38 7 otherstructure Objects misc 40 +130 cups cup 2 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 +372 exercise ball exercise ball 2 40 7 ball otherprop Objects n04285146 sports_equipment.n.01 gym_equipment 33 +289 easel easel 2 39 6 stand otherfurniture Furniture n03262809 easel.n.01 furniture 36 +440 garbage bag garbage bag 2 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +321 roomba roomba 2 40 7 otherprop Objects objects 39 +976 garage door garage door 2 38 7 garage door otherstructure Objects door door 4 +1256 luggage rack luggage stand 2 39 6 stand otherfurniture Furniture n04038440 shelving 31 +1284 bike lock bike lock 2 40 7 otherprop Objects objects 39 +1285 briefcase briefcase 2 40 7 otherprop Objects n02900705 briefcase.n.01 objects 39 +357 hand towel hand towel 2 27 7 towel towel Objects n03490006 hand_towel.n.01 towel 20 +1286 bath products bath product 2 40 7 otherprop Objects objects 39 +1287 star star 2 40 7 otherprop Objects n09444783 star.n.03 misc 40 +365 map map 2 40 7 map otherprop Objects n03720163 map.n.01 misc 40 +1288 coffee bean bag coffee bean bag 2 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +81 headboard headboard 2 39 6 headboard otherfurniture Furniture n03502200 headboard.n.01 bed 11 +1289 ipad ipad 2 40 7 otherprop Objects objects 39 +1290 display rack display rack 2 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +948 traffic cone traffic cone 2 40 7 cone otherprop Objects cone objects 39 +174 toiletry toiletry 2 40 7 otherprop Objects n04447443 toiletry.n.01 objects 39 +1028 canopy canopy 2 40 7 otherprop Objects misc 40 +1291 massage chair massage chair 2 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +1292 paper organizer paper organizer 2 40 7 otherprop Objects objects 39 +1005 barricade barricade 2 40 7 otherprop Objects misc 40 +235 platform platform 2 38 7 otherstructure Objects misc 40 +1293 cap cap 2 40 7 hat otherprop Objects n03497657 hat.n.01 clothes 38 +1294 dumbbell plates dumbbell plates 2 40 7 otherprop Objects objects 39 +1295 elevator elevator 2 38 7 otherstructure Objects misc 40 +1296 cooking pan cooking pan 2 40 7 pan otherprop Objects n03880531 pan.n.01 objects 39 +1297 trash bag trash bag 2 37 7 bag bag Objects objects 39 +1298 santa santa 2 40 7 otherprop Objects misc 40 +1299 jewelry box jewelry box 2 29 7 box box Objects n02883344 box.n.01 objects 39 +1300 boat boat 2 40 7 otherprop Objects misc 40 +1301 sock sock 2 21 7 clothes clothes Objects n04254777 sock.n.01 clothes 38 +1051 kinect kinect 2 40 7 kinect otherprop Objects objects 39 +566 crib crib 2 39 6 crib otherfurniture Furniture furniture 36 +1302 plastic storage bin plastic storage bin 2 40 7 container otherprop Objects n03094503 container.n.01 objects 39 +1062 cooler cooler 2 24 6 refridgerator refridgerator Furniture n03102654 cooler.n.01 appliances 37 +1303 kitchen apron kitchen apron 2 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +1304 dishwashing soap bottle dishwashing soap bottle 2 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +1305 xbox controller xbox controller 2 40 7 otherprop Objects objects 39 +1306 banana holder banana holder 2 40 7 otherprop Objects objects 39 +298 ping pong paddle ping pong paddle 2 40 7 otherprop Objects table 5 +1307 airplane airplane 2 40 7 otherprop Objects misc 40 +1308 conditioner bottle conditioner bottle 2 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +1309 tea kettle tea kettle 2 40 7 tea kettle otherprop Objects n04397768 teakettle.n.01 objects 39 +43 bedframe bedframe 2 39 6 otherfurniture Furniture n02822579 bedstead.n.01 bed 11 +1310 wood beam wood beam 2 38 7 otherstructure Objects beam 29 +593 toilet paper package toilet paper package 2 40 7 otherprop Objects objects 39 +1311 wall mounted coat rack wall mounted coat rack 2 40 7 otherprop Objects n03059103 coatrack.n.01 shelving 31 +1312 film light film light 2 40 7 otherprop Objects lighting 28 +749 ceiling lamp ceiling lamp 1 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +623 chain chain 1 40 7 otherprop Objects chair 3 +1313 sofa sofa 1 6 9 sofa sofa Sofa sofa sofa sofa 4256520 n04256520 sofa.n.01 sofa 10 +99 closet wardrobe wardrobe 1 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 furniture 36 +265 sweater sweater 1 40 7 otherprop Objects n04370048 sweater.n.01 clothes 38 +1314 kitchen mixer kitchen mixer 1 40 7 otherprop Objects appliances 37 +99 wardrobe wardrobe 1 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 furniture 36 +1315 water softener water softener 1 40 7 otherprop Objects misc 40 +448 banister banister 1 38 7 banister otherstructure Objects n02788148 bannister.n.02 railing 30 +257 trolley trolley 1 40 7 trolley otherprop Objects n04335435 streetcar.n.01 misc 40 +1316 pantry shelf pantry shelf 1 15 6 shelves shelves Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +786 sofa bed sofa bed 1 4 1 bed bed Bed bed bed bed 2818832 n02818832 bed.n.01 bed 11 +801 loofa loofa 1 40 7 otherprop Objects objects 39 +972 shower faucet handle shower faucet handle 1 40 7 handle otherprop Objects shower 23 +1317 toy piano toy piano 1 40 7 toy otherprop Objects n03964744 plaything.n.01 objects 39 +1318 fish fish 1 40 7 otherprop Objects n02512053 fish.n.01 objects 39 +75 file cabinets file cabinet 1 3 6 cabinet cabinet Furniture cabinet 2933112 n03337140 file.n.03 cabinet 7 +657 cat litter box cat litter box 1 29 7 box box Objects objects 39 +561 electric panel electric panel 1 40 7 otherprop Objects misc 40 +93 suitcases suitcase 1 40 7 luggage otherprop Objects n02774630 baggage.n.01 objects 39 +513 curtain rod curtain rod 1 38 7 curtain rod otherstructure Objects curtain 12 +411 bunk bed bunk bed 1 39 6 bunk bed otherfurniture Furniture bed bed bed 2818832 n02920259 bunk_bed.n.01 bed 11 +1122 chandelier chandelier 1 38 7 chandelier otherstructure Objects n03005285 chandelier.n.01 lighting 28 +922 tape tape 1 40 7 tape otherprop Objects objects 39 +88 plates plate 1 40 7 otherprop Objects n03959485 plate.n.04 objects 39 +518 alarm alarm 1 40 7 alarm otherprop Objects clock 3046257 n02694662 alarm_clock.n.01 objects 39 +814 fire hose fire hose 1 40 7 otherprop Objects n03346004 fire_hose.n.01 misc 40 +1319 toy dinosaur toy dinosaur 1 40 7 toy otherprop Objects n03964744 plaything.n.01 objects 39 +1320 cone cone 1 40 7 otherprop Objects objects 39 +649 glass doors glass door 1 8 12 door door Wall door n03221720 door.n.01 door 4 +607 hatrack hatrack 1 40 7 otherprop Objects n03059103 coatrack.n.01 shelving 31 +819 subwoofer subwoofer 1 40 7 speaker otherprop Objects speaker 3691459 n04349401 subwoofer.n.01 objects 39 +1321 fire sprinkler fire sprinkler 1 40 7 otherprop Objects misc 40 +1322 trash cabinet trash cabinet 1 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +1204 pantry walls pantry wall 1 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +227 photo photo 1 40 7 photo otherprop Objects n03925226 photograph.n.01 picture 6 +817 barrier barrier 1 40 7 otherprop Objects n02796623 barrier.n.01 misc 40 +130 stacks of cups cup 1 40 7 otherprop Objects n03147509 cup.n.01 objects 39 +712 beachball beachball 1 40 7 ball otherprop Objects n02814224 beach_ball.n.01 objects 39 +1323 folded boxes folded boxes 1 40 7 otherprop Objects objects 39 +1324 contact lens solution bottle contact lens solution bottle 1 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +673 covered box covered box 1 29 7 box box Objects objects 39 +459 folder folder 1 40 7 folder otherprop Objects n03376279 folder.n.02 objects 39 +643 mail trays mail tray 1 40 7 mail tray otherprop Objects objects 39 +238 slipper slipper 1 40 7 otherprop Objects n04241394 slipper.n.01 clothes 38 +765 magazine rack magazine rack 1 39 6 stand otherfurniture Furniture n03704549 magazine_rack.n.01 shelving 31 +1008 sticker sticker 1 40 7 sticker otherprop Objects n07272545 gummed_label.n.01 objects 39 +225 lotion lotion 1 40 7 otherprop Objects n03690938 lotion.n.01 objects 39 +1083 buddha buddha 1 40 7 otherprop Objects objects 39 +813 file organizer file organizer 1 40 7 otherprop Objects objects 39 +138 paper towel rolls paper towel roll 1 40 7 paper towel otherprop Objects n03887697 paper_towel.n.01 towel 20 +1145 night lamp night lamp 1 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +796 fuse box fuse box 1 40 7 otherprop Objects misc 40 +1325 knife block knife block 1 40 7 otherprop Objects objects 39 +363 furnace furnace 1 39 6 furnace otherfurniture Furniture n03404449 furnace.n.01 +1174 cd cases cd case 1 40 7 otherprop Objects objects 39 +38 stools stool 1 40 7 stool otherprop Objects stool n04326896 stool.n.01 stool 19 +1326 hand sanitzer dispenser hand sanitzer dispenser 1 40 7 otherprop Objects n04254120 soap_dispenser.n.01 objects 39 +997 teapot teapot 1 40 7 tea pot otherprop Objects n04398044 teapot.n.01 objects 39 +1327 pen holder pen holder 1 40 7 otherprop Objects objects 39 +1328 tray rack tray rack 1 40 7 otherprop Objects objects 39 +1329 wig wig 1 40 7 otherprop Objects n04584207 wig.n.01 objects 39 +182 switch switch 1 40 7 otherprop Objects n04372370 switch.n.01 misc 40 +280 plastic containers plastic container 1 40 7 container otherprop Objects n03094503 container.n.01 objects 39 +1330 night light night light 1 40 7 otherprop Objects lighting 28 +1331 notepad notepad 1 40 7 otherprop Objects objects 39 +1332 mail bin mail bin 1 40 7 otherprop Objects misc 40 +1333 elevator button elevator button 1 40 7 otherprop Objects misc 40 +939 gaming wheel gaming wheel 1 40 7 otherprop Objects objects 39 +1334 drum set drum set 1 40 7 otherprop Objects objects 39 +480 cosmetic bag cosmetic bag 1 37 7 bag bag Objects objects 39 +907 coffee mug coffee mug 1 40 7 vessel otherprop Objects cup or mug 3797390 n03063599 coffee_mug.n.01 objects 39 +1335 closet shelf closet shelf 1 15 6 shelves shelves Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +1336 baby mobile baby mobile 1 40 7 otherprop Objects objects 39 +829 diaper bin diaper bin 1 40 7 bin otherprop Objects objects 39 +947 door wall door wall 1 1 12 wall wall Wall wall 1 +1116 stepstool stepstool 1 40 7 step stool otherprop Objects objects 39 +599 paper shredder shredder 1 40 7 otherprop Objects n04210120 shredder.n.01 objects 39 +733 dress rack dress rack 1 40 7 otherprop Objects n03238762 dress_rack.n.01 misc 40 +123 cover cover 1 40 7 blanket otherprop Objects objects 39 +506 shopping bag shopping bag 1 37 7 bag bag Objects n04204081 shopping_bag.n.01 objects 39 +569 sliding door sliding door 1 8 12 door door Wall door n04239074 sliding_door.n.01 door 4 +1337 exercise bike exercise bike 1 40 7 machine otherprop Objects n04210120 shredder.n.01 gym_equipment 33 +1338 recliner chair recliner chair 1 5 4 chair chair Chair chair chair chair 3001627 n03238762 dress_rack.n.01 chair 3 +1314 kitchenaid mixer kitchen mixer 1 40 7 otherprop Objects appliances 37 +1339 soda can soda can 1 40 7 can otherprop Objects can 2946921 n02946921 can.n.01 objects 39 +1340 stovetop stovetop 1 38 7 stove otherstructure Objects stove 4330267 n04330267 stove.n.02 appliances 37 +851 stepladder stepladder 1 39 6 ladder otherfurniture Furniture stairs n04315599 step_ladder.n.01 stairs 16 +142 tap tap 1 40 7 faucet otherprop Objects faucet 3325088 n04559451 water_faucet.n.01 objects 39 +436 cable cable 1 40 7 cables otherprop Objects objects 39 +1341 baby changing station baby changing station 1 39 6 otherfurniture Furniture furniture 36 +1342 costume costume 1 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +885 rocking chair rocking chair 1 5 4 chair chair Chair chair chair chair 3001627 n04099969 rocking_chair.n.01 chair 3 +693 binder binder 1 40 7 binder otherprop Objects objects 39 +815 media center media center 1 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +401 towel rack towel rack 1 40 7 otherprop Objects n04459773 towel_rack.n.01 misc 40 +1343 medal medal 1 40 7 otherprop Objects objects 39 +1184 stack of folded chairs folded chair 1 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +1344 telescope telescope 1 40 7 otherprop Objects n04403638 telescope.n.01 objects 39 +1345 closet doorframe closet doorframe 1 8 12 door door Wall door door 4 +160 glass glass 1 38 7 glass otherstructure Objects n03438257 glass.n.02 misc 40 +1126 baseball cap baseball cap 1 40 7 otherprop Objects cap 2954340 n02799323 baseball_cap.n.01 clothes 38 +1346 battery disposal jar battery disposal jar 1 40 7 jar otherprop Objects jar 3593526 n03593526 jar.n.01 objects 39 +332 mop mop 1 40 7 otherprop Objects n04367480 swab.n.02 objects 39 +397 tank tank 1 40 7 otherprop Objects objects 39 +643 mail tray mail tray 1 40 7 mail tray otherprop Objects objects 39 +551 centerpiece centerpiece 1 40 7 centerpiece otherprop Objects n02994419 centerpiece.n.02 objects 39 +1163 stick object 1 40 7 stick otherprop Objects objects 39 +1347 closet floor closet floor 1 2 5 floor floor Floor n03365592 floor.n.01 floor 2 +1348 dryer sheets dryer sheets 1 40 7 otherprop Objects objects 39 +803 bycicle bycicle 1 40 7 otherprop Objects misc 40 +484 flower stand flower stand 1 39 6 stand otherfurniture Furniture furniture 36 +1349 air mattress air mattress 1 4 1 bed bed Bed bed bed bed 2818832 n02690809 air_mattress.n.01 bed 11 +1350 clip clip 1 40 7 otherprop Objects objects 39 +222 side table side table 1 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +1253 pizza boxes pizza box 1 29 7 box box Objects n02883344 box.n.01 objects 39 +1351 display display 1 39 7 otherfurniture Furniture n03211117 display.n.06 misc 40 +1352 postcard postcard 1 40 7 otherprop Objects objects 39 +828 display sign display sign 1 40 7 sign otherprop Objects misc 40 +1353 paper towel paper towel 1 40 7 paper towel otherprop Objects n03887697 paper_towel.n.01 towel 20 +612 boots boot 1 40 7 shoe otherprop Objects n04199027 shoe.n.01 clothes 38 +1354 tennis racket bag tennis racket bag 1 40 7 otherprop Objects objects 39 +1355 air hockey table air hockey table 1 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +1301 socks sock 1 21 7 clothes clothes Objects n04254777 sock.n.01 clothes 38 +1356 food bag food bag 1 37 7 bag bag Objects objects 39 +1199 clothes hangers clothes hanger 1 40 7 otherprop Objects n03057920 coat_hanger.n.01 misc 40 +1357 starbucks cup starbucks cup 1 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 \ No newline at end of file diff --git a/src/datasets/megadepth.py b/src/datasets/megadepth.py new file mode 100644 index 0000000000000000000000000000000000000000..90b4eea3c761b1a3ad5cb8a5e578b9eb6c9a90d7 --- /dev/null +++ b/src/datasets/megadepth.py @@ -0,0 +1,125 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed MegaDepth +# dataset at https://www.cs.cornell.edu/projects/megadepth/ +# See datasets_preprocess/preprocess_megadepth.py +# -------------------------------------------------------- +import os.path as osp +import numpy as np +import sys +sys.path.append("submodules/mast3r/dust3r") +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class MegaDepth(BaseStereoViewDataset): + def __init__(self, *args, split, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + self.num_views = 3 # render third view + self.loaded_data = self._load_data(self.split) + + if self.split is None: + pass + elif self.split == 'train': + self.select_scene(('0015', '0022'), opposite=True) + elif self.split == 'val': + self.select_scene(('0015', '0022')) + else: + raise ValueError(f'bad {self.split=}') + + def _load_data(self, split): + with np.load(osp.join(self.ROOT, 'all_metadata.npz')) as data: + self.all_scenes = data['scenes'] + self.all_images = data['images'] + self.pairs = data['pairs'] + + def __len__(self): + return len(self.pairs) + + def get_stats(self): + return f'{len(self)} pairs from {len(self.all_scenes)} scenes' + + def select_scene(self, scene, *instances, opposite=False): + scenes = (scene,) if isinstance(scene, str) else tuple(scene) + scene_id = [s.startswith(scenes) for s in self.all_scenes] + assert any(scene_id), 'no scene found' + + valid = np.in1d(self.pairs['scene_id'], np.nonzero(scene_id)[0]) + if instances: + image_id = [i.startswith(instances) for i in self.all_images] + image_id = np.nonzero(image_id)[0] + assert len(image_id), 'no instance found' + # both together? + if len(instances) == 2: + valid &= np.in1d(self.pairs['im1_id'], image_id) & np.in1d(self.pairs['im2_id'], image_id) + else: + valid &= np.in1d(self.pairs['im1_id'], image_id) | np.in1d(self.pairs['im2_id'], image_id) + + if opposite: + valid = ~valid + assert valid.any() + self.pairs = self.pairs[valid] + + def _get_views(self, pair_idx, resolution, rng): + scene_id, im1_id, im2_id, score = self.pairs[pair_idx] + im3_id = int((im1_id + im2_id) / 2) + scene, subscene = self.all_scenes[scene_id].split() + seq_path = osp.join(self.ROOT, scene, subscene) + + views = [] + + for im_id in [im1_id, im2_id, im2_id]: + img = self.all_images[im_id] + try: + image = imread_cv2(osp.join(seq_path, img + '.jpg')) + depthmap = imread_cv2(osp.join(seq_path, img + ".exr")) + camera_params = np.load(osp.join(seq_path, img + ".npz")) + except Exception as e: + raise OSError(f'cannot load {img}, got exception {e}') + + intrinsics = np.float32(camera_params['intrinsics']) + camera_pose = np.float32(camera_params['cam2world']) + + image, depthmap, intrinsics = self._crop_resize_if_necessary( + image, depthmap, intrinsics, resolution, rng, info=(seq_path, img)) + + views.append(dict( + img=image, + depthmap=depthmap, + camera_pose=camera_pose, # cam2world + camera_intrinsics=intrinsics, + dataset='MegaDepth', + label=osp.relpath(seq_path, self.ROOT), + instance=img)) + + return views + + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = MegaDepth(split='train', ROOT="data/megadepth_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 3 + print(idx, view_name(views[0]), view_name(views[1]), view_name(views[2])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1, 2]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1, 2]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() \ No newline at end of file diff --git a/src/datasets/scannet.py b/src/datasets/scannet.py new file mode 100644 index 0000000000000000000000000000000000000000..73479d2140a0b9e029f402e97f35c806671db8d2 --- /dev/null +++ b/src/datasets/scannet.py @@ -0,0 +1,109 @@ +import os +import os.path as osp +import sys +sys.path.append("submodules/mast3r/dust3r") +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +import numpy as np +import cv2 +from dust3r.utils.image import imread_cv2 + +class Scannet(BaseStereoViewDataset): + def __init__(self, *args, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + self.num_views = 3 # render third view + self._load_data() + + def _load_data(self): + # Traverse all the folders in the data_root + scene_names = [folder for folder in os.listdir(self.ROOT) if os.path.isdir(os.path.join(self.ROOT, folder))] + # Filter out scenes without scene_data.npz + valid_scenes = [] + for scene_name in scene_names: + scene_data_path = osp.join(self.ROOT, scene_name, "scene_data.npz") + if osp.exists(scene_data_path): + valid_scenes.append(scene_name) + else: + print(f"Skipping {scene_name}: scene_data.npz not found") + scene_names = valid_scenes + scene_names.sort() + if self.split == 'train': + scene_names = scene_names[:-150] + else: + scene_names = scene_names[-150:] + # merge all pairs and images + pairs = [] # (scene_name, image_idx1, image_idx2) + images = {} # (scene_name, image_idx) -> image_path + for scene_name in scene_names: + scene_path = osp.join(self.ROOT, scene_name, "scene_data.npz") + scene_data = np.load(scene_path) + pairs.extend([(scene_name, *pair) for pair in scene_data['pairs']]) + images.update({(scene_name, idx): path for idx, path in enumerate(scene_data['images'])}) + self.pairs = pairs + self.images = images + + def __len__(self): + return len(self.pairs) + + def _get_views(self, idx, resolution, rng): + scene_name, image_idx1, image_idx2, _ = self.pairs[idx] + image_idx1 = int(image_idx1) + image_idx2 = int(image_idx2) + image_idx3 = int((image_idx1 + image_idx2) / 2) + views = [] + for view_idx in [image_idx1, image_idx2, image_idx3]: + basename = self.images[(scene_name, view_idx)] + # Load RGB image + rgb_path = osp.join(self.ROOT, scene_name, 'images', f'{basename}.jpg') + rgb_image = imread_cv2(rgb_path) + # Load depthmap + depthmap_path = osp.join(self.ROOT, scene_name, 'depths', f'{basename}.png') + depthmap = imread_cv2(depthmap_path, cv2.IMREAD_UNCHANGED) + depthmap = depthmap.astype(np.float32) / 1000 + depthmap[~np.isfinite(depthmap)] = 0 # invalid + # Load camera parameters + meta_path = osp.join(self.ROOT, scene_name, 'images', f'{basename}.npz') + meta = np.load(meta_path) + intrinsics = meta['camera_intrinsics'] + camera_pose = meta['camera_pose'] + # crop if necessary + rgb_image, depthmap, intrinsics = self._crop_resize_if_necessary( + rgb_image, depthmap, intrinsics, resolution, rng=rng, info=view_idx) + views.append(dict( + img=rgb_image, + depthmap=depthmap.astype(np.float32), + camera_pose=camera_pose.astype(np.float32), + camera_intrinsics=intrinsics.astype(np.float32), + dataset='ScanNet', + label=scene_name + '_' + basename, + instance=f'{str(idx)}_{str(view_idx)}', + )) + return views + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = Scannet(split='train', ROOT="data/scannet_processed", resolution=224, aug_crop=16) + + print(len(dataset)) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 3 + print(view_name(views[0]), view_name(views[1]), view_name(views[2])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1, 2]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1, 2]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx*255, (1 - idx)*255, 0), + image=colors, + cam_size=cam_size) + viz.show() \ No newline at end of file diff --git a/src/datasets/scannetpp.py b/src/datasets/scannetpp.py new file mode 100644 index 0000000000000000000000000000000000000000..f39edfbf6bad53b72aca5eb38e218be8a36b0fbe --- /dev/null +++ b/src/datasets/scannetpp.py @@ -0,0 +1,107 @@ +import os +import os.path as osp +import sys +sys.path.append("submodules/mast3r/dust3r") +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +import numpy as np +import cv2 +from dust3r.utils.image import imread_cv2 + +class Scannetpp(BaseStereoViewDataset): + def __init__(self, *args, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + assert self.split == 'train' # just for training + self.num_views = 3 # render third view + self._load_data() + + def _load_data(self): + # Traverse all the folders in the data_root + scene_names = [folder for folder in os.listdir(self.ROOT) if os.path.isdir(os.path.join(self.ROOT, folder))] + # Filter out scenes without scene_data.npz + valid_scenes = [] + for scene_name in scene_names: + scene_data_path = osp.join(self.ROOT, scene_name, "scene_data.npz") + if osp.exists(scene_data_path): + valid_scenes.append(scene_name) + else: + print(f"Skipping {scene_name}: scene_data.npz not found") + scene_names = valid_scenes + scene_names.sort() + + # merge all pairs and images + pairs = [] # (scene_name, image_idx1, image_idx2) + images = {} # (scene_name, image_idx) -> image_path + for scene_name in scene_names: + scene_path = osp.join(self.ROOT, scene_name, "scene_data.npz") + scene_data = np.load(scene_path) + pairs.extend([(scene_name, *pair) for pair in scene_data['pairs']]) + images.update({(scene_name, idx): path for idx, path in enumerate(scene_data['images'])}) + self.pairs = pairs + self.images = images + + def __len__(self): + return len(self.pairs) + + def _get_views(self, idx, resolution, rng): + scene_name, image_idx1, image_idx2, _ = self.pairs[idx] + image_idx1 = int(image_idx1) + image_idx2 = int(image_idx2) + image_idx3 = int((image_idx1 + image_idx2) / 2) + views = [] + for view_idx in [image_idx1, image_idx2, image_idx3]: + basename = self.images[(scene_name, view_idx)] + # Load RGB image + rgb_path = osp.join(self.ROOT, scene_name, 'images', f'{basename}.JPG') + rgb_image = imread_cv2(rgb_path) + # Load depthmap + depthmap_path = osp.join(self.ROOT, scene_name, 'depths', f'{basename}.png') + depthmap = imread_cv2(depthmap_path, cv2.IMREAD_UNCHANGED) + depthmap = depthmap.astype(np.float32) / 1000 + depthmap[~np.isfinite(depthmap)] = 0 # invalid + # Load camera parameters + meta_path = osp.join(self.ROOT, scene_name, 'images', f'{basename}.npz') + meta = np.load(meta_path) + intrinsics = meta['camera_intrinsics'] + camera_pose = meta['camera_pose'] + # crop if necessary + rgb_image, depthmap, intrinsics = self._crop_resize_if_necessary( + rgb_image, depthmap, intrinsics, resolution, rng=rng, info=view_idx) + views.append(dict( + img=rgb_image, + depthmap=depthmap.astype(np.float32), + camera_pose=camera_pose.astype(np.float32), + camera_intrinsics=intrinsics.astype(np.float32), + dataset='ScanNet++', + label=scene_name + '_' + basename, + instance=f'{str(idx)}_{str(view_idx)}', + )) + return views + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = Scannetpp(split='train', ROOT="data/scannetpp_processed", resolution=224, aug_crop=16) + + print(len(dataset)) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 3 + print(view_name(views[0]), view_name(views[1]), view_name(views[2])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1, 2]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1, 2]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx*255, (1 - idx)*255, 0), + image=colors, + cam_size=cam_size) + viz.show() \ No newline at end of file diff --git a/src/datasets_preprocess/scannet_preprocess.py b/src/datasets_preprocess/scannet_preprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..3ff01696d5c0f4b9312c10cb43e237acc58da6aa --- /dev/null +++ b/src/datasets_preprocess/scannet_preprocess.py @@ -0,0 +1,209 @@ +import os +import numpy as np +import cv2 +import torch +import torch.multiprocessing as mp + +def process_scene_on_gpu(gpu_id, scene_names, data_root, output_queue): + torch.cuda.set_device(gpu_id) + local_pairs = {} + local_images = {} + + for scene_name in scene_names: + save_path = os.path.join(data_root, scene_name, "scene_data.npz") + if os.path.exists(save_path): + print(f"Scene {scene_name} already processed, skipping") + continue + pairs, images = process_scene(data_root, scene_name) + np.savez_compressed(save_path, pairs=pairs, images=images) + + output_queue.put((local_pairs, local_images)) + +def preprocess_scannet(data_root, threads_per_gpu=4): + scene_names = [folder for folder in os.listdir(data_root) if os.path.isdir(os.path.join(data_root, folder))] + num_gpus = torch.cuda.device_count() + total_threads = num_gpus * threads_per_gpu + + # 将场景平均分配给所有线程 + scenes_per_thread = [scene_names[i::total_threads] for i in range(total_threads)] + + output_queue = mp.Queue() + processes = [] + + # 为每个GPU创建多个进程 + for gpu_id in range(num_gpus): + for thread_id in range(threads_per_gpu): + process_id = gpu_id * threads_per_gpu + thread_id + p = mp.Process( + target=process_scene_on_gpu, + args=(gpu_id, scenes_per_thread[process_id], data_root, output_queue) + ) + p.start() + processes.append(p) + + # 收集所有进程的结果 + all_pairs = {} + all_images = {} + for _ in range(total_threads): + local_pairs, local_images = output_queue.get() + all_pairs.update(local_pairs) + all_images.update(local_images) + + # Wait for all processes to complete + for p in processes: + p.join() + + # Save to npz file + np.savez_compressed(os.path.join(data_root, "scannet_image_pairs.npz"), **all_pairs) + np.savez_compressed(os.path.join(data_root, "scannet_images.npz"), **all_images) + + # print the number of image pairs + # sum up the number of image pairs for all scenes + total_pairs = sum(len(pairs) for pairs in all_pairs.values()) + print(f"Total number of image pairs: {total_pairs}") + return all_pairs, all_images + +def process_scene(data_root, scene_name): + pairs = [] + images_dir = os.path.join(data_root, scene_name, "images") + images = [os.path.splitext(file)[0] for file in os.listdir(images_dir) if file.endswith(".jpg")] + images.sort() + + # Check validity of c2w for each image + valid_images = [] + for image in images: + _, c2w, _ = load_image(data_root, scene_name, image) + if is_valid_c2w(c2w): + valid_images.append(image) + else: + print(f"Invalid c2w for image {image} in scene {scene_name}") + + # generate image pairs + slide_window = 50 + num_sub_intervals = 5 + + pairs = generate_image_pairs(data_root, scene_name, valid_images, slide_window, num_sub_intervals) + print(f"Scene {scene_name} has {len(pairs)} image pairs and {len(valid_images)} valid images out of {len(images)} total images") + return pairs, valid_images + +def is_valid_c2w(c2w): + return not np.any(np.isinf(c2w)) and not np.any(np.isnan(c2w)) + +def generate_image_pairs(data_root, scene_name, images, slide_window, num_sub_intervals=3): + pairs = [] + n = len(images) + + # Define IOU sub-intervals + iou_range = (0.3, 0.8) + sub_interval_size = (iou_range[1] - iou_range[0]) / num_sub_intervals + sub_intervals = [(iou_range[0] + i * sub_interval_size, iou_range[0] + (i + 1) * sub_interval_size) + for i in range(num_sub_intervals)] + + for i in range(n): + # Keep track of whether a pair has been added for each sub-interval + interval_selected = [False] * num_sub_intervals + + for j in range(i+1, min(i + slide_window, n)): + # Break early if all sub-intervals have been selected + if all(interval_selected): + break + + # Load image pair + depth1, c2w1, K1 = load_image(data_root, scene_name, images[i]) + depth2, c2w2, K2 = load_image(data_root, scene_name, images[j]) + + # Calculate mean IoU + try: + iou_1 = calculate_iou(depth1, c2w1, K1, depth2, c2w2, K2) + iou_2 = calculate_iou(depth2, c2w2, K2, depth1, c2w1, K1) + except Exception as e: + print(f"Error calculating IoU for images {images[i]} and {images[j]} in scene {scene_name}: {str(e)}") + continue + + mean_iou = (iou_1 + iou_2) / 2 + + # Check which sub-interval the mean IoU falls into + for idx, (lower, upper) in enumerate(sub_intervals): + if lower <= mean_iou <= upper and not interval_selected[idx]: + pairs.append((i, j, mean_iou)) + interval_selected[idx] = True # Mark this interval as selected + break # Move to the next pair after adding one in the current sub-interval + + return pairs + + +def load_image(data_root, scene_name, image_id): + # load depthmap + depth_path = f"{data_root}/{scene_name}/depths/{image_id}.png" + depth = cv2.imread(depth_path, cv2.IMREAD_UNCHANGED).astype(np.float32) / 1000.0 + # load camera parameters + meta_path = f"{data_root}/{scene_name}/images/{image_id}.npz" + meta = np.load(meta_path) + c2w = meta['camera_pose'] + K = meta['camera_intrinsics'] + return depth, c2w, K + +# Unproject depthmap to point cloud and project to another camera +def calculate_iou(depth1, c2w1, K1, depth2, c2w2, K2): + # Move data to GPU and ensure float32 dtype + depth1 = torch.from_numpy(depth1).cuda().float() + depth2 = torch.from_numpy(depth2).cuda().float() + c2w1 = torch.from_numpy(c2w1).cuda().float() + c2w2 = torch.from_numpy(c2w2).cuda().float() + K1 = torch.from_numpy(K1).cuda().float() + K2 = torch.from_numpy(K2).cuda().float() + + # Get image dimensions + h, w = depth1.shape + + # Create pixel coordinates + y, x = torch.meshgrid(torch.arange(h, device='cuda', dtype=torch.float32), + torch.arange(w, device='cuda', dtype=torch.float32)) + pixels = torch.stack((x.flatten(), y.flatten(), torch.ones_like(x.flatten())), dim=-1).T + + # Unproject pixels to 3D points + pixels_3d = torch.linalg.inv(K1) @ pixels + pixels_3d *= depth1.flatten().unsqueeze(0) + + # Transform 3D points to world coordinates + pixels_world = c2w1[:3, :3] @ pixels_3d + c2w1[:3, 3:4] + + # Check if c2w2[:3, :3] is invertible + if torch.det(c2w2[:3, :3]) == 0: + return 0, False # Calculation failed + + # Project world points to second camera + pixels_cam2 = torch.linalg.inv(c2w2[:3, :3]) @ (pixels_world - c2w2[:3, 3:4]) + pixels_img2 = K2 @ pixels_cam2 + + # Normalize homogeneous coordinates + pixels_img2 = pixels_img2[:2] / pixels_img2[2] + pixels_img2 = pixels_img2.T + + # Filter valid pixels + valid_mask = (pixels_img2[:, 0] >= 0) & (pixels_img2[:, 0] < w) & \ + (pixels_img2[:, 1] >= 0) & (pixels_img2[:, 1] < h) + + pixels_img2 = pixels_img2[valid_mask].long() + + # Compare depths + projected_depth = pixels_cam2[2, valid_mask] + actual_depth = depth2[pixels_img2[:, 1], pixels_img2[:, 0]] + + depth_diff = torch.abs(projected_depth - actual_depth) + depth_threshold = 0.1 # 10cm threshold + + overlap_mask = depth_diff < depth_threshold + + # Calculate IoU + intersection = torch.sum(overlap_mask) + union = torch.sum(valid_mask) + torch.sum(depth2 > 0) - intersection + + iou = intersection.float() / union.float() if union > 0 else torch.tensor(0.0, device='cuda') + + return iou.item() + +if __name__ == "__main__": + data_root = "data/scannet_processed" + # 可以通过参数指定每个GPU的线程数 + preprocess_scannet(data_root, threads_per_gpu=12) diff --git a/src/datasets_preprocess/scannetpp_preprocess.py b/src/datasets_preprocess/scannetpp_preprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..df01b9436594bc2f39e9ba2b7bda0fa54a7a091f --- /dev/null +++ b/src/datasets_preprocess/scannetpp_preprocess.py @@ -0,0 +1,227 @@ +import os +import numpy as np +import cv2 +import torch +import torch.multiprocessing as mp +import shutil + +def process_scene_on_gpu(gpu_id, scene_names, data_root, target_root, output_queue): + torch.cuda.set_device(gpu_id) + local_pairs = {} + local_images = {} + + for scene_name in scene_names: + save_path = os.path.join(target_root, scene_name, "scene_data.npz") + if os.path.exists(save_path): + print(f"Scene {scene_name} already processed, skipping") + continue + pairs, images = process_scene(data_root, target_root, scene_name) + np.savez_compressed(save_path, pairs=pairs, images=images) + + output_queue.put((local_pairs, local_images)) + +def preprocess_scannetpp(data_root, target_root): + # Traverse all the folders in the data_root + scene_names = [folder for folder in os.listdir(data_root) if os.path.isdir(os.path.join(data_root, folder))] + + # Get the number of available GPUs + num_gpus = torch.cuda.device_count() + + # Distribute scenes across GPUs + scenes_per_gpu = [scene_names[i::num_gpus] for i in range(num_gpus)] + + # Create a multiprocessing queue to collect results + output_queue = mp.Queue() + + # Launch parallel processes + processes = [] + for gpu_id in range(num_gpus): + p = mp.Process(target=process_scene_on_gpu, args=(gpu_id, scenes_per_gpu[gpu_id], data_root, target_root, output_queue)) + p.start() + processes.append(p) + + # Collect results from all processes + all_pairs = {} + all_images = {} + for _ in range(num_gpus): + local_pairs, local_images = output_queue.get() + all_pairs.update(local_pairs) + all_images.update(local_images) + + # Wait for all processes to complete + for p in processes: + p.join() + + # Save to npz file + np.savez_compressed(os.path.join(data_root, "scannet_image_pairs.npz"), **all_pairs) + np.savez_compressed(os.path.join(data_root, "scannet_images.npz"), **all_images) + + # print the number of image pairs + # sum up the number of image pairs for all scenes + total_pairs = sum(len(pairs) for pairs in all_pairs.values()) + print(f"Total number of image pairs: {total_pairs}") + return all_pairs, all_images + +# def preprocess_scannetpp(data_root, target_root): +# # Traverse all the folders in the data_root +# scene_names = [folder for folder in os.listdir(data_root) if os.path.isdir(os.path.join(data_root, folder))] + +# for scene_name in scene_names: +# save_path = os.path.join(target_root, scene_name, "scene_data.npz") +# if os.path.exists(save_path): +# print(f"Scene {scene_name} already processed, skipping") +# continue +# pairs, images = process_scene(data_root, target_root, scene_name) +# np.savez_compressed(save_path, pairs=pairs, images=images) + +def process_scene(data_root, target_root, scene_name): + pairs = [] + images_dir = os.path.join(data_root, scene_name, "images") + images = [os.path.splitext(file)[0] for file in os.listdir(images_dir) if file.endswith(".JPG")] + images.sort() + # copy images, depths, and camera parameters to target_root + os.makedirs(os.path.join(target_root, scene_name, "images"), exist_ok=True) + os.makedirs(os.path.join(target_root, scene_name, "depths"), exist_ok=True) + for image in images: + shutil.copy(os.path.join(data_root, scene_name, "images", f"{image}.JPG"), os.path.join(target_root, scene_name, "images", f"{image}.JPG")) + shutil.copy(os.path.join(data_root, scene_name, "depths", f"{image}.png"), os.path.join(target_root, scene_name, "depths", f"{image}.png")) + shutil.copy(os.path.join(data_root, scene_name, "images", f"{image}.npz"), os.path.join(target_root, scene_name, "images", f"{image}.npz")) + + # Check validity of c2w for each image + valid_images = [] + for image in images: + _, c2w, _ = load_image(data_root, scene_name, image) + if is_valid_c2w(c2w): + valid_images.append(image) + else: + print(f"Invalid c2w for image {image} in scene {scene_name}") + + # generate image pairs + slide_window = 100 + num_sub_intervals = 5 + + pairs = generate_image_pairs(data_root, scene_name, valid_images, slide_window, num_sub_intervals) + print(f"Scene {scene_name} has {len(pairs)} image pairs and {len(valid_images)} valid images out of {len(images)} total images") + return pairs, valid_images + +def is_valid_c2w(c2w): + return not np.any(np.isinf(c2w)) and not np.any(np.isnan(c2w)) + +def generate_image_pairs(data_root, scene_name, images, slide_window, num_sub_intervals=3): + pairs = [] + n = len(images) + + # Define IOU sub-intervals + iou_range = (0.3, 0.8) + sub_interval_size = (iou_range[1] - iou_range[0]) / num_sub_intervals + sub_intervals = [(iou_range[0] + i * sub_interval_size, iou_range[0] + (i + 1) * sub_interval_size) + for i in range(num_sub_intervals)] + + for i in range(n): + # Keep track of whether a pair has been added for each sub-interval + interval_selected = [False] * num_sub_intervals + + for j in range(i+1, min(i + slide_window, n)): + # Break early if all sub-intervals have been selected + if all(interval_selected): + break + + # Load image pair + depth1, c2w1, K1 = load_image(data_root, scene_name, images[i]) + depth2, c2w2, K2 = load_image(data_root, scene_name, images[j]) + + # Calculate mean IoU + try: + iou_1 = calculate_iou(depth1, c2w1, K1, depth2, c2w2, K2) + iou_2 = calculate_iou(depth2, c2w2, K2, depth1, c2w1, K1) + except Exception as e: + print(f"Error calculating IoU for images {images[i]} and {images[j]} in scene {scene_name}: {str(e)}") + continue + + mean_iou = (iou_1 + iou_2) / 2 + + # Check which sub-interval the mean IoU falls into + for idx, (lower, upper) in enumerate(sub_intervals): + if lower <= mean_iou <= upper and not interval_selected[idx]: + pairs.append((i, j, mean_iou)) + interval_selected[idx] = True # Mark this interval as selected + break # Move to the next pair after adding one in the current sub-interval + + return pairs + + +def load_image(data_root, scene_name, image_id): + # load depthmap + depth_path = f"{data_root}/{scene_name}/depths/{image_id}.png" + depth = cv2.imread(depth_path, cv2.IMREAD_UNCHANGED).astype(np.float32) / 1000.0 + # load camera parameters + meta_path = f"{data_root}/{scene_name}/images/{image_id}.npz" + meta = np.load(meta_path) + c2w = meta['camera_pose'] + K = meta['camera_intrinsics'] + return depth, c2w, K + +# Unproject depthmap to point cloud and project to another camera +def calculate_iou(depth1, c2w1, K1, depth2, c2w2, K2): + # Move data to GPU and ensure float32 dtype + depth1 = torch.from_numpy(depth1).cuda().float() + depth2 = torch.from_numpy(depth2).cuda().float() + c2w1 = torch.from_numpy(c2w1).cuda().float() + c2w2 = torch.from_numpy(c2w2).cuda().float() + K1 = torch.from_numpy(K1).cuda().float() + K2 = torch.from_numpy(K2).cuda().float() + + # Get image dimensions + h, w = depth1.shape + + # Create pixel coordinates + y, x = torch.meshgrid(torch.arange(h, device='cuda', dtype=torch.float32), + torch.arange(w, device='cuda', dtype=torch.float32)) + pixels = torch.stack((x.flatten(), y.flatten(), torch.ones_like(x.flatten())), dim=-1).T + + # Unproject pixels to 3D points + pixels_3d = torch.linalg.inv(K1) @ pixels + pixels_3d *= depth1.flatten().unsqueeze(0) + + # Transform 3D points to world coordinates + pixels_world = c2w1[:3, :3] @ pixels_3d + c2w1[:3, 3:4] + + # Check if c2w2[:3, :3] is invertible + if torch.det(c2w2[:3, :3]) == 0: + return 0, False # Calculation failed + + # Project world points to second camera + pixels_cam2 = torch.linalg.inv(c2w2[:3, :3]) @ (pixels_world - c2w2[:3, 3:4]) + pixels_img2 = K2 @ pixels_cam2 + + # Normalize homogeneous coordinates + pixels_img2 = pixels_img2[:2] / pixels_img2[2] + pixels_img2 = pixels_img2.T + + # Filter valid pixels + valid_mask = (pixels_img2[:, 0] >= 0) & (pixels_img2[:, 0] < w) & \ + (pixels_img2[:, 1] >= 0) & (pixels_img2[:, 1] < h) + + pixels_img2 = pixels_img2[valid_mask].long() + + # Compare depths + projected_depth = pixels_cam2[2, valid_mask] + actual_depth = depth2[pixels_img2[:, 1], pixels_img2[:, 0]] + + depth_diff = torch.abs(projected_depth - actual_depth) + depth_threshold = 0.1 # 10cm threshold + + overlap_mask = depth_diff < depth_threshold + + # Calculate IoU + intersection = torch.sum(overlap_mask) + union = torch.sum(valid_mask) + torch.sum(depth2 > 0) - intersection + + iou = intersection.float() / union.float() if union > 0 else torch.tensor(0.0, device='cuda') + + return iou.item() + +if __name__ == "__main__": + data_root = "data/scannetpp_processed" + target_root = "data/scannetpp_target" + preprocess_scannetpp(data_root, target_root) diff --git a/src/gaussian_head.py b/src/gaussian_head.py new file mode 100644 index 0000000000000000000000000000000000000000..bd5eed01452c024af295e68573b3c811d3c42618 --- /dev/null +++ b/src/gaussian_head.py @@ -0,0 +1,142 @@ +import torch +import torch.nn as nn +from einops import rearrange +from src.utils.gaussian_model import build_covariance +from simple_knn._C import distCUDA2 +from src.utils.sh_utils import RGB2SH + +class GaussianHead(nn.Module): + def __init__(self, d_pt_feat=64, **kwargs): + super().__init__() + # args + self.args = kwargs + self.d_means = 3 + self.d_scales = 3 + self.d_rotations = 4 + self.d_opacities = 1 + self.sh_degree = 3 + self.d_view_dep_features = 3 # RGB + self.d_sh = (self.sh_degree + 1) ** 2 + self.d_attr = (self.d_scales + self.d_rotations + self.d_opacities + self.d_view_dep_features * self.d_sh) + if self.args.get('d_gs_feats'): + self.d_attr += self.args['d_gs_feats'] + + # Create a mask for the spherical harmonics coefficients. + # This ensures that at initialization, the coefficients are biased + # towards having a large DC component and small view-dependent components. + self.register_buffer( + "sh_mask", + torch.ones((self.d_sh,), dtype=torch.float32), + persistent=False, + ) + for degree in range(1, self.sh_degree + 1): + self.sh_mask[degree**2 : (degree + 1) ** 2] = 0.5 * 0.25**degree + + self.gaussian_proj = nn.Linear(d_pt_feat, self.d_attr) + + # Activation functions + self.scale_activation = torch.exp + self.rotation_activation = torch.nn.functional.normalize + self.opacity_activation = torch.sigmoid + + def forward(self, point_transformer_output, lseg_features=None): + pred1 = {} + pred2 = {} + + scene_scale = point_transformer_output['scale'] # B, 1, 1 + scene_center = point_transformer_output['center'] # B, 1, 3 + B, H, W, _ = point_transformer_output['shape'] + normalized_means = point_transformer_output['coord'] # B * V * H * W, 3 + colors = point_transformer_output['color'] # B * V * H * W, 3 + + # split normalized_means to 2 views + normalized_means = rearrange(normalized_means, '(b v h w) c -> v b (h w) c', v=2, b=B, h=H, w=W) + means = normalized_means * scene_scale + scene_center # V, B, H * W, 3 + means = rearrange(means, 'v b (h w) c -> b (v h w) c', b=B, v=2, h=H, w=W) + + # get features + feat = point_transformer_output['feat'] + gaussian_attr = self.gaussian_proj(feat) + + # # split gaussian attributes + # scales, rotations, opacities, sh_coeffs = torch.split(gaussian_attr, + # [ + # self.d_scales, + # self.d_rotations, + # self.d_opacities, + # self.d_view_dep_features * self.d_sh + # ], + # dim=-1) + + scales, rotations, opacities, sh_coeffs, gs_feats = torch.split(gaussian_attr, + [ + self.d_scales, + self.d_rotations, + self.d_opacities, + self.d_view_dep_features * self.d_sh, + self.args['d_gs_feats'] + ], + dim=-1) + + # scales + # calculate the distance between each point and its nearest neighbor + all_dist = torch.stack([torch.sqrt(torch.clamp_min(distCUDA2(pts3d), 0.0000001)) for pts3d in means]) # B, V * H * W + median_dist = all_dist.median(dim=-1)[0][:, None, None] # B, 1, 1 + scales = self.scale_activation(scales) + scales = rearrange(scales, '(b v h w) c -> b (v h w) c', b=B, v=2, h=H, w=W) + scales = scales * all_dist[..., None] + # clip scales + scales = torch.clamp(scales, min=0.1 * median_dist, max=3.0 * median_dist) + scales = rearrange(scales, 'b (v h w) c -> (b v h w) c', b=B, v=2, h=H, w=W) + + # activation + rotations = self.rotation_activation(rotations) + opacities = self.opacity_activation(opacities) + + # build covariance matrix + covs = build_covariance(scales, rotations) + + # sh_mask + sh_coeffs = rearrange(sh_coeffs, '(b v h w) (c d) -> (b v h w) c d', b=B, v=2, h=H, w=W, c=self.d_sh, d=self.d_view_dep_features) + sh_dc = sh_coeffs[..., 0, :] + sh_rest = sh_coeffs[..., 1:, :] + if self.args.get('rgb_residual'): + # denormalize colors + colors = colors * 0.5 + 0.5 + sh_rgb = RGB2SH(colors) # (B * V * H * W, 3) + # add rgb residual to dc component + sh_dc = sh_dc + sh_rgb + # concatenate dc and rest + sh_coeffs = torch.cat([sh_dc[..., None, :], sh_rest], dim=-2) + sh_coeffs = sh_coeffs * self.sh_mask[None, :, None] + + # lseg_features(learning residual) + lseg_features = rearrange(lseg_features, '(v b) c h w -> (b v h w) c', b=B, v=2, h=H, w=W) + gs_feats = gs_feats + lseg_features + + # split to 2 views + scales = rearrange(scales, '(b v h w) ... -> v b h w ...', v=2, b=B, h=H, w=W) + rotations = rearrange(rotations, '(b v h w) ... -> v b h w ...', v=2, b=B, h=H, w=W) + opacities = rearrange(opacities, '(b v h w) ... -> v b h w ...', v=2, b=B, h=H, w=W) + sh_coeffs = rearrange(sh_coeffs, '(b v h w) ... -> v b h w ...', v=2, b=B, h=H, w=W) + covs = rearrange(covs, '(b v h w) ... -> v b h w ...', v=2, b=B, h=H, w=W) + means = rearrange(means, 'b (v h w) ... -> v b h w ...', v=2, b=B, h=H, w=W) + gs_feats = rearrange(gs_feats, '(b v h w) ... -> v b h w ...', v=2, b=B, h=H, w=W) + + pred1['scales'] = scales[0] + pred1['rotations'] = rotations[0] + pred1['covs'] = covs[0] + pred1['opacities'] = opacities[0] + pred1['sh_coeffs'] = sh_coeffs[0] + pred1['means'] = means[0] + pred1['gs_feats'] = gs_feats[0] + + pred2['scales'] = scales[1] + pred2['rotations'] = rotations[1] + pred2['covs'] = covs[1] + pred2['opacities'] = opacities[1] + pred2['sh_coeffs'] = sh_coeffs[1] + pred2['means'] = means[1] + pred2['gs_feats'] = gs_feats[1] + + return pred1, pred2 diff --git a/src/infer.py b/src/infer.py new file mode 100644 index 0000000000000000000000000000000000000000..b52d5e2832fff66560afe6afd4929b39b0037fbe --- /dev/null +++ b/src/infer.py @@ -0,0 +1,23 @@ +import argparse +import sys + +sys.path.append('.') +from src.model import LSM_MASt3R +from src.utils.visualization_utils import render_video_from_file + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--file_list', type=str, nargs='+', required=True, + help='List of input image files or directories') + parser.add_argument('--model_path', type=str, required=True) + parser.add_argument('--output_path', type=str, required=True) + parser.add_argument('--resolution', type=int, default=512) + parser.add_argument('--n_interp', type=int, default=90) + parser.add_argument('--fps', type=int, default=30) + + args = parser.parse_args() + + # 1. load model + model = LSM_MASt3R.from_pretrained(args.model_path) + # 2. render video + render_video_from_file(args.file_list, model, args.output_path, resolution=args.resolution, n_interp=args.n_interp, fps=args.fps) diff --git a/src/losses.py b/src/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..ce1d979548ca4fc25d7da960341330b8d27d010b --- /dev/null +++ b/src/losses.py @@ -0,0 +1,193 @@ +from submodules.mast3r.dust3r.dust3r.losses import * +from torchmetrics import PeakSignalNoiseRatio, StructuralSimilarityIndexMeasure, JaccardIndex, Accuracy +import lpips +from src.utils.gaussian_model import GaussianModel +from src.utils.cuda_splatting import render, DummyPipeline +from einops import rearrange +from src.utils.camera_utils import get_scaled_camera +from torchvision.utils import save_image +from dust3r.inference import make_batch_symmetric + +class L2Loss (LLoss): + """ Euclidean distance between 3d points """ + + def distance(self, a, b): + return torch.norm(a - b, dim=-1) # normalized L2 distance + +class L1Loss (LLoss): + """ Manhattan distance between 3d points """ + + def distance(self, a, b): + return torch.abs(a - b).mean() # L1 distance + +L2 = L2Loss() +L1 = L1Loss() + +def merge_and_split_predictions(pred1, pred2): + merged = {} + for key in pred1.keys(): + merged_pred = torch.stack([pred1[key], pred2[key]], dim=1) + merged_pred = rearrange(merged_pred, 'b v h w ... -> b (v h w) ...') + merged[key] = merged_pred + + # Split along the batch dimension + batch_size = next(iter(merged.values())).shape[0] + split = [{key: value[i] for key, value in merged.items()} for i in range(batch_size)] + + return split + +class GaussianLoss(MultiLoss): + def __init__(self, ssim_weight=0.2): + super().__init__() + self.ssim_weight = ssim_weight + self.ssim = StructuralSimilarityIndexMeasure(data_range=1.0).cuda() + self.psnr = PeakSignalNoiseRatio(data_range=1.0).cuda() + self.lpips_vgg = lpips.LPIPS(net='vgg').cuda() + self.pipeline = DummyPipeline() + # bg_color + self.register_buffer('bg_color', torch.tensor([0.0, 0.0, 0.0]).cuda()) + + def get_name(self): + return f'GaussianLoss(ssim_weight={self.ssim_weight})' + + # def compute_loss(self, gt1, gt2, target_view, pred1, pred2, model): + # # render images + # # 1. merge predictions + # pred = merge_and_split_predictions(pred1, pred2) + + # # 2. calculate optimal scaling + # pred_pts1 = pred1['means'] + # pred_pts2 = pred2['means'] + # # convert to camera1 coordinates + # # everything is normalized w.r.t. camera of view1 + # valid1 = gt1['valid_mask'].clone() + # valid2 = gt2['valid_mask'].clone() + # in_camera1 = inv(gt1['camera_pose']) + # gt_pts1 = geotrf(in_camera1, gt1['pts3d'].to(in_camera1.device)) # B,H,W,3 + # gt_pts2 = geotrf(in_camera1, gt2['pts3d'].to(in_camera1.device)) # B,H,W,3 + # scaling = find_opt_scaling(gt_pts1, gt_pts2, pred_pts1, pred_pts2, valid1=valid1, valid2=valid2) + + # # 3. render images(need gaussian model, camera, pipeline) + # rendered_images = [] + # rendered_feats = [] + # for i in range(len(pred)): + # # get gaussian model + # gaussians = GaussianModel.from_predictions(pred[i], sh_degree=3) + # # get camera + # ref_camera_extrinsics = gt1['camera_pose'][i] + # target_extrinsics = target_view['camera_pose'][i] + # target_intrinsics = target_view['camera_intrinsics'][i] + # image_shape = target_view['true_shape'][i] + # scale = scaling[i] + # camera = get_scaled_camera(ref_camera_extrinsics, target_extrinsics, target_intrinsics, scale, image_shape) + # # render(image and features) + # rendered_output = render(camera, gaussians, self.pipeline, self.bg_color) + # rendered_images.append(rendered_output['render']) + # rendered_feats.append(rendered_output['feature_map']) + + # rendered_images = torch.stack(rendered_images, dim=0) # B, 3, H, W + # rendered_feats = torch.stack(rendered_feats, dim=0) # B, d_feats, H, W + # rendered_feats = model.feature_expansion(rendered_feats) # B, 512, H//2, W//2 + + # gt_images = target_view['img'] * 0.5 + 0.5 + # gt_feats = model.lseg_feature_extractor.extract_features(target_view['img']) # B, 512, H//2, W//2 + # image_loss = torch.abs(rendered_images - gt_images).mean() + # feature_loss = torch.abs(rendered_feats - gt_feats).mean() + # loss = image_loss + 100 * feature_loss + + # # # temp + # # gt_logits = model.lseg_feature_extractor.decode_feature(gt_feats, ['wall', 'floor', 'others']) + # # gt_labels = torch.argmax(gt_logits, dim=1, keepdim=True) + # # rendered_logits = model.lseg_feature_extractor.decode_feature(rendered_feats, ['wall', 'floor', 'others']) + # # rendered_labels = torch.argmax(rendered_logits, dim=1, keepdim=True) + + # # calculate metric + # with torch.no_grad(): + # ssim = self.ssim(rendered_images, gt_images) + # psnr = self.psnr(rendered_images, gt_images) + # lpips = self.lpips_vgg(rendered_images, gt_images).mean() + + # return loss, {'ssim': ssim, 'psnr': psnr, 'lpips': lpips, 'image_loss': image_loss, 'feature_loss': feature_loss} + + def compute_loss(self, gt1, gt2, target_view, pred1, pred2, model): + # render images + # 1. merge predictions + pred = merge_and_split_predictions(pred1, pred2) + + # 2. calculate optimal scaling + pred_pts1 = pred1['means'] + pred_pts2 = pred2['means'] + # convert to camera1 coordinates + # everything is normalized w.r.t. camera of view1 + valid1 = gt1['valid_mask'].clone() + valid2 = gt2['valid_mask'].clone() + in_camera1 = inv(gt1['camera_pose']) + gt_pts1 = geotrf(in_camera1, gt1['pts3d'].to(in_camera1.device)) # B,H,W,3 + gt_pts2 = geotrf(in_camera1, gt2['pts3d'].to(in_camera1.device)) # B,H,W,3 + scaling = find_opt_scaling(gt_pts1, gt_pts2, pred_pts1, pred_pts2, valid1=valid1, valid2=valid2) + + # 3. render images(need gaussian model, camera, pipeline) + rendered_images = [] + rendered_feats = [] + gt_images = [] + + for i in range(len(pred)): + # get gaussian model + gaussians = GaussianModel.from_predictions(pred[i], sh_degree=3) + # get camera + ref_camera_extrinsics = gt1['camera_pose'][i] + target_view_list = [gt1, gt2, target_view] # use gt1, gt2, and target_view + for j in range(len(target_view_list)): + target_extrinsics = target_view_list[j]['camera_pose'][i] + target_intrinsics = target_view_list[j]['camera_intrinsics'][i] + image_shape = target_view_list[j]['true_shape'][i] + scale = scaling[i] + camera = get_scaled_camera(ref_camera_extrinsics, target_extrinsics, target_intrinsics, scale, image_shape) + # render(image and features) + rendered_output = render(camera, gaussians, self.pipeline, self.bg_color) + rendered_images.append(rendered_output['render']) + rendered_feats.append(rendered_output['feature_map']) + gt_images.append(target_view_list[j]['img'][i] * 0.5 + 0.5) + + rendered_images = torch.stack(rendered_images, dim=0) # B, 3, H, W + gt_images = torch.stack(gt_images, dim=0) + rendered_feats = torch.stack(rendered_feats, dim=0) # B, d_feats, H, W + rendered_feats = model.feature_expansion(rendered_feats) # B, 512, H//2, W//2 + gt_feats = model.lseg_feature_extractor.extract_features(gt_images) # B, 512, H//2, W//2 + image_loss = torch.abs(rendered_images - gt_images).mean() + feature_loss = torch.abs(rendered_feats - gt_feats).mean() + loss = image_loss + feature_loss + + # calculate metric + with torch.no_grad(): + ssim = self.ssim(rendered_images, gt_images) + psnr = self.psnr(rendered_images, gt_images) + lpips = self.lpips_vgg(rendered_images, gt_images).mean() + + return loss, {'ssim': ssim, 'psnr': psnr, 'lpips': lpips, 'image_loss': image_loss, 'feature_loss': feature_loss} + +# loss for one batch +def loss_of_one_batch(batch, model, criterion, device, symmetrize_batch=False, use_amp=False, ret=None): + view1, view2, target_view = batch + ignore_keys = set(['depthmap', 'dataset', 'label', 'instance', 'idx', 'true_shape', 'rng', 'pts3d']) + for view in batch: + for name in view.keys(): # pseudo_focal + if name in ignore_keys: + continue + view[name] = view[name].to(device, non_blocking=True) + + if symmetrize_batch: + view1, view2 = make_batch_symmetric(batch) + + # Get the actual model if it's distributed + actual_model = model.module if hasattr(model, 'module') else model + + with torch.cuda.amp.autocast(enabled=bool(use_amp)): + pred1, pred2 = actual_model(view1, view2) + + # loss is supposed to be symmetric + with torch.cuda.amp.autocast(enabled=False): + loss = criterion(view1, view2, target_view, pred1, pred2, actual_model) if criterion is not None else None + + result = dict(view1=view1, view2=view2, target_view=target_view, pred1=pred1, pred2=pred2, loss=loss) + return result[ret] if ret else result \ No newline at end of file diff --git a/src/lseg.py b/src/lseg.py new file mode 100644 index 0000000000000000000000000000000000000000..81adabefac4e46864682df6ec5ca83bb0e3c4775 --- /dev/null +++ b/src/lseg.py @@ -0,0 +1,171 @@ +import torch +import torch.nn as nn +from submodules.lang_seg.modules.models.lseg_net import LSegNet, clip + +class LSegFeatureExtractor(LSegNet): + def __init__(self, half_res=True): + super().__init__( + labels='', + backbone='clip_vitl16_384', + features=256, + crop_size=224, + arch_option=0, + block_depth=0, + activation='lrelu' + ) + + self.half_res = half_res + + @torch.no_grad() + def extract_features(self, x): + layer_1, layer_2, layer_3, layer_4 = forward_layers(self.pretrained, x) + # layer:(b, 1024, h//16, w//16) + # image_features = torch.cat([layer_1, layer_2, layer_3, layer_4], dim=1) + # # image_features:(b, 4096, h//16, w//16) + + # dense feature + # DPT head + pretrained = self.pretrained + layer_1 = pretrained.act_postprocess1[3 : len(pretrained.act_postprocess1)](layer_1) + layer_2 = pretrained.act_postprocess2[3 : len(pretrained.act_postprocess2)](layer_2) + layer_3 = pretrained.act_postprocess3[3 : len(pretrained.act_postprocess3)](layer_3) + layer_4 = pretrained.act_postprocess4[3 : len(pretrained.act_postprocess4)](layer_4) + + # refinenet + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + # (b, 512, h//2, w//2) + image_features = self.scratch.head1(path_1) + if self.half_res: + return image_features + + # (b, 512, h, w) + image_features = self.scratch.output_conv(image_features) + + return image_features + + @torch.no_grad() + def decode_feature(self, image_features, labelset=''): + # # image_features:(b, 4096, h//16, w//16) + # # split image_features into 4 parts + # layer_1, layer_2, layer_3, layer_4 = torch.split(image_features, 1024, dim=1) + + # # DPT head + # pretrained = self.pretrained + # layer_1 = pretrained.act_postprocess1[3 : len(pretrained.act_postprocess1)](layer_1) + # layer_2 = pretrained.act_postprocess2[3 : len(pretrained.act_postprocess2)](layer_2) + # layer_3 = pretrained.act_postprocess3[3 : len(pretrained.act_postprocess3)](layer_3) + # layer_4 = pretrained.act_postprocess4[3 : len(pretrained.act_postprocess4)](layer_4) + + # # refinenet + # layer_1_rn = self.scratch.layer1_rn(layer_1) + # layer_2_rn = self.scratch.layer2_rn(layer_2) + # layer_3_rn = self.scratch.layer3_rn(layer_3) + # layer_4_rn = self.scratch.layer4_rn(layer_4) + + # path_4 = self.scratch.refinenet4(layer_4_rn) + # path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + # path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + # path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + # image_features = self.scratch.head1(path_1) + imshape = image_features.shape + + # encode text + if labelset == '': + text = self.text + else: + text = clip.tokenize(labelset) + + self.logit_scale = self.logit_scale.to(image_features.device) + text = text.to(image_features.device) + text_features = self.clip_pretrained.encode_text(text) + image_features = image_features.permute(0,2,3,1).reshape(-1, self.out_c) + + # normalized features + image_features = image_features / image_features.norm(dim=-1, keepdim=True) + text_features = text_features / text_features.norm(dim=-1, keepdim=True) + + logits_per_image = self.logit_scale * image_features.half() @ text_features.t() + out = logits_per_image.float().view(imshape[0], imshape[2], imshape[3], -1).permute(0,3,1,2) + + if self.arch_option in [1, 2]: + for _ in range(self.block_depth - 1): + out = self.scratch.head_block(out) + out = self.scratch.head_block(out, False) + + if self.half_res: + out = self.scratch.output_conv(out) + + return out + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs): + print(f"Loading checkpoint from: {pretrained_model_name_or_path}") + ckpt = torch.load(pretrained_model_name_or_path, map_location='cpu') + print(f"Checkpoint loaded. Keys in checkpoint: {ckpt.keys()}") + + print("Processing state dict...") + new_state_dict = {k[len("net."):]: v for k, v in ckpt['state_dict'].items() if k.startswith("net.")} + print(f"Processed state dict. Number of keys: {len(new_state_dict)}") + + print("Initializing model...") + model = cls(*args, **kwargs) + + print("Loading state dict into model...") + model.load_state_dict(new_state_dict, strict=True) + print("State dict loaded successfully.") + + print("Cleaning up...") + del ckpt + del new_state_dict + + print("Model loading complete.") + return model + +def forward_layers(pretrained, x): + b, c, h, w = x.shape + + # encoder + glob = pretrained.model.forward_flex(x) + + layer_1 = pretrained.activations["1"] + layer_2 = pretrained.activations["2"] + layer_3 = pretrained.activations["3"] + layer_4 = pretrained.activations["4"] + + layer_1 = pretrained.act_postprocess1[0:2](layer_1) + layer_2 = pretrained.act_postprocess2[0:2](layer_2) + layer_3 = pretrained.act_postprocess3[0:2](layer_3) + layer_4 = pretrained.act_postprocess4[0:2](layer_4) + + unflatten = nn.Sequential( + nn.Unflatten( + 2, + torch.Size( + [ + h // pretrained.model.patch_size[1], + w // pretrained.model.patch_size[0], + ] + ), + ) + ) + + if layer_1.ndim == 3: + layer_1 = unflatten(layer_1) + if layer_2.ndim == 3: + layer_2 = unflatten(layer_2) + if layer_3.ndim == 3: + layer_3 = unflatten(layer_3) + if layer_4.ndim == 3: + layer_4 = unflatten(layer_4) + + return layer_1, layer_2, layer_3, layer_4 diff --git a/src/model.py b/src/model.py new file mode 100644 index 0000000000000000000000000000000000000000..021d01da35ab0b7153d1cd683609730e381d9563 --- /dev/null +++ b/src/model.py @@ -0,0 +1,176 @@ +import torch +import torch.nn as nn +import yaml +import sys +sys.path.append(".") +sys.path.append("submodules") +sys.path.append("submodules/mast3r") +from mast3r.model import AsymmetricMASt3R +from src.ptv3 import PTV3 +from src.gaussian_head import GaussianHead +from src.utils.points_process import merge_points +from src.losses import GaussianLoss +from src.lseg import LSegFeatureExtractor +import argparse + +class LSM_MASt3R(nn.Module): + def __init__(self, + mast3r_config, + point_transformer_config, + gaussian_head_config, + lseg_config, + ): + + super().__init__() + # self.config + self.config = { + 'mast3r_config': mast3r_config, + 'point_transformer_config': point_transformer_config, + 'gaussian_head_config': gaussian_head_config, + 'lseg_config': lseg_config + } + + # Initialize AsymmetricMASt3R + self.mast3r = AsymmetricMASt3R.from_pretrained(**mast3r_config) + + # Freeze MASt3R parameters + for param in self.mast3r.parameters(): + param.requires_grad = False + self.mast3r.eval() + + # Initialize PointTransformerV3 + self.point_transformer = PTV3(**point_transformer_config) + + # Initialize the gaussian head + self.gaussian_head = GaussianHead(**gaussian_head_config) + + # Initialize the lseg feature extractor + self.lseg_feature_extractor = LSegFeatureExtractor.from_pretrained(**lseg_config) + for param in self.lseg_feature_extractor.parameters(): + param.requires_grad = False + self.lseg_feature_extractor.eval() + + # Define two linear layers + d_gs_feats = gaussian_head_config.get('d_gs_feats', 32) + self.feature_reduction = nn.Sequential( + nn.Conv2d(512, d_gs_feats, kernel_size=1), + nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True) + ) # (b, 512, h//2, w//2) -> (b, d_features, h, w) + + self.feature_expansion = nn.Sequential( + nn.Conv2d(d_gs_feats, 512, kernel_size=1), + nn.Upsample(scale_factor=0.5, mode='bilinear', align_corners=True) + ) # (b, d_features, h, w) -> (b, 512, h//2, w//2) + + def forward(self, view1, view2): + # AsymmetricMASt3R forward pass + mast3r_output = self.mast3r(view1, view2) + + # merge points from two views + data_dict = merge_points(mast3r_output, view1, view2) + + # PointTransformerV3 forward pass + point_transformer_output = self.point_transformer(data_dict) + + # extract lseg features + lseg_features = self.extract_lseg_features(view1, view2) + + # Gaussian head forward pass + final_output = self.gaussian_head(point_transformer_output, lseg_features) + + return final_output + + def extract_lseg_features(self, view1, view2): + # concat view1 and view2 + img = torch.cat([view1['img'], view2['img']], dim=0) # (v*b, 3, h, w) + # extract features + lseg_features = self.lseg_feature_extractor.extract_features(img) # (v*b, 512, h//2, w//2) + # reduce dimensions + lseg_features = self.feature_reduction(lseg_features) # (v*b, d_features, h, w) + + return lseg_features + + @staticmethod + def from_pretrained(checkpoint_path, device='cuda'): + # Load the checkpoint + ckpt = torch.load(checkpoint_path, map_location='cpu') + + # Extract the configuration from the checkpoint + config = ckpt['args'] + + # Create a new instance of LSM_MASt3R + model = eval(config.model) + + # Load the state dict + model.load_state_dict(ckpt['model']) + + # Move the model to the specified device + model = model.to(device) + + return model + + def state_dict(self, destination=None, prefix='', keep_vars=False): + # 获取所有参数的state_dict + full_state_dict = super().state_dict(destination=destination, prefix=prefix, keep_vars=keep_vars) + + # 只保留需要训练的参数 + trainable_state_dict = { + k: v for k, v in full_state_dict.items() + if not (k.startswith('mast3r.') or k.startswith('lseg_feature_extractor.')) + } + + return trainable_state_dict + + def load_state_dict(self, state_dict, strict=True): + # 获取当前模型的完整state_dict + model_state = super().state_dict() + + # 只更新需要训练的参数 + for k in list(state_dict.keys()): + if k in model_state and not (k.startswith('mast3r.') or k.startswith('lseg_feature_extractor.')): + model_state[k] = state_dict[k] + + # 使用更新后的state_dict加载模型 + super().load_state_dict(model_state, strict=False) + +if __name__ == "__main__": + from torch.utils.data import DataLoader + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--checkpoint', type=str) + args = parser.parse_args() + + # Load config + with open("configs/model_config.yaml", "r") as f: + config = yaml.safe_load(f) + # Initialize model + if args.checkpoint is not None: + model = LSM_MASt3R.from_pretrained(args.checkpoint, device='cuda') + else: + model = LSM_MASt3R(**config).to('cuda') + + model.eval() + + # Print model + print(model) + # Load dataset + from src.datasets.scannet import Scannet + dataset = Scannet(split='train', ROOT="data/scannet_processed", resolution=[(512, 384)]) + # Print dataset + print(dataset) + # Test model + data_loader = DataLoader(dataset, batch_size=3, shuffle=True) + data = next(iter(data_loader)) + # move data to cuda + for view in data: + view['img'] = view['img'].to('cuda') + view['depthmap'] = view['depthmap'].to('cuda') + view['camera_pose'] = view['camera_pose'].to('cuda') + view['camera_intrinsics'] = view['camera_intrinsics'].to('cuda') + # Forward pass + output = model(*data[:2]) + + # Loss + loss = GaussianLoss() + loss_value = loss(*data, *output, model) + print(loss_value) diff --git a/src/ptv3.py b/src/ptv3.py new file mode 100644 index 0000000000000000000000000000000000000000..fb645bebf266bfffb49d574d81a02e026fde5823 --- /dev/null +++ b/src/ptv3.py @@ -0,0 +1,13 @@ +from PointTransformerV3.model import * + +class PTV3(PointTransformerV3): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def encode(self, data_dict): + point = Point(data_dict) + point.serialization(order=self.order, shuffle_orders=self.shuffle_orders) + point.sparsify() + point = self.embedding(point) + point = self.enc(point) + return point.feats \ No newline at end of file diff --git a/src/train.py b/src/train.py new file mode 100644 index 0000000000000000000000000000000000000000..cc016ba48508bf16ddf5bee7fa9b86f90496d158 --- /dev/null +++ b/src/train.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# training executable for MASt3R +# -------------------------------------------------------- +import sys +sys.path.append('.') +sys.path.append('submodules/mast3r') +from mast3r.model import AsymmetricMASt3R +from mast3r.losses import ConfMatchingLoss, MatchingLoss, APLoss, Regr3D, InfoNCE, Regr3D_ScaleShiftInv +from mast3r.datasets import ARKitScenes, BlendedMVS, Co3d, MegaDepth, ScanNetpp, StaticThings3D, Waymo, WildRGBD + +import mast3r.utils.path_to_dust3r # noqa +# add mast3r classes to dust3r imports +import dust3r.training +dust3r.training.AsymmetricMASt3R = AsymmetricMASt3R +dust3r.training.Regr3D = Regr3D +dust3r.training.Regr3D_ScaleShiftInv = Regr3D_ScaleShiftInv +dust3r.training.MatchingLoss = MatchingLoss +dust3r.training.ConfMatchingLoss = ConfMatchingLoss +dust3r.training.InfoNCE = InfoNCE +dust3r.training.APLoss = APLoss + +import dust3r.datasets +dust3r.datasets.ARKitScenes = ARKitScenes +dust3r.datasets.BlendedMVS = BlendedMVS +dust3r.datasets.Co3d = Co3d +dust3r.datasets.MegaDepth = MegaDepth +dust3r.datasets.ScanNetpp = ScanNetpp +dust3r.datasets.StaticThings3D = StaticThings3D +dust3r.datasets.Waymo = Waymo +dust3r.datasets.WildRGBD = WildRGBD +from src.datasets.scannet import Scannet +from src.datasets.scannetpp import Scannetpp +from src.datasets.megadepth import MegaDepth +dust3r.datasets.Scannet = Scannet +dust3r.datasets.Scannetpp = Scannetpp +dust3r.datasets.MegaDepth = MegaDepth + +from src.model import LSM_MASt3R +dust3r.training.LSM_MASt3R = LSM_MASt3R +from src.losses import GaussianLoss +dust3r.training.GaussianLoss = GaussianLoss + +from dust3r.training import get_args_parser as dust3r_get_args_parser # noqa +from dust3r.training import train # noqa + +import yaml + + +def get_args_parser(): + parser = dust3r_get_args_parser() + parser.prog = 'LSM_MASt3R training' + + # Load the configuration + with open("configs/model_config.yaml", "r") as f: + config = yaml.safe_load(f) + + # Convert the config dict to a string of keyword arguments + config_str = ", ".join(f"{k}={v}" for k, v in config.items()) + + # Set the default model string with parameters + parser.set_defaults(model=f"LSM_MASt3R({config_str})") + + return parser + + +if __name__ == '__main__': + args = get_args_parser() + args = args.parse_args() + train(args) diff --git a/src/utils/camera_utils.py b/src/utils/camera_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..783a96d4fd385ce1218631c809350e8fcad12951 --- /dev/null +++ b/src/utils/camera_utils.py @@ -0,0 +1,60 @@ +import math +import torch +from dust3r.utils.geometry import inv +from src.utils.cuda_splatting import DummyCamera + +def get_scaled_camera(ref_camera_extrinsics, target_camera_extrinsics, target_camera_intrinsics, scale, image_shape): + """ + get a scaled camera from a reference camera to a target camera + + """ + + # get extrinsics(target_camera to ref_camera) + target_camera_extrinsics = inv(ref_camera_extrinsics) @ target_camera_extrinsics + # scale translation + target_camera_extrinsics[:3, 3] = target_camera_extrinsics[:3, 3] * scale + # invert extrinsics(ref_camera to target_camera) + target_camera_extrinsics_inv = inv(target_camera_extrinsics) + # calculate fov + fovx = 2 * math.atan(image_shape[1] / (2 * target_camera_intrinsics[0, 0])) + fovy = 2 * math.atan(image_shape[0] / (2 * target_camera_intrinsics[1, 1])) + # return camera(numpy) + R = target_camera_extrinsics_inv[:3, :3].cpu().numpy().transpose() # R.transpose() : ref_camera_2_target_camera + T = target_camera_extrinsics_inv[:3, 3].cpu().numpy() # T : ref_camera_2_target_camera + image_shape = image_shape.cpu().numpy() + return DummyCamera(R, T, fovx, fovy, image_shape[1], image_shape[0]) + +def move_c2w_along_z(extrinsics: torch.Tensor, distance: float) -> torch.Tensor: + """ + 向后移动多个 Camera-to-World (C2W) 矩阵，使相机沿各自 Z 轴方向远离原点。 + + 参数: + extrinsics (torch.Tensor): 形状为 [N, 4, 4] 的张量，包含 N 个 C2W 矩阵。 + distance (float): 向后移动的距离。 + + 返回: + torch.Tensor: 更新后的 C2W 矩阵，形状与输入相同。 + """ + # 确保输入是一个四维矩阵，且最后一维是 4x4 + assert extrinsics.dim() == 3 and extrinsics.shape[1:] == (4, 4), \ + "输入的 extrinsics 必须是形状为 [N, 4, 4] 的张量" + + # 创建一个拷贝以免修改原矩阵 + updated_extrinsics = extrinsics.clone() + + # 遍历每个 C2W 矩阵 + for i in range(updated_extrinsics.shape[0]): + # 提取旋转矩阵 R 和平移向量 t + R = updated_extrinsics[i, :3, :3] # 形状为 [3, 3] + t = updated_extrinsics[i, :3, 3] # 形状为 [3] + + # 获取相机的 Z 轴方向（第三列） + z_axis = R[:, 2] # 形状为 [3] + + # 计算新的平移向量，沿 Z 轴方向向后移动 + t_new = t - distance * z_axis + + # 更新 C2W 矩阵的平移部分 + updated_extrinsics[i, :3, 3] = t_new + + return updated_extrinsics diff --git a/src/utils/cuda_splatting.py b/src/utils/cuda_splatting.py new file mode 100644 index 0000000000000000000000000000000000000000..540d3d2cb066027eb8ff5a206a07dfc27ed45116 --- /dev/null +++ b/src/utils/cuda_splatting.py @@ -0,0 +1,216 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# +import numpy as np +import torch +import math +from diff_gaussian_rasterization import GaussianRasterizationSettings, GaussianRasterizer +from .gaussian_model import GaussianModel +from .sh_utils import eval_sh +from .graphics_utils import getWorld2View2, getProjectionMatrix + +class DummyCamera: + def __init__(self, R, T, FoVx, FoVy, W, H): + self.projection_matrix = getProjectionMatrix(znear=0.01, zfar=100.0, fovX=FoVx, fovY=FoVy).transpose(0,1).cuda() + self.R = R + self.T = T + self.world_view_transform = torch.tensor(getWorld2View2(R, T, np.array([0,0,0]), 1.0)).transpose(0, 1).cuda() + self.full_proj_transform = (self.world_view_transform.unsqueeze(0).bmm(self.projection_matrix.unsqueeze(0))).squeeze(0) + self.camera_center = self.world_view_transform.inverse()[3, :3] + self.image_width = W + self.image_height = H + self.FoVx = FoVx + self.FoVy = FoVy + +class DummyPipeline: + convert_SHs_python = False + compute_cov3D_python = False + debug = False + +def calculate_fov(output_width, output_height, focal_length, aspect_ratio=1.0, invert_y=False): + fovx = 2 * math.atan((output_width / (2 * focal_length))) + fovy = 2 * math.atan((output_height / aspect_ratio) / (2 * focal_length)) + + if invert_y: + fovy = -fovy + + return fovx, fovy + +# def render(viewpoint_camera, pc : GaussianModel, pipe, bg_color : torch.Tensor, scaling_modifier = 1.0, override_color = None): +# """ +# Render the scene. + +# Background tensor (bg_color) must be on GPU! +# """ + +# # Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means +# screenspace_points = torch.zeros_like(pc.get_xyz, dtype=pc.get_xyz.dtype, requires_grad=True, device="cuda") + 0 +# try: +# screenspace_points.retain_grad() +# except: +# pass + +# # Set up rasterization configuration +# tanfovx = math.tan(viewpoint_camera.FoVx * 0.5) +# tanfovy = math.tan(viewpoint_camera.FoVy * 0.5) + +# raster_settings = GaussianRasterizationSettings( +# image_height=int(viewpoint_camera.image_height), +# image_width=int(viewpoint_camera.image_width), +# tanfovx=tanfovx, +# tanfovy=tanfovy, +# bg=bg_color, +# scale_modifier=scaling_modifier, +# viewmatrix=viewpoint_camera.world_view_transform, +# projmatrix=viewpoint_camera.full_proj_transform, +# sh_degree=pc.active_sh_degree, +# campos=viewpoint_camera.camera_center, +# prefiltered=False, +# debug=pipe.debug +# ) + +# rasterizer = GaussianRasterizer(raster_settings=raster_settings) + +# means3D = pc.get_xyz +# means2D = screenspace_points +# opacity = pc.get_opacity + +# # If precomputed 3d covariance is provided, use it. If not, then it will be computed from +# # scaling / rotation by the rasterizer. +# scales = None +# rotations = None +# cov3D_precomp = None +# if pipe.compute_cov3D_python: +# cov3D_precomp = pc.get_covariance(scaling_modifier) +# else: +# scales = pc.get_scaling +# rotations = pc.get_rotation + +# # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors +# # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer. +# shs = None +# colors_precomp = None +# if override_color is None: +# if pipe.convert_SHs_python: +# shs_view = pc.get_features.transpose(1, 2).view(-1, 3, (pc.max_sh_degree+1)**2) +# dir_pp = (pc.get_xyz - viewpoint_camera.camera_center.repeat(pc.get_features.shape[0], 1)) +# dir_pp_normalized = dir_pp/dir_pp.norm(dim=1, keepdim=True) +# sh2rgb = eval_sh(pc.active_sh_degree, shs_view, dir_pp_normalized) +# colors_precomp = torch.clamp_min(sh2rgb + 0.5, 0.0) +# else: +# shs = pc.get_features +# else: +# colors_precomp = override_color + +# # Rasterize visible Gaussians to image, obtain their radii (on screen). +# rendered_image, radii = rasterizer( +# means3D = means3D, +# means2D = means2D, +# shs = shs, +# colors_precomp = colors_precomp, +# opacities = opacity, +# scales = scales, +# rotations = rotations, +# cov3D_precomp = cov3D_precomp) + +# # Those Gaussians that were frustum culled or had a radius of 0 were not visible. +# # They will be excluded from value updates used in the splitting criteria. +# return {"render": rendered_image, +# "viewspace_points": screenspace_points, +# "visibility_filter" : radii > 0, +# "radii": radii} + +def render(viewpoint_camera, pc : GaussianModel, pipe, bg_color : torch.Tensor, scaling_modifier = 1.0, override_color = None): + """ + Render the scene. + + Background tensor (bg_color) must be on GPU! + """ + + # Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means + screenspace_points = torch.zeros_like(pc.get_xyz, dtype=pc.get_xyz.dtype, requires_grad=True, device="cuda") + 0 + try: + screenspace_points.retain_grad() + except: + pass + + # Set up rasterization configuration + tanfovx = math.tan(viewpoint_camera.FoVx * 0.5) + tanfovy = math.tan(viewpoint_camera.FoVy * 0.5) + + raster_settings = GaussianRasterizationSettings( + image_height=int(viewpoint_camera.image_height), + image_width=int(viewpoint_camera.image_width), + tanfovx=tanfovx, + tanfovy=tanfovy, + bg=bg_color, + scale_modifier=scaling_modifier, + viewmatrix=viewpoint_camera.world_view_transform, + projmatrix=viewpoint_camera.full_proj_transform, + sh_degree=pc.active_sh_degree, + campos=viewpoint_camera.camera_center, + prefiltered=False, + debug=pipe.debug + ) + + rasterizer = GaussianRasterizer(raster_settings=raster_settings) + + means3D = pc.get_xyz + means2D = screenspace_points + opacity = pc.get_opacity + + # If precomputed 3d covariance is provided, use it. If not, then it will be computed from + # scaling / rotation by the rasterizer. + scales = None + rotations = None + cov3D_precomp = None + if pipe.compute_cov3D_python: + cov3D_precomp = pc.get_covariance(scaling_modifier) + else: + scales = pc.get_scaling + rotations = pc.get_rotation + + # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors + # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer. + shs = None + colors_precomp = None + if override_color is None: + if pipe.convert_SHs_python: + shs_view = pc.get_features.transpose(1, 2).view(-1, 3, (pc.max_sh_degree+1)**2) + dir_pp = (pc.get_xyz - viewpoint_camera.camera_center.repeat(pc.get_features.shape[0], 1)) + dir_pp_normalized = dir_pp/dir_pp.norm(dim=1, keepdim=True) + sh2rgb = eval_sh(pc.active_sh_degree, shs_view, dir_pp_normalized) + colors_precomp = torch.clamp_min(sh2rgb + 0.5, 0.0) + else: + shs = pc.get_features + else: + colors_precomp = override_color + semantic_feature = pc.get_semantic_feature + + # Rasterize visible Gaussians to image, obtain their radii (on screen). + rendered_image, feature_map, radii, depth = rasterizer( + means3D = means3D, + means2D = means2D, + shs = shs, + colors_precomp = colors_precomp, + semantic_feature = semantic_feature, + opacities = opacity, + scales = scales, + rotations = rotations, + cov3D_precomp = cov3D_precomp) + + # Those Gaussians that were frustum culled or had a radius of 0 were not visible. + # They will be excluded from value updates used in the splitting criteria. + return {"render": rendered_image, + "viewspace_points": screenspace_points, + "visibility_filter" : radii > 0, + "radii": radii, + 'feature_map': feature_map, + "depth": depth} ###d \ No newline at end of file diff --git a/src/utils/gaussian_model.py b/src/utils/gaussian_model.py new file mode 100644 index 0000000000000000000000000000000000000000..c868a0ce0628b4abbff58f1bab036d746e23fa73 --- /dev/null +++ b/src/utils/gaussian_model.py @@ -0,0 +1,160 @@ +import os +import torch +from einops import rearrange +import numpy as np +from plyfile import PlyData, PlyElement +from os import makedirs, path +from errno import EEXIST + +def mkdir_p(folder_path): + # Creates a directory. equivalent to using mkdir -p on the command line + try: + makedirs(folder_path) + except OSError as exc: # Python >2.5 + if exc.errno == EEXIST and path.isdir(folder_path): + pass + else: + raise + +def RGB2SH(rgb): + return (rgb - 0.5) / C0 + +C0 = 0.28209479177387814 + +# https://github.com/facebookresearch/pytorch3d/blob/main/pytorch3d/transforms/rotation_conversions.py +def quaternion_to_matrix( + quaternions, + eps=1e-8, +) : + # Order changed to match scipy format! + i, j, k, r = torch.unbind(quaternions, dim=-1) + two_s = 2 / ((quaternions * quaternions).sum(dim=-1) + eps) + + o = torch.stack( + ( + 1 - two_s * (j * j + k * k), + two_s * (i * j - k * r), + two_s * (i * k + j * r), + two_s * (i * j + k * r), + 1 - two_s * (i * i + k * k), + two_s * (j * k - i * r), + two_s * (i * k - j * r), + two_s * (j * k + i * r), + 1 - two_s * (i * i + j * j), + ), + -1, + ) + return rearrange(o, "... (i j) -> ... i j", i=3, j=3) + + +def build_covariance( + scale, + rotation_xyzw, +): + scale = scale.diag_embed() + rotation = quaternion_to_matrix(rotation_xyzw) + return ( + rotation + @ scale + @ rearrange(scale, "... i j -> ... j i") + @ rearrange(rotation, "... i j -> ... j i") + ) + +def inverse_sigmoid(x): + return torch.log(x/(1-x)) + +class GaussianModel: + + def __init__(self, sh_degree : int): + self.active_sh_degree = 0 + self.max_sh_degree = sh_degree + self._xyz = torch.empty(0) + self._features_dc = torch.empty(0) + self._features_rest = torch.empty(0) + self._scaling = torch.empty(0) + self._rotation = torch.empty(0) + self._opacity = torch.empty(0) + self.max_radii2D = torch.empty(0) + self.xyz_gradient_accum = torch.empty(0) + self.denom = torch.empty(0) + self.optimizer = None + self.percent_dense = 0 + self.spatial_lr_scale = 0 + self._semantic_feature = torch.empty(0) + + @property + def get_scaling(self): + return self._scaling + + @property + def get_rotation(self): + return self._rotation + + @property + def get_xyz(self): + return self._xyz + + @property + def get_features(self): + features_dc = self._features_dc + features_rest = self._features_rest + return torch.cat((features_dc, features_rest), dim=1) + + @property + def get_opacity(self): + return self._opacity + + @property + def get_semantic_feature(self): + return self._semantic_feature + + def construct_list_of_attributes(self): + l = ['x', 'y', 'z', 'nx', 'ny', 'nz'] + # All channels except the 3 DC + for i in range(self._features_dc.shape[1]*self._features_dc.shape[2]): + l.append('f_dc_{}'.format(i)) + for i in range(self._features_rest.shape[1]*self._features_rest.shape[2]): + l.append('f_rest_{}'.format(i)) + + l.append('opacity') + for i in range(self._scaling.shape[1]): + l.append('scale_{}'.format(i)) + for i in range(self._rotation.shape[1]): + l.append('rot_{}'.format(i)) + # Add semantic features + for i in range(self._semantic_feature.shape[1]*self._semantic_feature.shape[2]): + l.append('semantic_{}'.format(i)) + return l + + @staticmethod + def from_predictions(pred, sh_degree): + gaussians = GaussianModel(sh_degree=sh_degree) + gaussians._xyz = pred['means'] + gaussians._features_dc = pred['sh_coeffs'][:, :1] # N, 1, d_sh + gaussians._features_rest = pred['sh_coeffs'][:, 1:] # N, d_sh-1, d_sh + gaussians._opacity = pred['opacities'] # N, 1 + gaussians._scaling = pred['scales'] # N, 3, 3 + gaussians._rotation = pred['rotations'] # N, 4 + gaussians._semantic_feature = pred['gs_feats'][:, None, :] # N, 1, d_feats + return gaussians + + def save_ply(self, path): + mkdir_p(os.path.dirname(path)) + + xyz = self._xyz.detach().cpu().numpy() + normals = np.zeros_like(xyz) + f_dc = self._features_dc.detach().transpose(1, 2).flatten(start_dim=1).contiguous().cpu().numpy() + f_rest = self._features_rest.detach().transpose(1, 2).flatten(start_dim=1).contiguous().cpu().numpy() + opacities = inverse_sigmoid(self._opacity).detach().cpu().numpy() + scale = torch.log(self._scaling).detach().cpu().numpy() + rotation = self._rotation.detach().cpu().numpy() + semantic_feature = self._semantic_feature.detach().transpose(1, 2).flatten(start_dim=1).contiguous().cpu().numpy() + + dtype_full = [(attribute, 'f4') for attribute in self.construct_list_of_attributes()] + + elements = np.empty(xyz.shape[0], dtype=dtype_full) + attributes = np.concatenate((xyz, normals, f_dc, f_rest, opacities, scale, rotation, semantic_feature), axis=1) + # attributes = np.concatenate((xyz, normals, f_dc, f_rest, opacities, scale, rotation), axis=1) + elements[:] = list(map(tuple, attributes)) + el = PlyElement.describe(elements, 'vertex') + PlyData([el]).write(path) \ No newline at end of file diff --git a/src/utils/graphics_utils.py b/src/utils/graphics_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b4627d837c74fcdffc898fa0c3071cb7b316802b --- /dev/null +++ b/src/utils/graphics_utils.py @@ -0,0 +1,77 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + +import torch +import math +import numpy as np +from typing import NamedTuple + +class BasicPointCloud(NamedTuple): + points : np.array + colors : np.array + normals : np.array + +def geom_transform_points(points, transf_matrix): + P, _ = points.shape + ones = torch.ones(P, 1, dtype=points.dtype, device=points.device) + points_hom = torch.cat([points, ones], dim=1) + points_out = torch.matmul(points_hom, transf_matrix.unsqueeze(0)) + + denom = points_out[..., 3:] + 0.0000001 + return (points_out[..., :3] / denom).squeeze(dim=0) + +def getWorld2View(R, t): + Rt = np.zeros((4, 4)) + Rt[:3, :3] = R.transpose() + Rt[:3, 3] = t + Rt[3, 3] = 1.0 + return np.float32(Rt) + +def getWorld2View2(R, t, translate=np.array([.0, .0, .0]), scale=1.0): + Rt = np.zeros((4, 4)) + Rt[:3, :3] = R.transpose() + Rt[:3, 3] = t + Rt[3, 3] = 1.0 + + C2W = np.linalg.inv(Rt) + cam_center = C2W[:3, 3] + cam_center = (cam_center + translate) * scale + C2W[:3, 3] = cam_center + Rt = np.linalg.inv(C2W) + return np.float32(Rt) + +def getProjectionMatrix(znear, zfar, fovX, fovY): + tanHalfFovY = math.tan((fovY / 2)) + tanHalfFovX = math.tan((fovX / 2)) + + top = tanHalfFovY * znear + bottom = -top + right = tanHalfFovX * znear + left = -right + + P = torch.zeros(4, 4) + + z_sign = 1.0 + + P[0, 0] = 2.0 * znear / (right - left) + P[1, 1] = 2.0 * znear / (top - bottom) + P[0, 2] = (right + left) / (right - left) + P[1, 2] = (top + bottom) / (top - bottom) + P[3, 2] = z_sign + P[2, 2] = z_sign * zfar / (zfar - znear) + P[2, 3] = -(zfar * znear) / (zfar - znear) + return P + +def fov2focal(fov, pixels): + return pixels / (2 * math.tan(fov / 2)) + +def focal2fov(focal, pixels): + return 2*math.atan(pixels/(2*focal)) \ No newline at end of file diff --git a/src/utils/points_process.py b/src/utils/points_process.py new file mode 100644 index 0000000000000000000000000000000000000000..2ecba5b501c12f244c3d8cbb2e97cb1a39fb5a25 --- /dev/null +++ b/src/utils/points_process.py @@ -0,0 +1,37 @@ +import torch +from einops import rearrange + +# merge points from two views and add color information +def merge_points(mast3r_output, view1, view2, grid_size=0.01): + # get points from mast3r_output + points1 = mast3r_output[0]['pts3d'].detach() # B, H, W, 3 + points2 = mast3r_output[1]['pts3d_in_other_view'].detach() # B, H, W, 3 + shape = points1.shape + # add color information + colors = torch.stack([view1['img'], view2['img']], dim=1) # B, V, 3, H, W + colors = rearrange(colors, 'b v c h w -> b (v h w) c') # B, V * H * W, 3 + # merge points + points = torch.stack([points1, points2], dim=1) # B, V, H, W, 3 + points = rearrange(points, 'b v h w c -> b (v h w) c') # B, V * H * W, 3 + B, N, _ = points.shape + offset = torch.arange(1, B + 1, device=points.device) * N + # Center and normalize points + center = torch.mean(points, dim=1, keepdim=True) + points = points - center + scale = torch.max(torch.norm(points, dim=2, keepdim=True), dim=1, keepdim=True)[0] + points = points / scale + # concat points and colors + feat = torch.cat([points, colors], dim=-1) # B, V * H * W, 6 + + data_dict = { + 'coord': rearrange(points, 'b n c -> (b n) c'), + 'color': rearrange(colors, 'b n c -> (b n) c'), + 'feat': rearrange(feat, 'b n c -> (b n) c'), + 'offset': offset, + 'grid_size': grid_size, + 'center': center, + 'scale': scale, + 'shape': shape, + } + + return data_dict diff --git a/src/utils/sh_utils.py b/src/utils/sh_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b3c773f13b884af4aee7b4608fa205f8d62d38d1 --- /dev/null +++ b/src/utils/sh_utils.py @@ -0,0 +1,117 @@ +# Copyright 2021 The PlenOctree Authors. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + + +C0 = 0.28209479177387814 +C1 = 0.4886025119029199 +C2 = [ + 1.0925484305920792, + -1.0925484305920792, + 0.31539156525252005, + -1.0925484305920792, + 0.5462742152960396 +] +C3 = [ + -0.5900435899266435, + 2.890611442640554, + -0.4570457994644658, + 0.3731763325901154, + -0.4570457994644658, + 1.445305721320277, + -0.5900435899266435 +] +C4 = [ + 2.5033429417967046, + -1.7701307697799304, + 0.9461746957575601, + -0.6690465435572892, + 0.10578554691520431, + -0.6690465435572892, + 0.47308734787878004, + -1.7701307697799304, + 0.6258357354491761, +] + + +def eval_sh(deg, sh, dirs): + """ + Evaluate spherical harmonics at unit directions + using hardcoded SH polynomials. + Works with torch/np/jnp. + ... Can be 0 or more batch dimensions. + Args: + deg: int SH deg. Currently, 0-3 supported + sh: jnp.ndarray SH coeffs [..., C, (deg + 1) ** 2] + dirs: jnp.ndarray unit directions [..., 3] + Returns: + [..., C] + """ + assert deg <= 4 and deg >= 0 + coeff = (deg + 1) ** 2 + assert sh.shape[-1] >= coeff + + result = C0 * sh[..., 0] + if deg > 0: + x, y, z = dirs[..., 0:1], dirs[..., 1:2], dirs[..., 2:3] + result = (result - + C1 * y * sh[..., 1] + + C1 * z * sh[..., 2] - + C1 * x * sh[..., 3]) + + if deg > 1: + xx, yy, zz = x * x, y * y, z * z + xy, yz, xz = x * y, y * z, x * z + result = (result + + C2[0] * xy * sh[..., 4] + + C2[1] * yz * sh[..., 5] + + C2[2] * (2.0 * zz - xx - yy) * sh[..., 6] + + C2[3] * xz * sh[..., 7] + + C2[4] * (xx - yy) * sh[..., 8]) + + if deg > 2: + result = (result + + C3[0] * y * (3 * xx - yy) * sh[..., 9] + + C3[1] * xy * z * sh[..., 10] + + C3[2] * y * (4 * zz - xx - yy)* sh[..., 11] + + C3[3] * z * (2 * zz - 3 * xx - 3 * yy) * sh[..., 12] + + C3[4] * x * (4 * zz - xx - yy) * sh[..., 13] + + C3[5] * z * (xx - yy) * sh[..., 14] + + C3[6] * x * (xx - 3 * yy) * sh[..., 15]) + + if deg > 3: + result = (result + C4[0] * xy * (xx - yy) * sh[..., 16] + + C4[1] * yz * (3 * xx - yy) * sh[..., 17] + + C4[2] * xy * (7 * zz - 1) * sh[..., 18] + + C4[3] * yz * (7 * zz - 3) * sh[..., 19] + + C4[4] * (zz * (35 * zz - 30) + 3) * sh[..., 20] + + C4[5] * xz * (7 * zz - 3) * sh[..., 21] + + C4[6] * (xx - yy) * (7 * zz - 1) * sh[..., 22] + + C4[7] * xz * (xx - 3 * yy) * sh[..., 23] + + C4[8] * (xx * (xx - 3 * yy) - yy * (3 * xx - yy)) * sh[..., 24]) + return result + +def RGB2SH(rgb): + return (rgb - 0.5) / C0 + +def SH2RGB(sh): + return sh * C0 + 0.5 \ No newline at end of file diff --git a/src/utils/visualization_utils.py b/src/utils/visualization_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ce852aff1aad00f14ab9c464252b494dc3130cf7 --- /dev/null +++ b/src/utils/visualization_utils.py @@ -0,0 +1,355 @@ +import sys +import os +import numpy as np +import scipy.interpolate +import PIL +import torch +import matplotlib.pyplot as plt +from sklearn.preprocessing import StandardScaler +from sklearn.decomposition import PCA +import moviepy.editor as mpy + +sys.path.append('submodules/mast3r/dust3r') +from dust3r.utils.image import heif_support_enabled, exif_transpose, _resize_pil_image, ImgNorm +from dust3r.image_pairs import make_pairs +from dust3r.inference import inference +from dust3r.cloud_opt import global_aligner, GlobalAlignerMode + +sys.path.append('.') +from src.utils.cuda_splatting import render, DummyPipeline +from src.utils.gaussian_model import GaussianModel +from src.utils.camera_utils import get_scaled_camera +from src.losses import merge_and_split_predictions +from src.utils.camera_utils import move_c2w_along_z + +from einops import rearrange +LABELS = ['wall', 'floor', 'ceiling', 'chair', 'table', 'sofa', 'bed', 'other'] +NUM_LABELS = len(LABELS) + 1 +PALLETE = plt.cm.get_cmap('tab10', NUM_LABELS) +COLORS_LIST = [PALLETE(i)[:3] for i in range(NUM_LABELS)] +COLORS = torch.tensor(COLORS_LIST, dtype=torch.float32) + +def load_images(folder_or_list, size, square_ok=False, verbose=True, save_dir=None): + """ open and convert all images in a list or folder to proper input format for DUSt3R + """ + if isinstance(folder_or_list, str): + if verbose: + print(f'>> Loading images from {folder_or_list}') + root, folder_content = folder_or_list, sorted(os.listdir(folder_or_list)) + + elif isinstance(folder_or_list, list): + if verbose: + print(f'>> Loading a list of {len(folder_or_list)} images') + root, folder_content = '', folder_or_list + + else: + raise ValueError(f'bad {folder_or_list=} ({type(folder_or_list)})') + + supported_images_extensions = ['.jpg', '.jpeg', '.png'] + if heif_support_enabled: + supported_images_extensions += ['.heic', '.heif'] + supported_images_extensions = tuple(supported_images_extensions) + + imgs = [] + for path in folder_content: + if not path.lower().endswith(supported_images_extensions): + continue + img = exif_transpose(PIL.Image.open(os.path.join(root, path))).convert('RGB') + W1, H1 = img.size + if size == 224: + # resize short side to 224 (then crop) + img = _resize_pil_image(img, round(size * max(W1/H1, H1/W1))) + else: + # resize long side to 512 + img = _resize_pil_image(img, size) + W, H = img.size + cx, cy = W//2, H//2 + if size == 224: + half = min(cx, cy) + img = img.crop((cx-half, cy-half, cx+half, cy+half)) + else: + halfw, halfh = ((2*cx)//32)*16, ((2*cy)//32)*16 + if not (square_ok) and W == H: + halfh = 3*halfw/4 + img = img.crop((cx-halfw, cy-halfh, cx+halfw, cy+halfh)) + + W2, H2 = img.size + if verbose: + print(f' - adding {path} with resolution {W1}x{H1} --> {W2}x{H2}') + + # Save the processed image if save_dir is provided + if save_dir: + os.makedirs(save_dir, exist_ok=True) + save_path = os.path.join(save_dir, f"processed_{len(imgs):03d}.png") + img.save(save_path) + if verbose: + print(f' - saved processed image to {save_path}') + + imgs.append(dict(img=ImgNorm(img)[None], true_shape=np.int32( + [img.size[::-1]]), idx=len(imgs), instance=str(len(imgs)))) + + assert imgs, 'no images foud at '+root + if verbose: + print(f' (Found {len(imgs)} images)') + return imgs + +def normalize(x): + """Normalization helper function.""" + return x / np.linalg.norm(x) + +def viewmatrix(lookdir, up, position): + """Construct lookat view matrix.""" + vec2 = normalize(lookdir) + vec0 = normalize(np.cross(up, vec2)) + vec1 = normalize(np.cross(vec2, vec0)) + m = np.stack([vec0, vec1, vec2, position], axis=1) + return m + +def poses_to_points(poses, dist): + """Converts from pose matrices to (position, lookat, up) format.""" + pos = poses[:, :3, -1] + lookat = poses[:, :3, -1] - dist * poses[:, :3, 2] + up = poses[:, :3, -1] + dist * poses[:, :3, 1] + return np.stack([pos, lookat, up], 1) + +def points_to_poses(points): + """Converts from (position, lookat, up) format to pose matrices.""" + return np.array([viewmatrix(p - l, u - p, p) for p, l, u in points]) + +def interp(points, n, k, s): + """Runs multidimensional B-spline interpolation on the input points.""" + sh = points.shape + pts = np.reshape(points, (sh[0], -1)) + k = min(k, sh[0] - 1) + tck, _ = scipy.interpolate.splprep(pts.T, k=k, s=s) + u = np.linspace(0, 1, n, endpoint=False) + new_points = np.array(scipy.interpolate.splev(u, tck)) + new_points = np.reshape(new_points.T, (n, sh[1], sh[2])) + return new_points + +def generate_interpolated_path(poses, n_interp, spline_degree=5, + smoothness=.03, rot_weight=.1): + """Creates a smooth spline path between input keyframe camera poses. + + Spline is calculated with poses in format (position, lookat-point, up-point). + + Args: + poses: (n, 3, 4) array of input pose keyframes. + n_interp: returned path will have n_interp * (n - 1) total poses. + spline_degree: polynomial degree of B-spline. + smoothness: parameter for spline smoothing, 0 forces exact interpolation. + rot_weight: relative weighting of rotation/translation in spline solve. + + Returns: + Array of new camera poses with shape (n_interp * (n - 1), 3, 4). + """ + + points = poses_to_points(poses, dist=rot_weight) + new_points = interp(points, + n_interp * (points.shape[0] - 1), + k=spline_degree, + s=smoothness) + return points_to_poses(new_points) + +def batch_visualize_tensor_global_pca(tensor_batch, num_components=3): + B, C, H, W = tensor_batch.shape + + tensor_flat_all = tensor_batch.reshape(B, C, -1).permute(1, 0, 2).reshape(C, -1).T + + tensor_flat_all_np = tensor_flat_all.cpu().numpy() + + scaler = StandardScaler() + tensor_flat_all_np = scaler.fit_transform(tensor_flat_all_np) + + pca = PCA(n_components=num_components) + tensor_reduced_all_np = pca.fit_transform(tensor_flat_all_np) + + tensor_reduced_all = torch.tensor(tensor_reduced_all_np, dtype=tensor_batch.dtype).T.reshape(num_components, B, H * W).permute(1, 0, 2) + + output_tensor = torch.zeros((B, 3, H, W)) + + for i in range(B): + tensor_reduced = tensor_reduced_all[i].reshape(num_components, H, W) + tensor_reduced -= tensor_reduced.min() + tensor_reduced /= tensor_reduced.max() + output_tensor[i] = tensor_reduced[:3] + + return output_tensor + +def depth_to_colormap(depth_tensor, colormap='jet'): + B, _, _, _ = depth_tensor.shape + + depth_tensor = (depth_tensor - depth_tensor.min()) / (depth_tensor.max() - depth_tensor.min()) + + depth_np = depth_tensor.squeeze(1).cpu().numpy() + + cmap = plt.get_cmap(colormap) + colored_images = [] + + for i in range(B): + colored_image = cmap(depth_np[i]) + colored_images.append(colored_image[..., :3]) + + colored_tensor = torch.tensor(np.array(colored_images), dtype=torch.float32).permute(0, 3, 1, 2) + + return colored_tensor + +def save_video(frames, video_path, fps=24): + clips = [mpy.ImageClip(frame).set_duration(1/fps) for frame in frames] + video = mpy.concatenate_videoclips(clips, method="compose") + video.write_videofile(video_path, fps=fps) + +def tensors_to_videos(all_images, all_depth_vis, all_fmap_vis, all_sems_vis, video_dir='videos', fps=24): + B, C, H, W = all_images.shape + assert all_depth_vis.shape == (B, C, H, W) + assert all_fmap_vis.shape == (B, C, H, W) + assert all_sems_vis.shape == (B, C, H, W) + os.makedirs(video_dir, exist_ok=True) + + all_images = (all_images.permute(0, 2, 3, 1).cpu().numpy() * 255).astype(np.uint8) + all_depth_vis = (all_depth_vis.permute(0, 2, 3, 1).cpu().numpy() * 255).astype(np.uint8) + all_fmap_vis = (all_fmap_vis.permute(0, 2, 3, 1).cpu().numpy() * 255).astype(np.uint8) + all_sems_vis = (all_sems_vis.permute(0, 2, 3, 1).cpu().numpy() * 255).astype(np.uint8) + + save_video(all_images, os.path.join(video_dir, 'output_images_video.mp4'), fps=fps) + save_video(all_depth_vis, os.path.join(video_dir, 'output_depth_video.mp4'), fps=fps) + save_video(all_fmap_vis, os.path.join(video_dir, 'output_fmap_video.mp4'), fps=fps) + # save_video(all_sems_vis, os.path.join(video_dir, 'output_sems_video.mp4'), fps=fps) + + print(f'Videos saved to {video_dir}') + +def transfer_images_to_device(images, device): + """ + Transfer the loaded images to the specified device. + + Args: + images (list): List of dictionaries containing image data. + device (str or torch.device): The device to transfer the data to. + + Returns: + list: List of dictionaries with image data transferred to the specified device. + """ + transferred_images = [] + for img_dict in images: + transferred_dict = { + 'img': img_dict['img'].to(device), + 'true_shape': torch.tensor(img_dict['true_shape'], device=device), + 'idx': img_dict['idx'], + 'instance': img_dict['instance'] + } + transferred_images.append(transferred_dict) + return transferred_images + +def render_camera_path(video_poses, camera_params, gaussians, model, device, pipeline, bg_color, image_shape): + """渲染相机路径的帮助函数 + + Args: + video_poses: 相机位姿列表 + camera_params: 包含extrinsics和intrinsics的相机参数 + gaussians: 高斯模型 + model: 特征提取模型 + device: 计算设备 + pipeline: 渲染管线 + bg_color: 背景颜色 + image_shape: 图像尺寸 + + Returns: + rendered_images: 渲染的图像 + rendered_feats: 渲染的特征图 + rendered_depths: 渲染的深度图 + rendered_sems: 渲染的语义图 + """ + extrinsics, intrinsics = camera_params + rendered_images = [] + rendered_feats = [] + rendered_depths = [] + rendered_sems = [] + + for i in range(len(video_poses)): + target_extrinsics = torch.zeros(4, 4).to(device) + target_extrinsics[3, 3] = 1.0 + target_extrinsics[:3, :4] = torch.tensor(video_poses[i], device=device) + camera = get_scaled_camera(extrinsics[0], target_extrinsics, intrinsics[0], 1.0, image_shape) + + rendered_output = render(camera, gaussians, pipeline, bg_color) + rendered_images.append(rendered_output['render']) + + # 处理特征图 + feature_map = rendered_output['feature_map'] + feature_map = model.feature_expansion(feature_map[None, ...]) + + # 处理语义图 + logits = model.lseg_feature_extractor.decode_feature(feature_map, labelset=LABELS) + semantic_map = torch.argmax(logits, dim=1) + 1 + mask = COLORS[semantic_map.cpu()] + mask = rearrange(mask, 'b h w c -> b c h w') + rendered_sems.append(mask.squeeze(0)) + + # 降采样并上采样特征图 + feature_map = feature_map[:, ::16, ...] + feature_map = torch.nn.functional.interpolate(feature_map, scale_factor=2, mode='bilinear', align_corners=True) + rendered_feats.append(feature_map[0]) + del feature_map + + rendered_depths.append(rendered_output['depth']) + + # 堆叠并处理结果 + rendered_images = torch.clamp(torch.stack(rendered_images, dim=0), 0, 1) + rendered_feats = torch.stack(rendered_feats, dim=0) + rendered_depths = torch.stack(rendered_depths, dim=0) + rendered_sems = torch.stack(rendered_sems, dim=0) + + return rendered_images, rendered_feats, rendered_depths, rendered_sems + +@torch.no_grad() +def render_video_from_file(file_list, model, output_path, device='cuda', resolution=224, n_interp=90, fps=30, path_type='default'): + # 1. load images + images = load_images(file_list, resolution, save_dir=os.path.join(output_path, 'processed_images')) + images = transfer_images_to_device(images, device) # Transfer images to the specified device + image_shape = images[0]['true_shape'][0] + # 2. get camera pose + pairs = make_pairs(images, prefilter=None, symmetrize=True) + output = inference(pairs, model.mast3r, device, batch_size=1) + mode = GlobalAlignerMode.PairViewer + scene = global_aligner(output, device=device, mode=mode) + extrinsics = scene.get_im_poses() + intrinsics = scene.get_intrinsics() + video_poses = generate_interpolated_path(extrinsics[:, :3, :].cpu().numpy(), n_interp=n_interp) # extrinsics: (b, 3, 4) + # 3. get gaussians + pred1, pred2 = model(*images) + pred = merge_and_split_predictions(pred1, pred2) + gaussians = GaussianModel.from_predictions(pred[0], sh_degree=3) + # 4. 渲染原始视角 + pipeline = DummyPipeline() + bg_color = torch.tensor([0.0, 0.0, 0.0]).to(device) + camera_params = (extrinsics, intrinsics) + + rendered_images, rendered_feats, rendered_depths, rendered_sems = render_camera_path( + video_poses, camera_params, gaussians, model, device, pipeline, bg_color, image_shape) + + # 5. 可视化 + all_fmap_vis = batch_visualize_tensor_global_pca(rendered_feats) + all_depth_vis = depth_to_colormap(rendered_depths) + all_sems_vis = rendered_sems + + # 6. 保存视频和高斯点云 + tensors_to_videos(rendered_images, all_depth_vis, all_fmap_vis, all_sems_vis, output_path, fps=fps) + gaussians.save_ply(os.path.join(output_path, 'gaussians.ply')) + + # 7. 渲染移动后的视角 + moved_extrinsics = move_c2w_along_z(extrinsics, 2.0) + moved_video_poses = generate_interpolated_path(moved_extrinsics[:, :3, :].cpu().numpy(), n_interp=n_interp) + camera_params = (extrinsics, intrinsics) + + moved_rendered_images, moved_rendered_feats, moved_rendered_depths, moved_rendered_sems = render_camera_path( + moved_video_poses, camera_params, gaussians, model, device, pipeline, bg_color, image_shape) + + # 8. 可视化和保存移动后的结果 + moved_all_fmap_vis = batch_visualize_tensor_global_pca(moved_rendered_feats) + moved_all_depth_vis = depth_to_colormap(moved_rendered_depths) + moved_all_sems_vis = moved_rendered_sems + + moved_output_path = os.path.join(output_path, 'moved') + os.makedirs(moved_output_path, exist_ok=True) + tensors_to_videos(moved_rendered_images, moved_all_depth_vis, moved_all_fmap_vis, moved_all_sems_vis, + moved_output_path, fps=fps) diff --git a/submodules/PointTransformerV3/.gitmodules b/submodules/PointTransformerV3/.gitmodules new file mode 100644 index 0000000000000000000000000000000000000000..ff58019469bcb12b67cf63404872ea5a36f8f9df --- /dev/null +++ b/submodules/PointTransformerV3/.gitmodules @@ -0,0 +1,3 @@ +[submodule "Pointcept"] + path = Pointcept + url = https://github.com/Pointcept/Pointcept diff --git a/submodules/PointTransformerV3/LICENSE b/submodules/PointTransformerV3/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..cfab869a7238443a78ff1b2eabd09e8d5ded7618 --- /dev/null +++ b/submodules/PointTransformerV3/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Pointcept + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/submodules/PointTransformerV3/Pointcept/.github/workflows/formatter.yml b/submodules/PointTransformerV3/Pointcept/.github/workflows/formatter.yml new file mode 100644 index 0000000000000000000000000000000000000000..a95391b200250e18be9b7e28fb04dfca8a902c2e --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/.github/workflows/formatter.yml @@ -0,0 +1,20 @@ +name: Formatter + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + types: [opened, reopened, synchronize] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + formatter: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: psf/black@stable diff --git a/submodules/PointTransformerV3/Pointcept/.gitignore b/submodules/PointTransformerV3/Pointcept/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..7288b0a938b724007d2705abdd611bc5967b8e3e --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/.gitignore @@ -0,0 +1,16 @@ +image/ +__pycache__ +**/build/ +**/*.egg-info/ +**/dist/ +*.so +exp +weights +data +log +outputs/ +.vscode +.idea +*/.DS_Store +**/*.out +Dockerfile diff --git a/submodules/PointTransformerV3/Pointcept/LICENSE b/submodules/PointTransformerV3/Pointcept/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..ee1fac1b22ae96f38e681900a3181d3e70ac6e4f --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Pointcept + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/submodules/PointTransformerV3/Pointcept/README.md b/submodules/PointTransformerV3/Pointcept/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cf84efb36681d48d7f6d7500ff3723d853d3f709 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/README.md @@ -0,0 +1,896 @@ +

+ + + + + + pointcept + +
+ +

+ +[![Formatter](https://github.com/pointcept/pointcept/actions/workflows/formatter.yml/badge.svg)](https://github.com/pointcept/pointcept/actions/workflows/formatter.yml) + +**Pointcept** is a powerful and flexible codebase for point cloud perception research. It is also an official implementation of the following paper: +- **Point Transformer V3: Simpler, Faster, Stronger** +*Xiaoyang Wu, Li Jiang, Peng-Shuai Wang, Zhijian Liu, Xihui Liu, Yu Qiao, Wanli Ouyang, Tong He, Hengshuang Zhao* +IEEE Conference on Computer Vision and Pattern Recognition (**CVPR**) 2024 - Oral +[ Backbone ] [PTv3] - [ [arXiv](https://arxiv.org/abs/2312.10035) ] [ [Bib](https://xywu.me/research/ptv3/bib.txt) ] [ [Project](https://github.com/Pointcept/PointTransformerV3) ] → [here](https://github.com/Pointcept/PointTransformerV3) + +- **OA-CNNs: Omni-Adaptive Sparse CNNs for 3D Semantic Segmentation** +*Bohao Peng, Xiaoyang Wu, Li Jiang, Yukang Chen, Hengshuang Zhao, Zhuotao Tian, Jiaya Jia* +IEEE Conference on Computer Vision and Pattern Recognition (**CVPR**) 2024 +[ Backbone ] [ OA-CNNs ] - [ [arXiv](https://arxiv.org/abs/2403.14418) ] [ [Bib](https://xywu.me/research/oacnns/bib.txt) ] → [here](#oa-cnns) + +- **PonderV2: Pave the Way for 3D Foundation Model with A Universal Pre-training Paradigm** +*Haoyi Zhu\*, Honghui Yang\*, Xiaoyang Wu\*, Di Huang\*, Sha Zhang, Xianglong He, Tong He, Hengshuang Zhao, Chunhua Shen, Yu Qiao, Wanli Ouyang* +arXiv Preprint 2023 +[ Pretrain ] [PonderV2] - [ [arXiv](https://arxiv.org/abs/2310.08586) ] [ [Bib](https://xywu.me/research/ponderv2/bib.txt) ] [ [Project](https://github.com/OpenGVLab/PonderV2) ] → [here](https://github.com/OpenGVLab/PonderV2) + + +- **Towards Large-scale 3D Representation Learning with Multi-dataset Point Prompt Training** +*Xiaoyang Wu, Zhuotao Tian, Xin Wen, Bohao Peng, Xihui Liu, Kaicheng Yu, Hengshuang Zhao* +IEEE Conference on Computer Vision and Pattern Recognition (**CVPR**) 2024 +[ Pretrain ] [PPT] - [ [arXiv](https://arxiv.org/abs/2308.09718) ] [ [Bib](https://xywu.me/research/ppt/bib.txt) ] → [here](#point-prompt-training-ppt) + +- **Masked Scene Contrast: A Scalable Framework for Unsupervised 3D Representation Learning** +*Xiaoyang Wu, Xin Wen, Xihui Liu, Hengshuang Zhao* +IEEE Conference on Computer Vision and Pattern Recognition (**CVPR**) 2023 +[ Pretrain ] [ MSC ] - [ [arXiv](https://arxiv.org/abs/2303.14191) ] [ [Bib](https://xywu.me/research/msc/bib.txt) ] → [here](#masked-scene-contrast-msc) + + +- **Learning Context-aware Classifier for Semantic Segmentation** (3D Part) +*Zhuotao Tian, Jiequan Cui, Li Jiang, Xiaojuan Qi, Xin Lai, Yixin Chen, Shu Liu, Jiaya Jia* +AAAI Conference on Artificial Intelligence (**AAAI**) 2023 - Oral +[ SemSeg ] [ CAC ] - [ [arXiv](https://arxiv.org/abs/2303.11633) ] [ [Bib](https://xywu.me/research/cac/bib.txt) ] [ [2D Part](https://github.com/tianzhuotao/CAC) ] → [here](#context-aware-classifier) + + +- **Point Transformer V2: Grouped Vector Attention and Partition-based Pooling** +*Xiaoyang Wu, Yixing Lao, Li Jiang, Xihui Liu, Hengshuang Zhao* +Conference on Neural Information Processing Systems (**NeurIPS**) 2022 +[ Backbone ] [ PTv2 ] - [ [arXiv](https://arxiv.org/abs/2210.05666) ] [ [Bib](https://xywu.me/research/ptv2/bib.txt) ] → [here](#point-transformers) + + +- **Point Transformer** +*Hengshuang Zhao, Li Jiang, Jiaya Jia, Philip Torr, Vladlen Koltun* +IEEE International Conference on Computer Vision (**ICCV**) 2021 - Oral +[ Backbone ] [ PTv1 ] - [ [arXiv](https://arxiv.org/abs/2012.09164) ] [ [Bib](https://hszhao.github.io/papers/iccv21_pointtransformer_bib.txt) ] → [here](#point-transformers) + +Additionally, **Pointcept** integrates the following excellent work (contain above): +Backbone: +[MinkUNet](https://github.com/NVIDIA/MinkowskiEngine) ([here](#sparseunet)), +[SpUNet](https://github.com/traveller59/spconv) ([here](#sparseunet)), +[SPVCNN](https://github.com/mit-han-lab/spvnas) ([here](#spvcnn)), +[OACNNs](https://arxiv.org/abs/2403.14418) ([here](#oa-cnns)), +[PTv1](https://arxiv.org/abs/2012.09164) ([here](#point-transformers)), +[PTv2](https://arxiv.org/abs/2210.05666) ([here](#point-transformers)), +[PTv3](https://arxiv.org/abs/2312.10035) ([here](#point-transformers)), +[StratifiedFormer](https://github.com/dvlab-research/Stratified-Transformer) ([here](#stratified-transformer)), +[OctFormer](https://github.com/octree-nn/octformer) ([here](#octformer)), +[Swin3D](https://github.com/microsoft/Swin3D) ([here](#swin3d)); +Semantic Segmentation: +[Mix3d](https://github.com/kumuji/mix3d) ([here](https://github.com/Pointcept/Pointcept/blob/main/configs/scannet/semseg-spunet-v1m1-0-base.py#L5)), +[CAC](https://arxiv.org/abs/2303.11633) ([here](#context-aware-classifier)); +Instance Segmentation: +[PointGroup](https://github.com/dvlab-research/PointGroup) ([here](#pointgroup)); +Pre-training: +[PointContrast](https://github.com/facebookresearch/PointContrast) ([here](#pointcontrast)), +[Contrastive Scene Contexts](https://github.com/facebookresearch/ContrastiveSceneContexts) ([here](#contrastive-scene-contexts)), +[Masked Scene Contrast](https://arxiv.org/abs/2303.14191) ([here](#masked-scene-contrast-msc)), +[Point Prompt Training](https://arxiv.org/abs/2308.09718) ([here](#point-prompt-training-ppt)); +Datasets: +[ScanNet](http://www.scan-net.org/) ([here](#scannet-v2)), +[ScanNet200](http://www.scan-net.org/) ([here](#scannet-v2)), +[ScanNet++](https://kaldir.vc.in.tum.de/scannetpp/) ([here](#scannet)), +[S3DIS](https://docs.google.com/forms/d/e/1FAIpQLScDimvNMCGhy_rmBA2gHfDu3naktRm6A8BPwAWWDv-Uhm6Shw/viewform?c=0&w=1) ([here](#s3dis)), +[Matterport3D](https://niessner.github.io/Matterport/) ([here](#matterport3d)), +[ArkitScene](https://github.com/apple/ARKitScenes), +[Structured3D](https://structured3d-dataset.org/) ([here](#structured3d)), +[SemanticKITTI](http://www.semantic-kitti.org/) ([here](#semantickitti)), +[nuScenes](https://www.nuscenes.org/nuscenes) ([here](#nuscenes)), +[ModelNet40](https://modelnet.cs.princeton.edu/) ([here](#modelnet)), +[Waymo](https://waymo.com/open/) ([here](#waymo)). + + +## Highlights +- *May, 2024*: In v1.5.2, we redesigned the default structure for each dataset for better performance. Please **re-preprocess** datasets or **download** our preprocessed datasets from **[here](https://huggingface.co/Pointcept)**. +- *Apr, 2024*: **PTv3** is selected as one of the 90 **Oral** papers (3.3% accepted papers, 0.78% submissions) by CVPR'24! +- *Mar, 2024*: We release code for **OA-CNNs**, accepted by CVPR'24. Issue related to **OA-CNNs** can @Pbihao. +- *Feb, 2024*: **PTv3** and **PPT** are accepted by CVPR'24, another **two** papers by our Pointcept team have also been accepted by CVPR'24 🎉🎉🎉. We will make them publicly available soon! +- *Dec, 2023*: **PTv3** is released on arXiv, and the code is available in Pointcept. PTv3 is an efficient backbone model that achieves SOTA performances across indoor and outdoor scenarios. +- *Aug, 2023*: **PPT** is released on arXiv. PPT presents a multi-dataset pre-training framework that achieves SOTA performance in both **indoor** and **outdoor** scenarios. It is compatible with various existing pre-training frameworks and backbones. A **pre-release** version of the code is accessible; for those interested, please feel free to contact me directly for access. +- *Mar, 2023*: We released our codebase, **Pointcept**, a highly potent tool for point cloud representation learning and perception. We welcome new work to join the _Pointcept_ family and highly recommend reading [Quick Start](#quick-start) before starting your trail. +- *Feb, 2023*: **MSC** and **CeCo** accepted by CVPR 2023. _MSC_ is a highly efficient and effective pretraining framework that facilitates cross-dataset large-scale pretraining, while _CeCo_ is a segmentation method specifically designed for long-tail datasets. Both approaches are compatible with all existing backbone models in our codebase, and we will soon make the code available for public use. +- *Jan, 2023*: **CAC**, oral work of AAAI 2023, has expanded its 3D result with the incorporation of Pointcept. This addition will allow CAC to serve as a pluggable segmentor within our codebase. +- *Sep, 2022*: **PTv2** accepted by NeurIPS 2022. It is a continuation of the Point Transformer. The proposed GVA theory can apply to most existing attention mechanisms, while Grid Pooling is also a practical addition to existing pooling methods. + +## Citation +If you find _Pointcept_ useful to your research, please cite our work as encouragement. (੭ˊ꒳​ˋ)੭✧ +``` +@misc{pointcept2023, + title={Pointcept: A Codebase for Point Cloud Perception Research}, + author={Pointcept Contributors}, + howpublished = {\url{https://github.com/Pointcept/Pointcept}}, + year={2023} +} +``` + +## Overview + +- [Installation](#installation) +- [Data Preparation](#data-preparation) +- [Quick Start](#quick-start) +- [Model Zoo](#model-zoo) +- [Citation](#citation) +- [Acknowledgement](#acknowledgement) + +## Installation + +### Requirements +- Ubuntu: 18.04 and above. +- CUDA: 11.3 and above. +- PyTorch: 1.10.0 and above. + +### Conda Environment + +```bash +conda create -n pointcept python=3.8 -y +conda activate pointcept +conda install ninja -y +# Choose version you want here: https://pytorch.org/get-started/previous-versions/ +conda install pytorch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 cudatoolkit=11.3 -c pytorch -y +conda install h5py pyyaml -c anaconda -y +conda install sharedarray tensorboard tensorboardx yapf addict einops scipy plyfile termcolor timm -c conda-forge -y +conda install pytorch-cluster pytorch-scatter pytorch-sparse -c pyg -y +pip install torch-geometric + +# spconv (SparseUNet) +# refer https://github.com/traveller59/spconv +pip install spconv-cu113 + +# PPT (clip) +pip install ftfy regex tqdm +pip install git+https://github.com/openai/CLIP.git + +# PTv1 & PTv2 or precise eval +cd libs/pointops +# usual +python setup.py install +# docker & multi GPU arch +TORCH_CUDA_ARCH_LIST="ARCH LIST" python setup.py install +# e.g. 7.5: RTX 3000; 8.0: a100 More available in: https://developer.nvidia.com/cuda-gpus +TORCH_CUDA_ARCH_LIST="7.5 8.0" python setup.py install +cd ../.. + +# Open3D (visualization, optional) +pip install open3d +``` + +## Data Preparation + +### ScanNet v2 + +The preprocessing supports semantic and instance segmentation for both `ScanNet20`, `ScanNet200`, and `ScanNet Data Efficient`. +- Download the [ScanNet](http://www.scan-net.org/) v2 dataset. +- Run preprocessing code for raw ScanNet as follows: + + ```bash + # RAW_SCANNET_DIR: the directory of downloaded ScanNet v2 raw dataset. + # PROCESSED_SCANNET_DIR: the directory of the processed ScanNet dataset (output dir). + python pointcept/datasets/preprocessing/scannet/preprocess_scannet.py --dataset_root ${RAW_SCANNET_DIR} --output_root ${PROCESSED_SCANNET_DIR} + ``` +- (Optional) Download ScanNet Data Efficient files: + ```bash + # download-scannet.py is the official download script + # or follow instructions here: https://kaldir.vc.in.tum.de/scannet_benchmark/data_efficient/documentation#download + python download-scannet.py --data_efficient -o ${RAW_SCANNET_DIR} + # unzip downloads + cd ${RAW_SCANNET_DIR}/tasks + unzip limited-annotation-points.zip + unzip limited-reconstruction-scenes.zip + # copy files to processed dataset folder + mkdir ${PROCESSED_SCANNET_DIR}/tasks + cp -r ${RAW_SCANNET_DIR}/tasks/points ${PROCESSED_SCANNET_DIR}/tasks + cp -r ${RAW_SCANNET_DIR}/tasks/scenes ${PROCESSED_SCANNET_DIR}/tasks + ``` +- (Alternative) Our preprocess data can be directly downloaded [[here](https://huggingface.co/datasets/Pointcept/scannet-compressed)], please agree the official license before download it. + +- Link processed dataset to codebase: + ```bash + # PROCESSED_SCANNET_DIR: the directory of the processed ScanNet dataset. + mkdir data + ln -s ${PROCESSED_SCANNET_DIR} ${CODEBASE_DIR}/data/scannet + ``` + +### ScanNet++ +- Download the [ScanNet++](https://kaldir.vc.in.tum.de/scannetpp/) dataset. +- Run preprocessing code for raw ScanNet++ as follows: + ```bash + # RAW_SCANNETPP_DIR: the directory of downloaded ScanNet++ raw dataset. + # PROCESSED_SCANNETPP_DIR: the directory of the processed ScanNet++ dataset (output dir). + # NUM_WORKERS: the number of workers for parallel preprocessing. + python pointcept/datasets/preprocessing/scannetpp/preprocess_scannetpp.py --dataset_root ${RAW_SCANNETPP_DIR} --output_root ${PROCESSED_SCANNETPP_DIR} --num_workers ${NUM_WORKERS} + ``` +- Sampling and chunking large point cloud data in train/val split as follows (only used for training): + ```bash + # PROCESSED_SCANNETPP_DIR: the directory of the processed ScanNet++ dataset (output dir). + # NUM_WORKERS: the number of workers for parallel preprocessing. + python pointcept/datasets/preprocessing/sampling_chunking_data.py --dataset_root ${PROCESSED_SCANNETPP_DIR} --grid_size 0.01 --chunk_range 6 6 --chunk_stride 3 3 --split train --num_workers ${NUM_WORKERS} + python pointcept/datasets/preprocessing/sampling_chunking_data.py --dataset_root ${PROCESSED_SCANNETPP_DIR} --grid_size 0.01 --chunk_range 6 6 --chunk_stride 3 3 --split val --num_workers ${NUM_WORKERS} + ``` +- (Alternative) Our preprocess data can be directly downloaded [[here](https://huggingface.co/datasets/Pointcept/scannetpp-compressed)], please agree the official license before download it. +- Link processed dataset to codebase: + ```bash + # PROCESSED_SCANNETPP_DIR: the directory of the processed ScanNet dataset. + mkdir data + ln -s ${PROCESSED_SCANNETPP_DIR} ${CODEBASE_DIR}/data/scannetpp + ``` + +### S3DIS + +- Download S3DIS data by filling this [Google form](https://docs.google.com/forms/d/e/1FAIpQLScDimvNMCGhy_rmBA2gHfDu3naktRm6A8BPwAWWDv-Uhm6Shw/viewform?c=0&w=1). Download the `Stanford3dDataset_v1.2.zip` file and unzip it. +- Fix error in `Area_5/office_19/Annotations/ceiling` Line 323474 (103.0�0000 => 103.000000). +- (Optional) Download Full 2D-3D S3DIS dataset (no XYZ) from [here](https://github.com/alexsax/2D-3D-Semantics) for parsing normal. +- Run preprocessing code for S3DIS as follows: + + ```bash + # S3DIS_DIR: the directory of downloaded Stanford3dDataset_v1.2 dataset. + # RAW_S3DIS_DIR: the directory of Stanford2d3dDataset_noXYZ dataset. (optional, for parsing normal) + # PROCESSED_S3DIS_DIR: the directory of processed S3DIS dataset (output dir). + + # S3DIS without aligned angle + python pointcept/datasets/preprocessing/s3dis/preprocess_s3dis.py --dataset_root ${S3DIS_DIR} --output_root ${PROCESSED_S3DIS_DIR} + # S3DIS with aligned angle + python pointcept/datasets/preprocessing/s3dis/preprocess_s3dis.py --dataset_root ${S3DIS_DIR} --output_root ${PROCESSED_S3DIS_DIR} --align_angle + # S3DIS with normal vector (recommended, normal is helpful) + python pointcept/datasets/preprocessing/s3dis/preprocess_s3dis.py --dataset_root ${S3DIS_DIR} --output_root ${PROCESSED_S3DIS_DIR} --raw_root ${RAW_S3DIS_DIR} --parse_normal + python pointcept/datasets/preprocessing/s3dis/preprocess_s3dis.py --dataset_root ${S3DIS_DIR} --output_root ${PROCESSED_S3DIS_DIR} --raw_root ${RAW_S3DIS_DIR} --align_angle --parse_normal + ``` + +- (Alternative) Our preprocess data can also be downloaded [[here](https://huggingface.co/datasets/Pointcept/s3dis-compressed +)] (with normal vector and aligned angle), please agree with the official license before downloading it. + +- Link processed dataset to codebase. + ```bash + # PROCESSED_S3DIS_DIR: the directory of processed S3DIS dataset. + mkdir data + ln -s ${PROCESSED_S3DIS_DIR} ${CODEBASE_DIR}/data/s3dis + ``` +### Structured3D + +- Download Structured3D panorama related and perspective (full) related zip files by filling this [Google form](https://docs.google.com/forms/d/e/1FAIpQLSc0qtvh4vHSoZaW6UvlXYy79MbcGdZfICjh4_t4bYofQIVIdw/viewform?pli=1) (no need to unzip them). +- Organize all downloaded zip file in one folder (`${STRUCT3D_DIR}`). +- Run preprocessing code for Structured3D as follows: + ```bash + # STRUCT3D_DIR: the directory of downloaded Structured3D dataset. + # PROCESSED_STRUCT3D_DIR: the directory of processed Structured3D dataset (output dir). + # NUM_WORKERS: Number for workers for preprocessing, default same as cpu count (might OOM). + export PYTHONPATH=./ + python pointcept/datasets/preprocessing/structured3d/preprocess_structured3d.py --dataset_root ${STRUCT3D_DIR} --output_root ${PROCESSED_STRUCT3D_DIR} --num_workers ${NUM_WORKERS} --grid_size 0.01 --fuse_prsp --fuse_pano + ``` +Following the instruction of [Swin3D](https://arxiv.org/abs/2304.06906), we keep 25 categories with frequencies of more than 0.001, out of the original 40 categories. + +[//]: # (- (Alternative) Our preprocess data can also be downloaded [[here]()], please agree the official license before download it.) + +- (Alternative) Our preprocess data can also be downloaded [[here](https://huggingface.co/datasets/Pointcept/structured3d-compressed +)] (with perspective views and panorama view, 471.7G after unzipping), please agree the official license before download it. + +- Link processed dataset to codebase. + ```bash + # PROCESSED_STRUCT3D_DIR: the directory of processed Structured3D dataset (output dir). + mkdir data + ln -s ${PROCESSED_STRUCT3D_DIR} ${CODEBASE_DIR}/data/structured3d + ``` +### Matterport3D +- Follow [this page](https://niessner.github.io/Matterport/#download) to request access to the dataset. +- Download the "region_segmentation" type, which represents the division of a scene into individual rooms. + ```bash + # download-mp.py is the official download script + # MATTERPORT3D_DIR: the directory of downloaded Matterport3D dataset. + python download-mp.py -o {MATTERPORT3D_DIR} --type region_segmentations + ``` +- Unzip the region_segmentations data + ```bash + # MATTERPORT3D_DIR: the directory of downloaded Matterport3D dataset. + python pointcept/datasets/preprocessing/matterport3d/unzip_matterport3d_region_segmentation.py --dataset_root {MATTERPORT3D_DIR} + ``` +- Run preprocessing code for Matterport3D as follows: + ```bash + # MATTERPORT3D_DIR: the directory of downloaded Matterport3D dataset. + # PROCESSED_MATTERPORT3D_DIR: the directory of processed Matterport3D dataset (output dir). + # NUM_WORKERS: the number of workers for this preprocessing. + python pointcept/datasets/preprocessing/matterport3d/preprocess_matterport3d_mesh.py --dataset_root ${MATTERPORT3D_DIR} --output_root ${PROCESSED_MATTERPORT3D_DIR} --num_workers ${NUM_WORKERS} + ``` +- Link processed dataset to codebase. + ```bash + # PROCESSED_MATTERPORT3D_DIR: the directory of processed Matterport3D dataset (output dir). + mkdir data + ln -s ${PROCESSED_MATTERPORT3D_DIR} ${CODEBASE_DIR}/data/matterport3d + ``` + +Following the instruction of [OpenRooms](https://github.com/ViLab-UCSD/OpenRooms), we remapped Matterport3D's categories to ScanNet 20 semantic categories with the addition of a ceiling category. +* (Alternative) Our preprocess data can also be downloaded [here](https://huggingface.co/datasets/Pointcept/matterport3d-compressed), please agree the official license before download it. + +### SemanticKITTI +- Download [SemanticKITTI](http://www.semantic-kitti.org/dataset.html#download) dataset. +- Link dataset to codebase. + ```bash + # SEMANTIC_KITTI_DIR: the directory of SemanticKITTI dataset. + # |- SEMANTIC_KITTI_DIR + # |- dataset + # |- sequences + # |- 00 + # |- 01 + # |- ... + + mkdir -p data + ln -s ${SEMANTIC_KITTI_DIR} ${CODEBASE_DIR}/data/semantic_kitti + ``` + +### nuScenes +- Download the official [NuScene](https://www.nuscenes.org/nuscenes#download) dataset (with Lidar Segmentation) and organize the downloaded files as follows: + ```bash + NUSCENES_DIR + │── samples + │── sweeps + │── lidarseg + ... + │── v1.0-trainval + │── v1.0-test + ``` +- Run information preprocessing code (modified from OpenPCDet) for nuScenes as follows: + ```bash + # NUSCENES_DIR: the directory of downloaded nuScenes dataset. + # PROCESSED_NUSCENES_DIR: the directory of processed nuScenes dataset (output dir). + # MAX_SWEEPS: Max number of sweeps. Default: 10. + pip install nuscenes-devkit pyquaternion + python pointcept/datasets/preprocessing/nuscenes/preprocess_nuscenes_info.py --dataset_root ${NUSCENES_DIR} --output_root ${PROCESSED_NUSCENES_DIR} --max_sweeps ${MAX_SWEEPS} --with_camera + ``` +- (Alternative) Our preprocess nuScenes information data can also be downloaded [[here]( +https://huggingface.co/datasets/Pointcept/nuscenes-compressed)] (only processed information, still need to download raw dataset and link to the folder), please agree the official license before download it. + +- Link raw dataset to processed NuScene dataset folder: + ```bash + # NUSCENES_DIR: the directory of downloaded nuScenes dataset. + # PROCESSED_NUSCENES_DIR: the directory of processed nuScenes dataset (output dir). + ln -s ${NUSCENES_DIR} {PROCESSED_NUSCENES_DIR}/raw + ``` + then the processed nuscenes folder is organized as follows: + ```bash + nuscene + |── raw + │── samples + │── sweeps + │── lidarseg + ... + │── v1.0-trainval + │── v1.0-test + |── info + ``` + +- Link processed dataset to codebase. + ```bash + # PROCESSED_NUSCENES_DIR: the directory of processed nuScenes dataset (output dir). + mkdir data + ln -s ${PROCESSED_NUSCENES_DIR} ${CODEBASE_DIR}/data/nuscenes + ``` + +### Waymo +- Download the official [Waymo](https://waymo.com/open/download/) dataset (v1.4.3) and organize the downloaded files as follows: + ```bash + WAYMO_RAW_DIR + │── training + │── validation + │── testing + ``` +- Install the following dependence: + ```bash + # If shows "No matching distribution found", download whl directly from Pypi and install the package. + conda create -n waymo python=3.10 -y + conda activate waymo + pip install waymo-open-dataset-tf-2-12-0 + ``` +- Run the preprocessing code as follows: + ```bash + # WAYMO_DIR: the directory of the downloaded Waymo dataset. + # PROCESSED_WAYMO_DIR: the directory of the processed Waymo dataset (output dir). + # NUM_WORKERS: num workers for preprocessing + python pointcept/datasets/preprocessing/waymo/preprocess_waymo.py --dataset_root ${WAYMO_DIR} --output_root ${PROCESSED_WAYMO_DIR} --splits training validation --num_workers ${NUM_WORKERS} + ``` + +- Link processed dataset to the codebase. + ```bash + # PROCESSED_WAYMO_DIR: the directory of the processed Waymo dataset (output dir). + mkdir data + ln -s ${PROCESSED_WAYMO_DIR} ${CODEBASE_DIR}/data/waymo + ``` + +### ModelNet +- Download [modelnet40_normal_resampled.zip](https://shapenet.cs.stanford.edu/media/modelnet40_normal_resampled.zip) and unzip +- Link dataset to the codebase. + ```bash + mkdir -p data + ln -s ${MODELNET_DIR} ${CODEBASE_DIR}/data/modelnet40_normal_resampled + ``` + +## Quick Start + +### Training +**Train from scratch.** The training processing is based on configs in `configs` folder. +The training script will generate an experiment folder in `exp` folder and backup essential code in the experiment folder. +Training config, log, tensorboard, and checkpoints will also be saved into the experiment folder during the training process. +```bash +export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} +# Script (Recommended) +sh scripts/train.sh -p ${INTERPRETER_PATH} -g ${NUM_GPU} -d ${DATASET_NAME} -c ${CONFIG_NAME} -n ${EXP_NAME} +# Direct +export PYTHONPATH=./ +python tools/train.py --config-file ${CONFIG_PATH} --num-gpus ${NUM_GPU} --options save_path=${SAVE_PATH} +``` + +For example: +```bash +# By script (Recommended) +# -p is default set as python and can be ignored +sh scripts/train.sh -p python -d scannet -c semseg-pt-v2m2-0-base -n semseg-pt-v2m2-0-base +# Direct +export PYTHONPATH=./ +python tools/train.py --config-file configs/scannet/semseg-pt-v2m2-0-base.py --options save_path=exp/scannet/semseg-pt-v2m2-0-base +``` +**Resume training from checkpoint.** If the training process is interrupted by accident, the following script can resume training from a given checkpoint. +```bash +export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} +# Script (Recommended) +# simply add "-r true" +sh scripts/train.sh -p ${INTERPRETER_PATH} -g ${NUM_GPU} -d ${DATASET_NAME} -c ${CONFIG_NAME} -n ${EXP_NAME} -r true +# Direct +export PYTHONPATH=./ +python tools/train.py --config-file ${CONFIG_PATH} --num-gpus ${NUM_GPU} --options save_path=${SAVE_PATH} resume=True weight=${CHECKPOINT_PATH} +``` + +### Testing +During training, model evaluation is performed on point clouds after grid sampling (voxelization), providing an initial assessment of model performance. However, to obtain precise evaluation results, testing is **essential**. The testing process involves subsampling a dense point cloud into a sequence of voxelized point clouds, ensuring comprehensive coverage of all points. These sub-results are then predicted and collected to form a complete prediction of the entire point cloud. This approach yields higher evaluation results compared to simply mapping/interpolating the prediction. In addition, our testing code supports TTA (test time augmentation) testing, which further enhances the stability of evaluation performance. + +```bash +# By script (Based on experiment folder created by training script) +sh scripts/test.sh -p ${INTERPRETER_PATH} -g ${NUM_GPU} -d ${DATASET_NAME} -n ${EXP_NAME} -w ${CHECKPOINT_NAME} +# Direct +export PYTHONPATH=./ +python tools/test.py --config-file ${CONFIG_PATH} --num-gpus ${NUM_GPU} --options save_path=${SAVE_PATH} weight=${CHECKPOINT_PATH} +``` +For example: +```bash +# By script (Based on experiment folder created by training script) +# -p is default set as python and can be ignored +# -w is default set as model_best and can be ignored +sh scripts/test.sh -p python -d scannet -n semseg-pt-v2m2-0-base -w model_best +# Direct +export PYTHONPATH=./ +python tools/test.py --config-file configs/scannet/semseg-pt-v2m2-0-base.py --options save_path=exp/scannet/semseg-pt-v2m2-0-base weight=exp/scannet/semseg-pt-v2m2-0-base/model/model_best.pth +``` + +The TTA can be disabled by replace `data.test.test_cfg.aug_transform = [...]` with: + +```python +data = dict( + train = dict(...), + val = dict(...), + test = dict( + ..., + test_cfg = dict( + ..., + aug_transform = [ + [dict(type="RandomRotateTargetAngle", angle=[0], axis="z", center=[0, 0, 0], p=1)] + ] + ) + ) +) +``` + +### Offset +`Offset` is the separator of point clouds in batch data, and it is similar to the concept of `Batch` in PyG. +A visual illustration of batch and offset is as follows: +

+ + + + + + pointcept + +
+ +

+ +## Model Zoo +### 1. Backbones and Semantic Segmentation +#### SparseUNet + +_Pointcept_ provides `SparseUNet` implemented by `SpConv` and `MinkowskiEngine`. The SpConv version is recommended since SpConv is easy to install and faster than MinkowskiEngine. Meanwhile, SpConv is also widely applied in outdoor perception. + +- **SpConv (recommend)** + +The SpConv version `SparseUNet` in the codebase was fully rewrite from `MinkowskiEngine` version, example running script is as follows: + +```bash +# ScanNet val +sh scripts/train.sh -g 4 -d scannet -c semseg-spunet-v1m1-0-base -n semseg-spunet-v1m1-0-base +# ScanNet200 +sh scripts/train.sh -g 4 -d scannet200 -c semseg-spunet-v1m1-0-base -n semseg-spunet-v1m1-0-base +# S3DIS +sh scripts/train.sh -g 4 -d s3dis -c semseg-spunet-v1m1-0-base -n semseg-spunet-v1m1-0-base +# S3DIS (with normal) +sh scripts/train.sh -g 4 -d s3dis -c semseg-spunet-v1m1-0-cn-base -n semseg-spunet-v1m1-0-cn-base +# SemanticKITTI +sh scripts/train.sh -g 4 -d semantic_kitti -c semseg-spunet-v1m1-0-base -n semseg-spunet-v1m1-0-base +# nuScenes +sh scripts/train.sh -g 4 -d nuscenes -c semseg-spunet-v1m1-0-base -n semseg-spunet-v1m1-0-base +# ModelNet40 +sh scripts/train.sh -g 2 -d modelnet40 -c cls-spunet-v1m1-0-base -n cls-spunet-v1m1-0-base + +# ScanNet Data Efficient +sh scripts/train.sh -g 4 -d scannet -c semseg-spunet-v1m1-2-efficient-la20 -n semseg-spunet-v1m1-2-efficient-la20 +sh scripts/train.sh -g 4 -d scannet -c semseg-spunet-v1m1-2-efficient-la50 -n semseg-spunet-v1m1-2-efficient-la50 +sh scripts/train.sh -g 4 -d scannet -c semseg-spunet-v1m1-2-efficient-la100 -n semseg-spunet-v1m1-2-efficient-la100 +sh scripts/train.sh -g 4 -d scannet -c semseg-spunet-v1m1-2-efficient-la200 -n semseg-spunet-v1m1-2-efficient-la200 +sh scripts/train.sh -g 4 -d scannet -c semseg-spunet-v1m1-2-efficient-lr1 -n semseg-spunet-v1m1-2-efficient-lr1 +sh scripts/train.sh -g 4 -d scannet -c semseg-spunet-v1m1-2-efficient-lr5 -n semseg-spunet-v1m1-2-efficient-lr5 +sh scripts/train.sh -g 4 -d scannet -c semseg-spunet-v1m1-2-efficient-lr10 -n semseg-spunet-v1m1-2-efficient-lr10 +sh scripts/train.sh -g 4 -d scannet -c semseg-spunet-v1m1-2-efficient-lr20 -n semseg-spunet-v1m1-2-efficient-lr20 + +# Profile model run time +sh scripts/train.sh -g 4 -d scannet -c semseg-spunet-v1m1-0-enable-profiler -n semseg-spunet-v1m1-0-enable-profiler +``` + +- **MinkowskiEngine** + +The MinkowskiEngine version `SparseUNet` in the codebase was modified from the original MinkowskiEngine repo, and example running scripts are as follows: +1. Install MinkowskiEngine, refer https://github.com/NVIDIA/MinkowskiEngine +2. Training with the following example scripts: +```bash +# Uncomment "# from .sparse_unet import *" in "pointcept/models/__init__.py" +# Uncomment "# from .mink_unet import *" in "pointcept/models/sparse_unet/__init__.py" +# ScanNet +sh scripts/train.sh -g 4 -d scannet -c semseg-minkunet34c-0-base -n semseg-minkunet34c-0-base +# ScanNet200 +sh scripts/train.sh -g 4 -d scannet200 -c semseg-minkunet34c-0-base -n semseg-minkunet34c-0-base +# S3DIS +sh scripts/train.sh -g 4 -d s3dis -c semseg-minkunet34c-0-base -n semseg-minkunet34c-0-base +# SemanticKITTI +sh scripts/train.sh -g 2 -d semantic_kitti -c semseg-minkunet34c-0-base -n semseg-minkunet34c-0-base +``` + +#### OA-CNNs +Introducing Omni-Adaptive 3D CNNs (**OA-CNNs**), a family of networks that integrates a lightweight module to greatly enhance the adaptivity of sparse CNNs at minimal computational cost. Without any self-attention modules, **OA-CNNs** favorably surpass point transformers in terms of accuracy in both indoor and outdoor scenes, with much less latency and memory cost. Issue related to **OA-CNNs** can @Pbihao. +```bash +# ScanNet +sh scripts/train.sh -g 4 -d scannet -c semseg-oacnns-v1m1-0-base -n semseg-oacnns-v1m1-0-base +``` + +#### Point Transformers +- **PTv3** + +[PTv3](https://arxiv.org/abs/2312.10035) is an efficient backbone model that achieves SOTA performances across indoor and outdoor scenarios. The full PTv3 relies on FlashAttention, while FlashAttention relies on CUDA 11.6 and above, make sure your local Pointcept environment satisfies the requirements. + +If you can not upgrade your local environment to satisfy the requirements (CUDA >= 11.6), then you can disable FlashAttention by setting the model parameter `enable_flash` to `false` and reducing the `enc_patch_size` and `dec_patch_size` to a level (e.g. 128). + +FlashAttention force disables RPE and forces the accuracy reduced to fp16. If you require these features, please disable `enable_flash` and adjust `enable_rpe`, `upcast_attention` and`upcast_softmax`. + +Detailed instructions and experiment records (containing weights) are available on the [project repository](https://github.com/Pointcept/PointTransformerV3). Example running scripts are as follows: +```bash +# Scratched ScanNet +sh scripts/train.sh -g 4 -d scannet -c semseg-pt-v3m1-0-base -n semseg-pt-v3m1-0-base +# PPT joint training (ScanNet + Structured3D) and evaluate in ScanNet +sh scripts/train.sh -g 8 -d scannet -c semseg-pt-v3m1-1-ppt-extreme -n semseg-pt-v3m1-1-ppt-extreme + +# Scratched ScanNet200 +sh scripts/train.sh -g 4 -d scannet200 -c semseg-pt-v3m1-0-base -n semseg-pt-v3m1-0-base +# Fine-tuning from PPT joint training (ScanNet + Structured3D) with ScanNet200 +# PTV3_PPT_WEIGHT_PATH: Path to model weight trained by PPT multi-dataset joint training +# e.g. exp/scannet/semseg-pt-v3m1-1-ppt-extreme/model/model_best.pth +sh scripts/train.sh -g 4 -d scannet200 -c semseg-pt-v3m1-1-ppt-ft -n semseg-pt-v3m1-1-ppt-ft -w ${PTV3_PPT_WEIGHT_PATH} + +# Scratched ScanNet++ +sh scripts/train.sh -g 4 -d scannetpp -c semseg-pt-v3m1-0-base -n semseg-pt-v3m1-0-base +# Scratched ScanNet++ test +sh scripts/train.sh -g 4 -d scannetpp -c semseg-pt-v3m1-1-submit -n semseg-pt-v3m1-1-submit + + +# Scratched S3DIS +sh scripts/train.sh -g 4 -d s3dis -c semseg-pt-v3m1-0-base -n semseg-pt-v3m1-0-base +# an example for disbale flash_attention and enable rpe. +sh scripts/train.sh -g 4 -d s3dis -c semseg-pt-v3m1-1-rpe -n semseg-pt-v3m1-0-rpe +# PPT joint training (ScanNet + S3DIS + Structured3D) and evaluate in ScanNet +sh scripts/train.sh -g 8 -d s3dis -c semseg-pt-v3m1-1-ppt-extreme -n semseg-pt-v3m1-1-ppt-extreme +# S3DIS 6-fold cross validation +# 1. The default configs are evaluated on Area_5, modify the "data.train.split", "data.val.split", and "data.test.split" to make the config evaluated on Area_1 ~ Area_6 respectively. +# 2. Train and evaluate the model on each split of areas and gather result files located in "exp/s3dis/EXP_NAME/result/Area_x.pth" in one single folder, noted as RECORD_FOLDER. +# 3. Run the following script to get S3DIS 6-fold cross validation performance: +export PYTHONPATH=./ +python tools/test_s3dis_6fold.py --record_root ${RECORD_FOLDER} + +# Scratched nuScenes +sh scripts/train.sh -g 4 -d nuscenes -c semseg-pt-v3m1-0-base -n semseg-pt-v3m1-0-base +# Scratched Waymo +sh scripts/train.sh -g 4 -d waymo -c semseg-pt-v3m1-0-base -n semseg-pt-v3m1-0-base + +# More configs and exp records for PTv3 will be available soon. +``` + +Indoor semantic segmentation +| Model | Benchmark | Additional Data | Num GPUs | Val mIoU | Config | Tensorboard | Exp Record | +| :---: | :---: |:---------------:| :---: | :---: | :---: | :---: | :---: | +| PTv3 | ScanNet | ✗ | 4 | 77.6% | [link](https://github.com/Pointcept/Pointcept/blob/main/configs/scannet/semseg-pt-v3m1-0-base.py) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tensorboard) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tree/main/scannet-semseg-pt-v3m1-0-base) | +| PTv3 + PPT | ScanNet | ✓ | 8 | 78.5% | [link](https://github.com/Pointcept/Pointcept/blob/main/configs/scannet/semseg-pt-v3m1-1-ppt-extreme.py) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tensorboard) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tree/main/scannet-semseg-pt-v3m1-1-ppt-extreme) | +| PTv3 | ScanNet200 | ✗ | 4 | 35.3% | [link](https://github.com/Pointcept/Pointcept/blob/main/configs/scannet200/semseg-pt-v3m1-0-base.py) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tensorboard) |[link](https://huggingface.co/Pointcept/PointTransformerV3/tree/main/scannet200-semseg-pt-v3m1-0-base)| +| PTv3 + PPT | ScanNet200 | ✓ (f.t.) | 4 | | | | | +| PTv3 | S3DIS (Area5) | ✗ | 4 | 73.6% | [link](https://github.com/Pointcept/Pointcept/blob/main/configs/s3dis/semseg-pt-v3m1-0-rpe.py) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tensorboard) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tree/main/s3dis-semseg-pt-v3m1-0-rpe) | +| PTv3 + PPT | S3DIS (Area5) | ✓ | 8 | 75.4% | [link](https://github.com/Pointcept/Pointcept/blob/main/configs/s3dis/semseg-pt-v3m1-1-ppt-extreme.py) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tensorboard) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tree/main/s3dis-semseg-pt-v3m1-1-ppt-extreme) | + +Outdoor semantic segmentation +| Model | Benchmark | Additional Data | Num GPUs | Val mIoU | Config | Tensorboard | Exp Record | +| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| PTv3 | nuScenes | ✗ | 4 | 80.3 | [link](https://github.com/Pointcept/Pointcept/blob/main/configs/nuscenes/semseg-pt-v3m1-0-base.py) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tensorboard)|[link](https://huggingface.co/Pointcept/PointTransformerV3/tree/main/nuscenes-semseg-pt-v3m1-0-base) | +| PTv3 + PPT | nuScenes | ✓ | 8 | | | | | +| PTv3 | SemanticKITTI | ✗ | 4 | | | | | +| PTv3 + PPT | SemanticKITTI | ✓ | 8 | | | | | +| PTv3 | Waymo | ✗ | 4 | 71.2 | [link](https://github.com/Pointcept/Pointcept/blob/main/configs/waymo/semseg-pt-v3m1-0-base.py) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tensorboard) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tree/main/waymo-semseg-pt-v3m1-0-base) (log only) | +| PTv3 + PPT | Waymo | ✓ | 8 | | | | | + +_**\*Released model weights are trained for v1.5.1, weights for v1.5.2 and later is still ongoing.**_ + +- **PTv2 mode2** + +The original PTv2 was trained on 4 * RTX a6000 (48G memory). Even enabling AMP, the memory cost of the original PTv2 is slightly larger than 24G. Considering GPUs with 24G memory are much more accessible, I tuned the PTv2 on the latest Pointcept and made it runnable on 4 * RTX 3090 machines. + +`PTv2 Mode2` enables AMP and disables _Position Encoding Multiplier_ & _Grouped Linear_. During our further research, we found that precise coordinates are not necessary for point cloud understanding (Replacing precise coordinates with grid coordinates doesn't influence the performance. Also, SparseUNet is an example). As for Grouped Linear, my implementation of Grouped Linear seems to cost more memory than the Linear layer provided by PyTorch. Benefiting from the codebase and better parameter tuning, we also relieve the overfitting problem. The reproducing performance is even better than the results reported in our paper. + +Example running scripts are as follows: + +```bash +# ptv2m2: PTv2 mode2, disable PEM & Grouped Linear, GPU memory cost < 24G (recommend) +# ScanNet +sh scripts/train.sh -g 4 -d scannet -c semseg-pt-v2m2-0-base -n semseg-pt-v2m2-0-base +sh scripts/train.sh -g 4 -d scannet -c semseg-pt-v2m2-3-lovasz -n semseg-pt-v2m2-3-lovasz + +# ScanNet test +sh scripts/train.sh -g 4 -d scannet -c semseg-pt-v2m2-1-submit -n semseg-pt-v2m2-1-submit +# ScanNet200 +sh scripts/train.sh -g 4 -d scannet200 -c semseg-pt-v2m2-0-base -n semseg-pt-v2m2-0-base +# ScanNet++ +sh scripts/train.sh -g 4 -d scannetpp -c semseg-pt-v2m2-0-base -n semseg-pt-v2m2-0-base +# ScanNet++ test +sh scripts/train.sh -g 4 -d scannetpp -c semseg-pt-v2m2-1-submit -n semseg-pt-v2m2-1-submit +# S3DIS +sh scripts/train.sh -g 4 -d s3dis -c semseg-pt-v2m2-0-base -n semseg-pt-v2m2-0-base +# SemanticKITTI +sh scripts/train.sh -g 4 -d semantic_kitti -c semseg-pt-v2m2-0-base -n semseg-pt-v2m2-0-base +# nuScenes +sh scripts/train.sh -g 4 -d nuscenes -c semseg-pt-v2m2-0-base -n semseg-pt-v2m2-0-base +``` + +- **PTv2 mode1** + +`PTv2 mode1` is the original PTv2 we reported in our paper, example running scripts are as follows: + +```bash +# ptv2m1: PTv2 mode1, Original PTv2, GPU memory cost > 24G +# ScanNet +sh scripts/train.sh -g 4 -d scannet -c semseg-pt-v2m1-0-base -n semseg-pt-v2m1-0-base +# ScanNet200 +sh scripts/train.sh -g 4 -d scannet200 -c semseg-pt-v2m1-0-base -n semseg-pt-v2m1-0-base +# S3DIS +sh scripts/train.sh -g 4 -d s3dis -c semseg-pt-v2m1-0-base -n semseg-pt-v2m1-0-base +``` + +- **PTv1** + +The original PTv1 is also available in our Pointcept codebase. I haven't run PTv1 for a long time, but I have ensured that the example running script works well. + +```bash +# ScanNet +sh scripts/train.sh -g 4 -d scannet -c semseg-pt-v1-0-base -n semseg-pt-v1-0-base +# ScanNet200 +sh scripts/train.sh -g 4 -d scannet200 -c semseg-pt-v1-0-base -n semseg-pt-v1-0-base +# S3DIS +sh scripts/train.sh -g 4 -d s3dis -c semseg-pt-v1-0-base -n semseg-pt-v1-0-base +``` + + +#### Stratified Transformer +1. Additional requirements: +```bash +pip install torch-points3d +# Fix dependence, caused by installing torch-points3d +pip uninstall SharedArray +pip install SharedArray==3.2.1 + +cd libs/pointops2 +python setup.py install +cd ../.. +``` +2. Uncomment `# from .stratified_transformer import *` in `pointcept/models/__init__.py`. +3. Refer [Optional Installation](installation) to install dependence. +4. Training with the following example scripts: +```bash +# stv1m1: Stratified Transformer mode1, Modified from the original Stratified Transformer code. +# PTv2m2: Stratified Transformer mode2, My rewrite version (recommend). + +# ScanNet +sh scripts/train.sh -g 4 -d scannet -c semseg-st-v1m2-0-refined -n semseg-st-v1m2-0-refined +sh scripts/train.sh -g 4 -d scannet -c semseg-st-v1m1-0-origin -n semseg-st-v1m1-0-origin +# ScanNet200 +sh scripts/train.sh -g 4 -d scannet200 -c semseg-st-v1m2-0-refined -n semseg-st-v1m2-0-refined +# S3DIS +sh scripts/train.sh -g 4 -d s3dis -c semseg-st-v1m2-0-refined -n semseg-st-v1m2-0-refined +``` + +#### SPVCNN +`SPVCNN` is a baseline model of [SPVNAS](https://github.com/mit-han-lab/spvnas), it is also a practical baseline for outdoor datasets. +1. Install torchsparse: +```bash +# refer https://github.com/mit-han-lab/torchsparse +# install method without sudo apt install +conda install google-sparsehash -c bioconda +export C_INCLUDE_PATH=${CONDA_PREFIX}/include:$C_INCLUDE_PATH +export CPLUS_INCLUDE_PATH=${CONDA_PREFIX}/include:CPLUS_INCLUDE_PATH +pip install --upgrade git+https://github.com/mit-han-lab/torchsparse.git +``` +2. Training with the following example scripts: +```bash +# SemanticKITTI +sh scripts/train.sh -g 2 -d semantic_kitti -c semseg-spvcnn-v1m1-0-base -n semseg-spvcnn-v1m1-0-base +``` + +#### OctFormer +OctFormer from _OctFormer: Octree-based Transformers for 3D Point Clouds_. +1. Additional requirements: +```bash +cd libs +git clone https://github.com/octree-nn/dwconv.git +pip install ./dwconv +pip install ocnn +``` +2. Uncomment `# from .octformer import *` in `pointcept/models/__init__.py`. +2. Training with the following example scripts: +```bash +# ScanNet +sh scripts/train.sh -g 4 -d scannet -c semseg-octformer-v1m1-0-base -n semseg-octformer-v1m1-0-base +``` + +#### Swin3D +Swin3D from _Swin3D: A Pretrained Transformer Backbone for 3D Indoor Scene Understanding_. +1. Additional requirements: +```bash +# 1. Install MinkEngine v0.5.4, follow readme in https://github.com/NVIDIA/MinkowskiEngine; +# 2. Install Swin3D, mainly for cuda operation: +cd libs +git clone https://github.com/microsoft/Swin3D.git +cd Swin3D +pip install ./ +``` +2. Uncomment `# from .swin3d import *` in `pointcept/models/__init__.py`. +3. Pre-Training with the following example scripts (Structured3D preprocessing refer [here](#structured3d)): +```bash +# Structured3D + Swin-S +sh scripts/train.sh -g 4 -d structured3d -c semseg-swin3d-v1m1-0-small -n semseg-swin3d-v1m1-0-small +# Structured3D + Swin-L +sh scripts/train.sh -g 4 -d structured3d -c semseg-swin3d-v1m1-1-large -n semseg-swin3d-v1m1-1-large + +# Addition +# Structured3D + SpUNet +sh scripts/train.sh -g 4 -d structured3d -c semseg-spunet-v1m1-0-base -n semseg-spunet-v1m1-0-base +# Structured3D + PTv2 +sh scripts/train.sh -g 4 -d structured3d -c semseg-pt-v2m2-0-base -n semseg-pt-v2m2-0-base +``` +4. Fine-tuning with the following example scripts: +```bash +# ScanNet + Swin-S +sh scripts/train.sh -g 4 -d scannet -w exp/structured3d/semseg-swin3d-v1m1-1-large/model/model_last.pth -c semseg-swin3d-v1m1-0-small -n semseg-swin3d-v1m1-0-small +# ScanNet + Swin-L +sh scripts/train.sh -g 4 -d scannet -w exp/structured3d/semseg-swin3d-v1m1-1-large/model/model_last.pth -c semseg-swin3d-v1m1-1-large -n semseg-swin3d-v1m1-1-large + +# S3DIS + Swin-S (here we provide config support S3DIS normal vector) +sh scripts/train.sh -g 4 -d s3dis -w exp/structured3d/semseg-swin3d-v1m1-1-large/model/model_last.pth -c semseg-swin3d-v1m1-0-small -n semseg-swin3d-v1m1-0-small +# S3DIS + Swin-L (here we provide config support S3DIS normal vector) +sh scripts/train.sh -g 4 -d s3dis -w exp/structured3d/semseg-swin3d-v1m1-1-large/model/model_last.pth -c semseg-swin3d-v1m1-1-large -n semseg-swin3d-v1m1-1-large +``` + +#### Context-Aware Classifier +`Context-Aware Classifier` is a segmentor that can further boost the performance of each backbone, as a replacement for `Default Segmentor`. Training with the following example scripts: +```bash +# ScanNet +sh scripts/train.sh -g 4 -d scannet -c semseg-cac-v1m1-0-spunet-base -n semseg-cac-v1m1-0-spunet-base +sh scripts/train.sh -g 4 -d scannet -c semseg-cac-v1m1-1-spunet-lovasz -n semseg-cac-v1m1-1-spunet-lovasz +sh scripts/train.sh -g 4 -d scannet -c semseg-cac-v1m1-2-ptv2-lovasz -n semseg-cac-v1m1-2-ptv2-lovasz + +# ScanNet200 +sh scripts/train.sh -g 4 -d scannet200 -c semseg-cac-v1m1-0-spunet-base -n semseg-cac-v1m1-0-spunet-base +sh scripts/train.sh -g 4 -d scannet200 -c semseg-cac-v1m1-1-spunet-lovasz -n semseg-cac-v1m1-1-spunet-lovasz +sh scripts/train.sh -g 4 -d scannet200 -c semseg-cac-v1m1-2-ptv2-lovasz -n semseg-cac-v1m1-2-ptv2-lovasz +``` + + +### 2. Instance Segmentation +#### PointGroup +[PointGroup](https://github.com/dvlab-research/PointGroup) is a baseline framework for point cloud instance segmentation. +1. Additional requirements: +```bash +conda install -c bioconda google-sparsehash +cd libs/pointgroup_ops +python setup.py install --include_dirs=${CONDA_PREFIX}/include +cd ../.. +``` +2. Uncomment `# from .point_group import *` in `pointcept/models/__init__.py`. +3. Training with the following example scripts: +```bash +# ScanNet +sh scripts/train.sh -g 4 -d scannet -c insseg-pointgroup-v1m1-0-spunet-base -n insseg-pointgroup-v1m1-0-spunet-base +# S3DIS +sh scripts/train.sh -g 4 -d scannet -c insseg-pointgroup-v1m1-0-spunet-base -n insseg-pointgroup-v1m1-0-spunet-base +``` + +### 3. Pre-training +#### Masked Scene Contrast (MSC) +1. Pre-training with the following example scripts: +```bash +# ScanNet +sh scripts/train.sh -g 8 -d scannet -c pretrain-msc-v1m1-0-spunet-base -n pretrain-msc-v1m1-0-spunet-base +``` + +2. Fine-tuning with the following example scripts: +enable PointGroup ([here](#pointgroup)) before fine-tuning on instance segmentation task. +```bash +# ScanNet20 Semantic Segmentation +sh scripts/train.sh -g 8 -d scannet -w exp/scannet/pretrain-msc-v1m1-0-spunet-base/model/model_last.pth -c semseg-spunet-v1m1-4-ft -n semseg-msc-v1m1-0f-spunet-base +# ScanNet20 Instance Segmentation (enable PointGroup before running the script) +sh scripts/train.sh -g 4 -d scannet -w exp/scannet/pretrain-msc-v1m1-0-spunet-base/model/model_last.pth -c insseg-pointgroup-v1m1-0-spunet-base -n insseg-msc-v1m1-0f-pointgroup-spunet-base +``` +3. Example log and weight: [[Pretrain](https://connecthkuhk-my.sharepoint.com/:u:/g/personal/wuxy_connect_hku_hk/EYvNV4XUJ_5Mlk-g15RelN4BW_P8lVBfC_zhjC_BlBDARg?e=UoGFWH)] [[Semseg](https://connecthkuhk-my.sharepoint.com/:u:/g/personal/wuxy_connect_hku_hk/EQkDiv5xkOFKgCpGiGtAlLwBon7i8W6my3TIbGVxuiTttQ?e=tQFnbr)] + +#### Point Prompt Training (PPT) +PPT presents a multi-dataset pre-training framework, and it is compatible with various existing pre-training frameworks and backbones. +1. PPT supervised joint training with the following example scripts: +```bash +# ScanNet + Structured3d, validate on ScanNet (S3DIS might cause long data time, w/o S3DIS for a quick validation) >= 3090 * 8 +sh scripts/train.sh -g 8 -d scannet -c semseg-ppt-v1m1-0-sc-st-spunet -n semseg-ppt-v1m1-0-sc-st-spunet +sh scripts/train.sh -g 8 -d scannet -c semseg-ppt-v1m1-1-sc-st-spunet-submit -n semseg-ppt-v1m1-1-sc-st-spunet-submit +# ScanNet + S3DIS + Structured3d, validate on S3DIS (>= a100 * 8) +sh scripts/train.sh -g 8 -d s3dis -c semseg-ppt-v1m1-0-s3-sc-st-spunet -n semseg-ppt-v1m1-0-s3-sc-st-spunet +# SemanticKITTI + nuScenes + Waymo, validate on SemanticKITTI (bs12 >= 3090 * 4 >= 3090 * 8, v1m1-0 is still on tuning) +sh scripts/train.sh -g 4 -d semantic_kitti -c semseg-ppt-v1m1-0-nu-sk-wa-spunet -n semseg-ppt-v1m1-0-nu-sk-wa-spunet +sh scripts/train.sh -g 4 -d semantic_kitti -c semseg-ppt-v1m2-0-sk-nu-wa-spunet -n semseg-ppt-v1m2-0-sk-nu-wa-spunet +sh scripts/train.sh -g 4 -d semantic_kitti -c semseg-ppt-v1m2-1-sk-nu-wa-spunet-submit -n semseg-ppt-v1m2-1-sk-nu-wa-spunet-submit +# SemanticKITTI + nuScenes + Waymo, validate on nuScenes (bs12 >= 3090 * 4; bs24 >= 3090 * 8, v1m1-0 is still on tuning)) +sh scripts/train.sh -g 4 -d nuscenes -c semseg-ppt-v1m1-0-nu-sk-wa-spunet -n semseg-ppt-v1m1-0-nu-sk-wa-spunet +sh scripts/train.sh -g 4 -d nuscenes -c semseg-ppt-v1m2-0-nu-sk-wa-spunet -n semseg-ppt-v1m2-0-nu-sk-wa-spunet +sh scripts/train.sh -g 4 -d nuscenes -c semseg-ppt-v1m2-1-nu-sk-wa-spunet-submit -n semseg-ppt-v1m2-1-nu-sk-wa-spunet-submit +``` + +#### PointContrast +1. Preprocess and link ScanNet-Pair dataset (pair-wise matching with ScanNet raw RGB-D frame, ~1.5T): +```bash +# RAW_SCANNET_DIR: the directory of downloaded ScanNet v2 raw dataset. +# PROCESSED_SCANNET_PAIR_DIR: the directory of processed ScanNet pair dataset (output dir). +python pointcept/datasets/preprocessing/scannet/scannet_pair/preprocess.py --dataset_root ${RAW_SCANNET_DIR} --output_root ${PROCESSED_SCANNET_PAIR_DIR} +ln -s ${PROCESSED_SCANNET_PAIR_DIR} ${CODEBASE_DIR}/data/scannet +``` +2. Pre-training with the following example scripts: +```bash +# ScanNet +sh scripts/train.sh -g 8 -d scannet -c pretrain-msc-v1m1-1-spunet-pointcontrast -n pretrain-msc-v1m1-1-spunet-pointcontrast +``` +3. Fine-tuning refer [MSC](#masked-scene-contrast-msc). + +#### Contrastive Scene Contexts +1. Preprocess and link ScanNet-Pair dataset (refer [PointContrast](#pointcontrast)): +2. Pre-training with the following example scripts: +```bash +# ScanNet +sh scripts/train.sh -g 8 -d scannet -c pretrain-msc-v1m2-0-spunet-csc -n pretrain-msc-v1m2-0-spunet-csc +``` +3. Fine-tuning refer [MSC](#masked-scene-contrast-msc). + +## Acknowledgement +_Pointcept_ is designed by [Xiaoyang](https://xywu.me/), named by [Yixing](https://github.com/yxlao) and the logo is created by [Yuechen](https://julianjuaner.github.io/). It is derived from [Hengshuang](https://hszhao.github.io/)'s [Semseg](https://github.com/hszhao/semseg) and inspirited by several repos, e.g., [MinkowskiEngine](https://github.com/NVIDIA/MinkowskiEngine), [pointnet2](https://github.com/charlesq34/pointnet2), [mmcv](https://github.com/open-mmlab/mmcv/tree/master/mmcv), and [Detectron2](https://github.com/facebookresearch/detectron2). diff --git a/submodules/PointTransformerV3/Pointcept/configs/_base_/dataset/scannetpp.py b/submodules/PointTransformerV3/Pointcept/configs/_base_/dataset/scannetpp.py new file mode 100644 index 0000000000000000000000000000000000000000..926850c22981b88f2b56f26507a7a1693e00800b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/_base_/dataset/scannetpp.py @@ -0,0 +1,104 @@ +data = dict( + names=[ + "wall", + "ceiling", + "floor", + "table", + "door", + "ceiling lamp", + "cabinet", + "blinds", + "curtain", + "chair", + "storage cabinet", + "office chair", + "bookshelf", + "whiteboard", + "window", + "box", + "window frame", + "monitor", + "shelf", + "doorframe", + "pipe", + "heater", + "kitchen cabinet", + "sofa", + "windowsill", + "bed", + "shower wall", + "trash can", + "book", + "plant", + "blanket", + "tv", + "computer tower", + "kitchen counter", + "refrigerator", + "jacket", + "electrical duct", + "sink", + "bag", + "picture", + "pillow", + "towel", + "suitcase", + "backpack", + "crate", + "keyboard", + "rack", + "toilet", + "paper", + "printer", + "poster", + "painting", + "microwave", + "board", + "shoes", + "socket", + "bottle", + "bucket", + "cushion", + "basket", + "shoe rack", + "telephone", + "file folder", + "cloth", + "blind rail", + "laptop", + "plant pot", + "exhaust fan", + "cup", + "coat hanger", + "light switch", + "speaker", + "table lamp", + "air vent", + "clothes hanger", + "kettle", + "smoke detector", + "container", + "power strip", + "slippers", + "paper bag", + "mouse", + "cutting board", + "toilet paper", + "paper towel", + "pot", + "clock", + "pan", + "tap", + "jar", + "soap dispenser", + "binder", + "bowl", + "tissue box", + "whiteboard eraser", + "toilet brush", + "spray bottle", + "headphones", + "stapler", + "marker", + ] +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/_base_/default_runtime.py b/submodules/PointTransformerV3/Pointcept/configs/_base_/default_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..1ec8bf179f3c462dd80e58dcd70debcbd775f5d2 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/_base_/default_runtime.py @@ -0,0 +1,39 @@ +weight = None # path to model weight +resume = False # whether to resume training process +evaluate = True # evaluate after each epoch training process +test_only = False # test process + +seed = None # train process will init a random seed and record +save_path = "exp/default" +num_worker = 16 # total worker in all gpu +batch_size = 16 # total batch size in all gpu +batch_size_val = None # auto adapt to bs 1 for each gpu +batch_size_test = None # auto adapt to bs 1 for each gpu +epoch = 100 # total epoch, data loop = epoch // eval_epoch +eval_epoch = 100 # sche total eval & checkpoint epoch +clip_grad = None # disable with None, enable with a float + +sync_bn = False +enable_amp = False +empty_cache = False +empty_cache_per_epoch = False +find_unused_parameters = False + +mix_prob = 0 +param_dicts = None # example: param_dicts = [dict(keyword="block", lr_scale=0.1)] + +# hook +hooks = [ + dict(type="CheckpointLoader"), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict(type="SemSegEvaluator"), + dict(type="CheckpointSaver", save_freq=None), + dict(type="PreciseEvaluator", test_last=False), +] + +# Trainer +train = dict(type="DefaultTrainer") + +# Tester +test = dict(type="SemSegTester", verbose=True) diff --git a/submodules/PointTransformerV3/Pointcept/configs/matterport3d/semseg-pt-v3m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/matterport3d/semseg-pt-v3m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..1559d97a2696fb7c9a5f6e2ec75238445ed13eb2 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/matterport3d/semseg-pt-v3m1-0-base.py @@ -0,0 +1,313 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentorV2", + num_classes=21, + backbone_out_channels=64, + backbone=dict( + type="PT-v3m1", + in_channels=6, + order=("z", "z-trans", "hilbert", "hilbert-trans"), + stride=(2, 2, 2, 2), + enc_depths=(2, 2, 2, 6, 2), + enc_channels=(32, 64, 128, 256, 512), + enc_num_head=(2, 4, 8, 16, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(2, 2, 2, 2), + dec_channels=(64, 64, 128, 256), + dec_num_head=(4, 4, 8, 16), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=False, + pdnorm_ln=False, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "S3DIS", "Structured3D"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.006, 0.0006], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="block", lr=0.0006)] + +# dataset settings +dataset_type = "DefaultDataset" +data_root = "data/matterport3d" + +data = dict( + num_classes=21, + ignore_index=-1, + names=( + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refrigerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "other", + "ceiling", + ), + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=102400, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/matterport3d/semseg-spunet-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/matterport3d/semseg-spunet-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..ef0305cd78a7fb58c029b4b69f2cfb48cc0d6648 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/matterport3d/semseg-spunet-v1m1-0-base.py @@ -0,0 +1,282 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=21, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "DefaultDataset" +data_root = "data/matterport3d" + +data = dict( + num_classes=21, + ignore_index=-1, + names=( + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refrigerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "other", + "ceiling", + ), + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/modelnet40/cls-ptv3-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/modelnet40/cls-ptv3-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..235a5567e5382e297cb285af9d1ceb2c82a20e9b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/modelnet40/cls-ptv3-v1m1-0-base.py @@ -0,0 +1,232 @@ +_base_ = ["../_base_/default_runtime.py"] +# misc custom setting +batch_size = 32 # bs: total bs in all gpus +num_worker = 16 +batch_size_val = 8 +empty_cache = False +enable_amp = False + +# model settings +model = dict( + type="DefaultClassifier", + num_classes=40, + backbone_embed_dim=512, + backbone=dict( + type="PT-v3m1", + in_channels=6, + order=("z", "z-trans", "hilbert", "hilbert-trans"), + stride=(2, 2, 2, 2), + enc_depths=(2, 2, 2, 6, 2), + enc_channels=(32, 64, 128, 256, 512), + enc_num_head=(2, 4, 8, 16, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(2, 2, 2, 2), + dec_channels=(64, 64, 128, 256), + dec_num_head=(4, 4, 8, 16), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=True, + pdnorm_bn=False, + pdnorm_ln=False, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "S3DIS", "Structured3D"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 300 +# optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +# scheduler = dict(type="MultiStepLR", milestones=[0.6, 0.8], gamma=0.1) +optimizer = dict(type="AdamW", lr=0.001, weight_decay=0.01) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.001, 0.0001], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="block", lr=0.0001)] + +# dataset settings +dataset_type = "ModelNetDataset" +data_root = "data/modelnet40_normal_resampled" +cache_data = False +class_names = [ + "airplane", + "bathtub", + "bed", + "bench", + "bookshelf", + "bottle", + "bowl", + "car", + "chair", + "cone", + "cup", + "curtain", + "desk", + "door", + "dresser", + "flower_pot", + "glass_box", + "guitar", + "keyboard", + "lamp", + "laptop", + "mantel", + "monitor", + "night_stand", + "person", + "piano", + "plant", + "radio", + "range_hood", + "sink", + "sofa", + "stairs", + "stool", + "table", + "tent", + "toilet", + "tv_stand", + "vase", + "wardrobe", + "xbox", +] + +data = dict( + num_classes=40, + ignore_index=-1, + names=class_names, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + class_names=class_names, + transform=[ + dict(type="NormalizeCoord"), + # dict(type="CenterShift", apply_z=True), + # dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1/24, 1/24], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/24, 1/24], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.7, 1.5], anisotropic=True), + dict(type="RandomShift", shift=((-0.2, 0.2), (-0.2, 0.2), (-0.2, 0.2))), + # dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + keys=("coord", "normal"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=10000, mode="random"), + # dict(type="CenterShift", apply_z=True), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "category"), + feat_keys=["coord", "normal"], + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="test", + data_root=data_root, + class_names=class_names, + transform=[ + dict(type="NormalizeCoord"), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + keys=("coord", "normal"), + return_grid_coord=True, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "category"), + feat_keys=["coord", "normal"], + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="test", + data_root=data_root, + class_names=class_names, + transform=[ + dict(type="NormalizeCoord"), + ], + test_mode=True, + test_cfg=dict( + post_transform=[ + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + keys=("coord", "normal"), + return_grid_coord=True, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord"), + feat_keys=["coord", "normal"], + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[1, 1], anisotropic=True)], # 1 + [dict(type="RandomScale", scale=[0.8, 1.2], anisotropic=True)], # 2 + [dict(type="RandomScale", scale=[0.8, 1.2], anisotropic=True)], # 3 + [dict(type="RandomScale", scale=[0.8, 1.2], anisotropic=True)], # 4 + [dict(type="RandomScale", scale=[0.8, 1.2], anisotropic=True)], # 5 + [dict(type="RandomScale", scale=[0.8, 1.2], anisotropic=True)], # 5 + [dict(type="RandomScale", scale=[0.8, 1.2], anisotropic=True)], # 6 + [dict(type="RandomScale", scale=[0.8, 1.2], anisotropic=True)], # 7 + [dict(type="RandomScale", scale=[0.8, 1.2], anisotropic=True)], # 8 + [dict(type="RandomScale", scale=[0.8, 1.2], anisotropic=True)], # 9 + ], + ), + ), +) + +# hooks +hooks = [ + dict(type="CheckpointLoader"), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict(type="ClsEvaluator"), + dict(type="CheckpointSaver", save_freq=None), + dict(type="PreciseEvaluator", test_last=False), +] + +# tester +test = dict(type="ClsVotingTester", num_repeat=100) diff --git a/submodules/PointTransformerV3/Pointcept/configs/modelnet40/cls-spunet-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/modelnet40/cls-spunet-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..8e6585af547d3658093f5203468c2b3c12108f67 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/modelnet40/cls-spunet-v1m1-0-base.py @@ -0,0 +1,176 @@ +_base_ = ["../_base_/default_runtime.py"] +# misc custom setting +batch_size = 16 # bs: total bs in all gpus +# batch_size_val = 8 +empty_cache = False +enable_amp = False + +# model settings +model = dict( + type="DefaultClassifier", + num_classes=40, + backbone_embed_dim=256, + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=0, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=True, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 200 +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict(type="MultiStepLR", milestones=[0.6, 0.8], gamma=0.1) + +# dataset settings +dataset_type = "ModelNetDataset" +data_root = "data/modelnet40_normal_resampled" +cache_data = False +class_names = [ + "airplane", + "bathtub", + "bed", + "bench", + "bookshelf", + "bottle", + "bowl", + "car", + "chair", + "cone", + "cup", + "curtain", + "desk", + "door", + "dresser", + "flower_pot", + "glass_box", + "guitar", + "keyboard", + "lamp", + "laptop", + "mantel", + "monitor", + "night_stand", + "person", + "piano", + "plant", + "radio", + "range_hood", + "sink", + "sofa", + "stairs", + "stool", + "table", + "tent", + "toilet", + "tv_stand", + "vase", + "wardrobe", + "xbox", +] + +data = dict( + num_classes=40, + ignore_index=-1, + names=class_names, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + class_names=class_names, + transform=[ + dict(type="NormalizeCoord"), + # dict(type="CenterShift", apply_z=True), + # dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1/24, 1/24], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/24, 1/24], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + dict(type="RandomShift", shift=((-0.2, 0.2), (-0.2, 0.2), (-0.2, 0.2))), + # dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + keys=("coord", "normal"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=10000, mode="random"), + # dict(type="CenterShift", apply_z=True), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "category"), + feat_keys=["coord", "normal"], + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="test", + data_root=data_root, + class_names=class_names, + transform=[ + dict(type="NormalizeCoord"), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + keys=("coord", "normal"), + return_grid_coord=True, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "category"), + feat_keys=["coord", "normal"], + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="test", + data_root=data_root, + class_names=class_names, + transform=[ + dict(type="NormalizeCoord"), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + keys=("coord", "normal"), + return_grid_coord=True, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "category"), + feat_keys=["coord", "normal"], + ), + ], + test_mode=True, + ), +) + +# hooks +hooks = [ + dict(type="CheckpointLoader"), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict(type="ClsEvaluator"), + dict(type="CheckpointSaver", save_freq=None), +] + +# tester +test = dict(type="ClsTester") diff --git a/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-ppt-v1m1-0-nu-sk-wa-spunet.py b/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-ppt-v1m1-0-nu-sk-wa-spunet.py new file mode 100644 index 0000000000000000000000000000000000000000..ed82be25301d2ac9650147cc68ebd7a2aa9534be --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-ppt-v1m1-0-nu-sk-wa-spunet.py @@ -0,0 +1,342 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +find_unused_parameters = True + +# trainer +train = dict( + type="MultiDatasetTrainer", +) + +# model settings +model = dict( + type="PPT-v1m1", + backbone=dict( + type="SpUNet-v1m3", + in_channels=4, + num_classes=0, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("SemanticKITTI", "nuScenes", "Waymo"), + zero_init=False, + norm_decouple=True, + norm_adaptive=False, + norm_affine=True, + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + backbone_out_channels=96, + context_channels=256, + conditions=("SemanticKITTI", "nuScenes", "Waymo"), + template="[x]", + clip_model="ViT-B/16", + # fmt: off + class_name=( + # SemanticKITTI + "car", "bicycle", "motorcycle", "truck", "other vehicle", + "person", "person who rides a bicycle", "person who rides a motorcycle", "road", "parking", + "path for pedestrians at the side of a road", "other ground", "building", "fence", "vegetation", + "trunk", "terrain", "pole", "traffic sign", + # nuScenes + "barrier", "bicycle", "bus", "car", "construction vehicle", + "motorcycle", "pedestrian", "traffic cone", "trailer", "truck", + "path suitable or safe for driving", "other flat", "sidewalk", "terrain", "man made", "vegetation", + # waymo + "car", "truck", "bus", "other vehicle", "person who rides a motorcycle", + "person who rides a bicycle", "pedestrian", "sign", "traffic light", "pole", + "construction cone", "bicycle", "motorcycle", "building", "vegetation", + "tree trunk", "curb", "road", "lane marker", "other ground", "horizontal surface that can not drive", + "surface when pedestrians most likely to walk on", + ), + valid_index=( + [i for i in range(19)], + [i for i in range(19, 19 + 16)], + [i for i in range(19 + 16, 19 + 16 + 22)], + ), + # fmt: on + backbone_mode=False, +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) +# param_dicts = [dict(keyword="modulation", lr=0.0002)] + +# dataset settings +data = dict( + num_classes=16, + ignore_index=-1, + names=[ + "barrier", + "bicycle", + "bus", + "car", + "construction_vehicle", + "motorcycle", + "pedestrian", + "traffic_cone", + "trailer", + "truck", + "driveable_surface", + "other_flat", + "sidewalk", + "terrain", + "manmade", + "vegetation", + ], + train=dict( + type="ConcatDataset", + datasets=[ + # nuScenes + dict( + type="NuScenesDataset", + split="train", + data_root="data/nuscenes", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='x', p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='y', p=0.5), + dict( + type="PointClip", + point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "nuScenes"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + # SemanticKITTI + dict( + type="SemanticKITTIDataset", + split="train", + data_root="data/semantic_kitti", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict( + type="PointClip", + point_cloud_range=(-75.2, -75.2, -4, 75.2, 75.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "SemanticKITTI"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + # Waymo + dict( + type="WaymoDataset", + split="training", + data_root="data/waymo", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict( + type="PointClip", + point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "Waymo"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + ], + ), + val=dict( + type="NuScenesDataset", + split="val", + data_root="data/nuscenes", + transform=[ + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + dict(type="Add", keys_dict={"condition": "nuScenes"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + ), + test=dict( + type="NuScenesDataset", + split="val", + data_root="data/nuscenes", + transform=[ + dict(type="Copy", keys_dict={"segment": "origin_segment"}), + dict( + type="GridSample", + grid_size=0.025, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_inverse=True, + ), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict(type="Add", keys_dict={"condition": "nuScenes"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ignore_index=-1, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-ppt-v1m2-0-nu-sk-wa-spunet.py b/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-ppt-v1m2-0-nu-sk-wa-spunet.py new file mode 100644 index 0000000000000000000000000000000000000000..bf0aba527a41fc745f24dbd5913a14a7698834f5 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-ppt-v1m2-0-nu-sk-wa-spunet.py @@ -0,0 +1,316 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +find_unused_parameters = True + +# trainer +train = dict( + type="MultiDatasetTrainer", +) + +# model settings +model = dict( + type="PPT-v1m2", + backbone=dict( + type="SpUNet-v1m3", + in_channels=4, + num_classes=0, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("SemanticKITTI", "nuScenes", "Waymo"), + zero_init=False, + norm_decouple=True, + norm_adaptive=False, + norm_affine=True, + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + backbone_out_channels=96, + context_channels=256, + conditions=("SemanticKITTI", "nuScenes", "Waymo"), + num_classes=(19, 16, 22), +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) +# param_dicts = [dict(keyword="modulation", lr=0.0002)] + +# dataset settings +data = dict( + num_classes=16, + ignore_index=-1, + names=[ + "barrier", + "bicycle", + "bus", + "car", + "construction_vehicle", + "motorcycle", + "pedestrian", + "traffic_cone", + "trailer", + "truck", + "driveable_surface", + "other_flat", + "sidewalk", + "terrain", + "manmade", + "vegetation", + ], + train=dict( + type="ConcatDataset", + datasets=[ + # nuScenes + dict( + type="NuScenesDataset", + split="train", + data_root="data/nuscenes", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='x', p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='y', p=0.5), + dict( + type="PointClip", + point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "nuScenes"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + # SemanticKITTI + dict( + type="SemanticKITTIDataset", + split="train", + data_root="data/semantic_kitti", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict( + type="PointClip", + point_cloud_range=(-75.2, -75.2, -4, 75.2, 75.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "SemanticKITTI"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + # Waymo + dict( + type="WaymoDataset", + split="training", + data_root="data/waymo", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict( + type="PointClip", + point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "Waymo"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + ], + ), + val=dict( + type="NuScenesDataset", + split="val", + data_root="data/nuscenes", + transform=[ + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + dict(type="Add", keys_dict={"condition": "nuScenes"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + ), + test=dict( + type="NuScenesDataset", + split="val", + data_root="data/nuscenes", + transform=[ + dict(type="Copy", keys_dict={"segment": "origin_segment"}), + dict( + type="GridSample", + grid_size=0.025, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_inverse=True, + ), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict(type="Add", keys_dict={"condition": "nuScenes"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ignore_index=-1, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-ppt-v1m2-1-nu-sk-wa-spunet-submit.py b/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-ppt-v1m2-1-nu-sk-wa-spunet-submit.py new file mode 100644 index 0000000000000000000000000000000000000000..d8f254757995c51401643a7db1e9c48455b4fefb --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-ppt-v1m2-1-nu-sk-wa-spunet-submit.py @@ -0,0 +1,292 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +find_unused_parameters = True +evaluate = False + +# trainer +train = dict( + type="MultiDatasetTrainer", +) + +# model settings +model = dict( + type="PPT-v1m2", + backbone=dict( + type="SpUNet-v1m3", + in_channels=4, + num_classes=0, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("SemanticKITTI", "nuScenes", "Waymo"), + zero_init=False, + norm_decouple=True, + norm_adaptive=False, + norm_affine=True, + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + backbone_out_channels=96, + context_channels=256, + conditions=("SemanticKITTI", "nuScenes", "Waymo"), + num_classes=(19, 16, 22), +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) +# param_dicts = [dict(keyword="modulation", lr=0.0002)] + +# dataset settings +data = dict( + num_classes=16, + ignore_index=-1, + names=[ + "barrier", + "bicycle", + "bus", + "car", + "construction_vehicle", + "motorcycle", + "pedestrian", + "traffic_cone", + "trailer", + "truck", + "driveable_surface", + "other_flat", + "sidewalk", + "terrain", + "manmade", + "vegetation", + ], + train=dict( + type="ConcatDataset", + datasets=[ + # nuScenes + dict( + type="NuScenesDataset", + split=["train", "val"], + data_root="data/nuscenes", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='x', p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='y', p=0.5), + dict( + type="PointClip", + point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "nuScenes"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + # SemanticKITTI + dict( + type="SemanticKITTIDataset", + split=["train", "val"], + data_root="data/semantic_kitti", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict( + type="PointClip", + point_cloud_range=(-75.2, -75.2, -4, 75.2, 75.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "SemanticKITTI"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + # Waymo + dict( + type="WaymoDataset", + split=["training", "validation"], + data_root="data/waymo", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict( + type="PointClip", + point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "Waymo"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + ], + ), + test=dict( + type="NuScenesDataset", + split="test", + data_root="data/nuscenes", + transform=[ + dict(type="Copy", keys_dict={"segment": "origin_segment"}), + dict( + type="GridSample", + grid_size=0.025, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_inverse=True, + ), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict(type="Add", keys_dict={"condition": "nuScenes"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ignore_index=-1, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-pt-v2m2-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-pt-v2m2-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..0ce53d7d872e69bd2e66614124f6d4e19a6fdc02 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-pt-v2m2-0-base.py @@ -0,0 +1,174 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=4, + num_classes=16, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.15, 0.375, 0.9375, 2.34375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) + +# dataset settings +dataset_type = "NuScenesDataset" +data_root = "data/nuscenes" +ignore_index = -1 +names = [ + "barrier", + "bicycle", + "bus", + "car", + "construction_vehicle", + "motorcycle", + "pedestrian", + "traffic_cone", + "trailer", + "truck", + "driveable_surface", + "other_flat", + "sidewalk", + "terrain", + "manmade", + "vegetation", +] + +data = dict( + num_classes=16, + ignore_index=ignore_index, + names=names, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='x', p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='y', p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + # dict(type="GridSample", grid_size=0.05, hash_type="fnv", mode="train", + # keys=("coord", "strength", "segment"), return_grid_coord=True), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + # dict(type="PointClip", point_cloud_range=(-51.2, -51.2, -4, 51.2, 51.2, 2.4)), + # dict(type="GridSample", grid_size=0.05, hash_type="fnv", mode="train", + # keys=("coord", "strength", "segment"), return_grid_coord=True), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[], + test_mode=True, + test_cfg=dict( + voxelize=None, + crop=None, + post_transform=[ + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ignore_index=ignore_index, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-pt-v2m2-1-benchmark-submit.py b/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-pt-v2m2-1-benchmark-submit.py new file mode 100644 index 0000000000000000000000000000000000000000..19f7e7512d4f809704be97ee64653c1d852aafff --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-pt-v2m2-1-benchmark-submit.py @@ -0,0 +1,157 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True +evaluate = False + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=4, + num_classes=16, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.15, 0.375, 0.9375, 2.34375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) + +# dataset settings +dataset_type = "NuScenesDataset" +data_root = "data/nuscenes" +ignore_index = -1 +names = [ + "barrier", + "bicycle", + "bus", + "car", + "construction_vehicle", + "motorcycle", + "pedestrian", + "traffic_cone", + "trailer", + "truck", + "driveable_surface", + "other_flat", + "sidewalk", + "terrain", + "manmade", + "vegetation", +] + +data = dict( + num_classes=16, + ignore_index=ignore_index, + names=names, + train=dict( + type=dataset_type, + split=["train", "val"], + data_root=data_root, + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='x', p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='y', p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + # dict(type="GridSample", grid_size=0.05, hash_type="fnv", mode="train", + # keys=("coord", "strength", "segment"), return_grid_coord=True), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + test=dict( + type=dataset_type, + split="test", + data_root=data_root, + transform=[], + test_mode=True, + test_cfg=dict( + voxelize=None, + crop=None, + post_transform=[ + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ignore_index=ignore_index, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-pt-v3m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-pt-v3m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..4f64b9e67dedcf0cd1a7f950d2d6677dce0aa088 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-pt-v3m1-0-base.py @@ -0,0 +1,215 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentorV2", + num_classes=16, + backbone_out_channels=64, + backbone=dict( + type="PT-v3m1", + in_channels=4, + order=["z", "z-trans", "hilbert", "hilbert-trans"], + stride=(2, 2, 2, 2), + enc_depths=(2, 2, 2, 6, 2), + enc_channels=(32, 64, 128, 256, 512), + enc_num_head=(2, 4, 8, 16, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(2, 2, 2, 2), + dec_channels=(64, 64, 128, 256), + dec_num_head=(4, 4, 8, 16), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=False, + pdnorm_ln=False, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("nuScenes", "SemanticKITTI", "Waymo"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.002, 0.0002], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) +param_dicts = [dict(keyword="block", lr=0.0002)] + +# dataset settings +dataset_type = "NuScenesDataset" +data_root = "data/nuscenes" +ignore_index = -1 +names = [ + "barrier", + "bicycle", + "bus", + "car", + "construction_vehicle", + "motorcycle", + "pedestrian", + "traffic_cone", + "trailer", + "truck", + "driveable_surface", + "other_flat", + "sidewalk", + "terrain", + "manmade", + "vegetation", +] + +data = dict( + num_classes=16, + ignore_index=ignore_index, + names=names, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + # dict(type="PointClip", point_cloud_range=(-51.2, -51.2, -4, 51.2, 51.2, 2.4)), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode='center'), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="Copy", keys_dict={"segment": "origin_segment"}), + dict( + type="GridSample", + grid_size=0.025, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_inverse=True, + ), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ignore_index=ignore_index, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-spunet-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-spunet-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..d6b6a126086a210335b27953a2e620bc74e56503 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/nuscenes/semseg-spunet-v1m1-0-base.py @@ -0,0 +1,183 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=4, + num_classes=16, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) + +# dataset settings +dataset_type = "NuScenesDataset" +data_root = "data/nuscenes" +ignore_index = -1 +names = [ + "barrier", + "bicycle", + "bus", + "car", + "construction_vehicle", + "motorcycle", + "pedestrian", + "traffic_cone", + "trailer", + "truck", + "driveable_surface", + "other_flat", + "sidewalk", + "terrain", + "manmade", + "vegetation", +] + +data = dict( + num_classes=16, + ignore_index=ignore_index, + names=names, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='x', p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='y', p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + # dict(type="PointClip", point_cloud_range=(-51.2, -51.2, -4, 51.2, 51.2, 2.4)), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode='center'), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="Copy", keys_dict={"segment": "origin_segment"}), + dict( + type="GridSample", + grid_size=0.025, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_inverse=True, + ), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ignore_index=ignore_index, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-pointgroup-v1m1-0-spunet-base-vs0p02-sc-aug.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-pointgroup-v1m1-0-spunet-base-vs0p02-sc-aug.py new file mode 100644 index 0000000000000000000000000000000000000000..2cb44ce8269da20036a4b2b7e61109bb97529709 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-pointgroup-v1m1-0-spunet-base-vs0p02-sc-aug.py @@ -0,0 +1,180 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 12 +mix_prob = 0.0 +empty_cache = False +enable_amp = True +evaluate = True + +class_names = [ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", +] +num_classes = 13 +segment_ignore_index = (-1,) + +# model settings +model = dict( + type="PG-v1m1", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=0, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + backbone_out_channels=96, + semantic_num_classes=num_classes, + semantic_ignore_index=-1, + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + cluster_thresh=1.5, + cluster_closed_points=300, + cluster_propose_points=100, + cluster_min_points=50, +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict(type="PolyLR") + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=num_classes, + ignore_index=-1, + names=class_names, + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.5 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.1), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "instance_centroid", + "bbox", + ), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={ + "coord": "origin_coord", + "segment": "origin_segment", + "instance": "origin_instance", + }, + ), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + # dict(type="SphereCrop", point_max=1000000, mode='center'), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "origin_coord", + "origin_segment", + "origin_instance", + "instance_centroid", + "bbox", + ), + feat_keys=("color", "normal"), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + ), + ], + test_mode=False, + ), + test=dict(), # currently not available +) + +hooks = [ + dict(type="CheckpointLoader", keywords="module.", replacement="module."), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict( + type="InsSegEvaluator", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="CheckpointSaver", save_freq=None), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-pointgroup-v1m1-0-spunet-base-vs0p02.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-pointgroup-v1m1-0-spunet-base-vs0p02.py new file mode 100644 index 0000000000000000000000000000000000000000..826d3731ac6662e03a956daa1f213f03b5c1984a --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-pointgroup-v1m1-0-spunet-base-vs0p02.py @@ -0,0 +1,180 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 12 +mix_prob = 0.0 +empty_cache = False +enable_amp = True +evaluate = True + +class_names = [ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", +] +num_classes = 13 +segment_ignore_index = (-1,) + +# model settings +model = dict( + type="PG-v1m1", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=0, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + backbone_out_channels=96, + semantic_num_classes=num_classes, + semantic_ignore_index=-1, + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + cluster_thresh=1.5, + cluster_closed_points=300, + cluster_propose_points=100, + cluster_min_points=50, +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict(type="PolyLR") + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=num_classes, + ignore_index=-1, + names=class_names, + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.5 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.005), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "instance_centroid", + "bbox", + ), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={ + "coord": "origin_coord", + "segment": "origin_segment", + "instance": "origin_instance", + }, + ), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + # dict(type="SphereCrop", point_max=1000000, mode='center'), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "origin_coord", + "origin_segment", + "origin_instance", + "instance_centroid", + "bbox", + ), + feat_keys=("color", "normal"), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + ), + ], + test_mode=False, + ), + test=dict(), # currently not available +) + +hooks = [ + dict(type="CheckpointLoader", keywords="module.", replacement="module."), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict( + type="InsSegEvaluator", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="CheckpointSaver", save_freq=None), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-pointgroup-v1m1-0-spunet-base.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-pointgroup-v1m1-0-spunet-base.py new file mode 100644 index 0000000000000000000000000000000000000000..3ce06b51d1a566c19305e486a614b73d4594bc58 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-pointgroup-v1m1-0-spunet-base.py @@ -0,0 +1,181 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 12 +mix_prob = 0.0 +empty_cache = False +enable_amp = True +evaluate = True + +class_names = [ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", +] +num_classes = 13 +segment_ignore_index = (-1,) + +# model settings +model = dict( + type="PG-v1m1", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=0, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + backbone_out_channels=96, + semantic_num_classes=num_classes, + semantic_ignore_index=-1, + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + cluster_thresh=1.5, + cluster_closed_points=300, + cluster_propose_points=100, + cluster_min_points=50, + voxel_size=0.05, +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict(type="PolyLR") + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=num_classes, + ignore_index=-1, + names=class_names, + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.5 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.005), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "instance_centroid", + "bbox", + ), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={ + "coord": "origin_coord", + "segment": "origin_segment", + "instance": "origin_instance", + }, + ), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + # dict(type="SphereCrop", point_max=1000000, mode='center'), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "origin_coord", + "origin_segment", + "origin_instance", + "instance_centroid", + "bbox", + ), + feat_keys=("color", "normal"), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + ), + ], + test_mode=False, + ), + test=dict(), # currently not available +) + +hooks = [ + dict(type="CheckpointLoader", keywords="module.", replacement="module."), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict( + type="InsSegEvaluator", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="CheckpointSaver", save_freq=None), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-ppt-v1m1-0-pointgroup-spunet-ft-vs0p05.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-ppt-v1m1-0-pointgroup-spunet-ft-vs0p05.py new file mode 100644 index 0000000000000000000000000000000000000000..b1f5d0dbf5c4363eabb7f017422d005dc6ff1b57 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-ppt-v1m1-0-pointgroup-spunet-ft-vs0p05.py @@ -0,0 +1,273 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0 +empty_cache = False +enable_amp = True +evaluate = True +find_unused_parameters = True + +class_names = [ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", +] +num_classes = 13 +segment_ignore_index = (-1,) + +# model settings +model = dict( + type="PG-v1m1", + backbone=dict( + type="PPT-v1m1", + backbone=dict( + type="SpUNet-v1m3", + in_channels=6, + num_classes=0, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("ScanNet", "S3DIS", "Structured3D"), + zero_init=False, + norm_decouple=True, + norm_adaptive=True, + norm_affine=True, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], + backbone_out_channels=96, + context_channels=256, + conditions=("Structured3D", "ScanNet", "S3DIS"), + template="[x]", + clip_model="ViT-B/16", + class_name=( + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "bookcase", + "picture", + "counter", + "desk", + "shelves", + "curtain", + "dresser", + "pillow", + "mirror", + "ceiling", + "refrigerator", + "television", + "shower curtain", + "nightstand", + "toilet", + "sink", + "lamp", + "bathtub", + "garbagebin", + "board", + "beam", + "column", + "clutter", + "otherstructure", + "otherfurniture", + "otherprop", + ), + valid_index=( + ( + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 11, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 23, + 25, + 26, + 33, + 34, + 35, + ), + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 20, 22, 24, 25, 27, 34), + (0, 1, 4, 5, 6, 7, 8, 10, 19, 29, 30, 31, 32), + ), + backbone_mode=True, + ), + backbone_out_channels=96, + semantic_num_classes=num_classes, + semantic_ignore_index=-1, + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + cluster_thresh=1.5, + cluster_closed_points=300, + cluster_propose_points=100, + cluster_min_points=50, + voxel_size=0.05, +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict(type="PolyLR") + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=num_classes, + ignore_index=-1, + names=class_names, + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.5 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.005), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="Add", keys_dict={"condition": "S3DIS"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "instance_centroid", + "bbox", + "condition", + ), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={ + "coord": "origin_coord", + "segment": "origin_segment", + "instance": "origin_instance", + }, + ), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + # dict(type="SphereCrop", point_max=1000000, mode='center'), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="Add", keys_dict={"condition": "S3DIS"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "origin_coord", + "origin_segment", + "origin_instance", + "instance_centroid", + "bbox", + "condition", + ), + feat_keys=("color", "normal"), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + ), + ], + test_mode=False, + ), + test=dict(), # currently not available +) + +hooks = [ + dict(type="CheckpointLoader", keywords="module.", replacement="module.backbone."), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict( + type="InsSegEvaluator", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="CheckpointSaver", save_freq=None), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-ppt-v1m1-0-pointgroup-spunet-ft.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-ppt-v1m1-0-pointgroup-spunet-ft.py new file mode 100644 index 0000000000000000000000000000000000000000..ca4aa554cedd24771d12b645241773757a0ef253 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/insseg-ppt-v1m1-0-pointgroup-spunet-ft.py @@ -0,0 +1,273 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0 +empty_cache = False +enable_amp = True +evaluate = True +find_unused_parameters = True + +class_names = [ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", +] +num_classes = 13 +segment_ignore_index = (-1,) + +# model settings +model = dict( + type="PG-v1m1", + backbone=dict( + type="PPT-v1m1", + backbone=dict( + type="SpUNet-v1m3", + in_channels=6, + num_classes=0, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("ScanNet", "S3DIS", "Structured3D"), + zero_init=False, + norm_decouple=True, + norm_adaptive=True, + norm_affine=True, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], + backbone_out_channels=96, + context_channels=256, + conditions=("Structured3D", "ScanNet", "S3DIS"), + template="[x]", + clip_model="ViT-B/16", + class_name=( + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "bookcase", + "picture", + "counter", + "desk", + "shelves", + "curtain", + "dresser", + "pillow", + "mirror", + "ceiling", + "refrigerator", + "television", + "shower curtain", + "nightstand", + "toilet", + "sink", + "lamp", + "bathtub", + "garbagebin", + "board", + "beam", + "column", + "clutter", + "otherstructure", + "otherfurniture", + "otherprop", + ), + valid_index=( + ( + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 11, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 23, + 25, + 26, + 33, + 34, + 35, + ), + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 20, 22, 24, 25, 27, 34), + (0, 1, 4, 5, 6, 7, 8, 10, 19, 29, 30, 31, 32), + ), + backbone_mode=True, + ), + backbone_out_channels=96, + semantic_num_classes=num_classes, + semantic_ignore_index=-1, + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + cluster_thresh=1.5, + cluster_closed_points=300, + cluster_propose_points=100, + cluster_min_points=50, + voxel_size=0.02, +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict(type="PolyLR") + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=num_classes, + ignore_index=-1, + names=class_names, + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.5 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.005), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="Add", keys_dict={"condition": "S3DIS"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "instance_centroid", + "bbox", + "condition", + ), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={ + "coord": "origin_coord", + "segment": "origin_segment", + "instance": "origin_instance", + }, + ), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + # dict(type="SphereCrop", point_max=1000000, mode='center'), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="Add", keys_dict={"condition": "S3DIS"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "origin_coord", + "origin_segment", + "origin_instance", + "instance_centroid", + "bbox", + "condition", + ), + feat_keys=("color", "normal"), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + ), + ], + test_mode=False, + ), + test=dict(), # currently not available +) + +hooks = [ + dict(type="CheckpointLoader", keywords="module.", replacement="module.backbone."), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict( + type="InsSegEvaluator", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="CheckpointSaver", save_freq=None), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-minkunet34c-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-minkunet34c-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..8234bb4b52de86edb05b540ee7250fdf53a7d02e --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-minkunet34c-0-base.py @@ -0,0 +1,174 @@ +_base_ = ["../_base_/default_runtime.py"] +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict(type="MinkUNet34C", in_channels=6, out_channels=13), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict(type="PolyLR") + + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=["coord", "color"], + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={"coord": "origin_coord", "segment": "origin_segment"}, + ), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "origin_coord", + "segment", + "origin_segment", + ), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + feat_keys=["coord", "color"], + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[dict(type="CenterShift", apply_z=True), dict(type="NormalizeColor")], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + keys=("coord", "color"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-ppt-v1m1-0-s3-sc-st-spunet.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-ppt-v1m1-0-s3-sc-st-spunet.py new file mode 100644 index 0000000000000000000000000000000000000000..e50ebbf10a1989342f39a476c1c3348671a78e95 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-ppt-v1m1-0-s3-sc-st-spunet.py @@ -0,0 +1,496 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 24 # bs: total bs in all gpus +num_worker = 48 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +find_unused_parameters = True + +# trainer +train = dict( + type="MultiDatasetTrainer", +) + +# model settings +model = dict( + type="PPT-v1m1", + backbone=dict( + type="SpUNet-v1m3", + in_channels=6, + num_classes=0, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("ScanNet", "S3DIS", "Structured3D"), + zero_init=False, + norm_decouple=True, + norm_adaptive=True, + norm_affine=True, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], + backbone_out_channels=96, + context_channels=256, + conditions=("Structured3D", "ScanNet", "S3DIS"), + template="[x]", + clip_model="ViT-B/16", + class_name=( + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "bookcase", + "picture", + "counter", + "desk", + "shelves", + "curtain", + "dresser", + "pillow", + "mirror", + "ceiling", + "refrigerator", + "television", + "shower curtain", + "nightstand", + "toilet", + "sink", + "lamp", + "bathtub", + "garbagebin", + "board", + "beam", + "column", + "clutter", + "otherstructure", + "otherfurniture", + "otherprop", + ), + valid_index=( + ( + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 11, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 23, + 25, + 26, + 33, + 34, + 35, + ), + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 20, 22, 24, 25, 27, 34), + (0, 1, 4, 5, 6, 7, 8, 10, 19, 29, 30, 31, 32), + ), + backbone_mode=False, +) + +# scheduler settings +epoch = 100 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) +# param_dicts = [dict(keyword="modulation", lr=0.005)] + +# dataset settings +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type="ConcatDataset", + datasets=[ + # Structured3D + dict( + type="Structured3DDataset", + split="train", + data_root="data/structured3d", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "Structured3D"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=4, # sampling weight + ), + # ScanNet + dict( + type="ScanNetDataset", + split="train", + data_root="data/scannet", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=2, # sampling weight + ), + # S3DIS + dict( + type="S3DISDataset", + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root="data/s3dis", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.6, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "S3DIS"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=1, # sampling weight + ), + ], + ), + val=dict( + type="S3DISDataset", + split="Area_5", + data_root="data/s3dis", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict(type="Add", keys_dict={"condition": "S3DIS"}), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type="S3DISDataset", + split="Area_5", + data_root="data/s3dis", + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "S3DIS"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..a925757abee6d3000ee4c3613e5b3bc49dbe7dc6 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v1-0-base.py @@ -0,0 +1,170 @@ +_base_ = ["../_base_/default_runtime.py"] +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PointTransformer-Seg50", + in_channels=6, + num_classes=13, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + + +# scheduler settings +epoch = 3000 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict(type="MultiStepLR", milestones=[0.6, 0.8], gamma=0.1) + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + # dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=["coord", "color"], + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={"coord": "origin_coord", "segment": "origin_segment"}, + ), + dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + offset_keys_dict=dict(offset="coord"), + feat_keys=["coord", "color"], + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[dict(type="CenterShift", apply_z=True), dict(type="NormalizeColor")], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="test", + keys=("coord", "color"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v2m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v2m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..3aca26a731096c1a1af9a34085c11184e6061d63 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v2m1-0-base.py @@ -0,0 +1,189 @@ +_base_ = ["../_base_/default_runtime.py"] +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = False + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m1", + in_channels=6, + num_classes=13, + patch_embed_depth=2, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=16, + enc_depths=(2, 6, 2), + enc_channels=(96, 192, 384), + enc_groups=(12, 24, 48), + enc_neighbours=(16, 16, 16), + dec_depths=(1, 1, 1), + dec_channels=(48, 96, 192), + dec_groups=(6, 12, 24), + dec_neighbours=(16, 16, 16), + grid_sizes=(0.1, 0.2, 0.4), + attn_qkv_bias=True, + pe_multiplier=True, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="interp", # map / interp + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict(type="MultiStepLR", milestones=[0.6, 0.8], gamma=0.1) + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + # dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=80000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=["coord", "color"], + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={"coord": "origin_coord", "segment": "origin_segment"}, + ), + dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + offset_keys_dict=dict(offset="coord"), + feat_keys=["coord", "color"], + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[dict(type="CenterShift", apply_z=True), dict(type="NormalizeColor")], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="test", + keys=("coord", "color"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v2m2-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v2m2-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..a99cb34fea520e3cb28fbb2e9bfcfd59dea1cdda --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v2m2-0-base.py @@ -0,0 +1,189 @@ +_base_ = ["../_base_/default_runtime.py"] +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=6, + num_classes=13, + patch_embed_depth=2, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=16, + enc_depths=(2, 6, 2), + enc_channels=(96, 192, 384), + enc_groups=(12, 24, 48), + enc_neighbours=(16, 16, 16), + dec_depths=(1, 1, 1), + dec_channels=(48, 96, 192), + dec_groups=(6, 12, 24), + dec_neighbours=(16, 16, 16), + grid_sizes=(0.1, 0.2, 0.4), + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="interp", # map / interp + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict(type="MultiStepLR", milestones=[0.6, 0.8], gamma=0.1) + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + # dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=80000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=["coord", "color"], + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={"coord": "origin_coord", "segment": "origin_segment"}, + ), + dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + offset_keys_dict=dict(offset="coord"), + feat_keys=["coord", "color"], + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[dict(type="CenterShift", apply_z=True), dict(type="NormalizeColor")], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="test", + keys=("coord", "color"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v2m2-0-lovasz.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v2m2-0-lovasz.py new file mode 100644 index 0000000000000000000000000000000000000000..0a8b71a9ae8c676bd2d4f86dfbfa8adc79cf06d3 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v2m2-0-lovasz.py @@ -0,0 +1,192 @@ +_base_ = ["../_base_/default_runtime.py"] +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=6, + num_classes=13, + patch_embed_depth=2, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=16, + enc_depths=(2, 6, 2), + enc_channels=(96, 192, 384), + enc_groups=(12, 24, 48), + enc_neighbours=(16, 16, 16), + dec_depths=(1, 1, 1), + dec_channels=(48, 96, 192), + dec_groups=(6, 12, 24), + dec_neighbours=(16, 16, 16), + grid_sizes=(0.1, 0.2, 0.4), + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="interp", # map / interp + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict(type="MultiStepLR", milestones=[0.6, 0.8], gamma=0.1) + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + # dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=80000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=["coord", "color"], + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={"coord": "origin_coord", "segment": "origin_segment"}, + ), + dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + offset_keys_dict=dict(offset="coord"), + feat_keys=["coord", "color"], + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[dict(type="CenterShift", apply_z=True), dict(type="NormalizeColor")], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="test", + keys=("coord", "color"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v2m2-1-one-cycle.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v2m2-1-one-cycle.py new file mode 100644 index 0000000000000000000000000000000000000000..e7d2493b92fcc21a00f2fecb270fb289dea58c0d --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v2m2-1-one-cycle.py @@ -0,0 +1,196 @@ +_base_ = ["../_base_/default_runtime.py"] +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=6, + num_classes=13, + patch_embed_depth=2, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=16, + enc_depths=(2, 6, 2), + enc_channels=(96, 192, 384), + enc_groups=(12, 24, 48), + enc_neighbours=(16, 16, 16), + dec_depths=(1, 1, 1), + dec_channels=(48, 96, 192), + dec_groups=(6, 12, 24), + dec_neighbours=(16, 16, 16), + grid_sizes=(0.1, 0.2, 0.4), + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="interp", # map / interp + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + # dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=80000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=["coord", "color"], + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={"coord": "origin_coord", "segment": "origin_segment"}, + ), + dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + offset_keys_dict=dict(offset="coord"), + feat_keys=["coord", "color"], + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[dict(type="CenterShift", apply_z=True), dict(type="NormalizeColor")], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="test", + keys=("coord", "color"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v3m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v3m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..89d34ba0ff00d52373f9cf791afb554e99ef7a57 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v3m1-0-base.py @@ -0,0 +1,225 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentorV2", + num_classes=13, + backbone_out_channels=64, + backbone=dict( + type="PT-v3m1", + in_channels=6, + order=("z", "z-trans", "hilbert", "hilbert-trans"), + stride=(2, 2, 2, 2), + enc_depths=(2, 2, 2, 6, 2), + enc_channels=(32, 64, 128, 256, 512), + enc_num_head=(2, 4, 8, 16, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(2, 2, 2, 2), + dec_channels=(64, 64, 128, 256), + dec_num_head=(4, 4, 8, 16), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=False, + pdnorm_ln=False, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "S3DIS", "Structured3D"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.006, 0.0006], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="block", lr=0.0006)] + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.6, mode="random"), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={"coord": "origin_coord", "segment": "origin_segment"}, + ), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "origin_coord", + "segment", + "origin_segment", + ), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1, 1]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v3m1-1-rpe.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v3m1-1-rpe.py new file mode 100644 index 0000000000000000000000000000000000000000..ab612fc5449920103df6ea037588c87833ba73d5 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v3m1-1-rpe.py @@ -0,0 +1,225 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentorV2", + num_classes=13, + backbone_out_channels=64, + backbone=dict( + type="PT-v3m1", + in_channels=6, + order=["z", "z-trans", "hilbert", "hilbert-trans"], + stride=(2, 2, 2, 2), + enc_depths=(2, 2, 2, 6, 2), + enc_channels=(32, 64, 128, 256, 512), + enc_num_head=(2, 4, 8, 16, 32), + enc_patch_size=(128, 128, 128, 128, 128), + dec_depths=(2, 2, 2, 2), + dec_channels=(64, 64, 128, 256), + dec_num_head=(4, 4, 8, 16), + dec_patch_size=(128, 128, 128, 128), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=True, + enable_flash=False, + upcast_attention=True, + upcast_softmax=True, + cls_mode=False, + pdnorm_bn=False, + pdnorm_ln=False, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "S3DIS", "Structured3D"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.006, 0.0006], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="block", lr=0.0006)] + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.6, mode="random"), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={"coord": "origin_coord", "segment": "origin_segment"}, + ), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "origin_coord", + "segment", + "origin_segment", + ), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1, 1]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v3m1-2-ppt-extreme.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v3m1-2-ppt-extreme.py new file mode 100644 index 0000000000000000000000000000000000000000..e2d892af5b5317eef3b3554c3a49c0c1861a2645 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-pt-v3m1-2-ppt-extreme.py @@ -0,0 +1,487 @@ +""" +PTv3 + PPT +Pre-trained on ScanNet + Structured3D +(S3DIS is commented by default as a long data time issue of S3DIS: https://github.com/Pointcept/Pointcept/issues/103) +In the original PPT paper, 3 datasets are jointly trained and validated on the three datasets jointly with +one shared weight model. In PTv3, we trained on multi-dataset but only validated on one single dataset to +achieve extreme performance on one single dataset. + +To enable joint training on three datasets, uncomment config for the S3DIS dataset and change the "loop" of + Structured3D and ScanNet to 4 and 2 respectively. +""" + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 24 # bs: total bs in all gpus +num_worker = 48 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +find_unused_parameters = True + +# trainer +train = dict( + type="MultiDatasetTrainer", +) + +# model settings +model = dict( + type="PPT-v1m1", + backbone=dict( + type="PT-v3m1", + in_channels=6, + order=("z", "z-trans", "hilbert", "hilbert-trans"), + stride=(2, 2, 2, 2), + enc_depths=(2, 2, 2, 6, 2), + enc_channels=(32, 64, 128, 256, 512), + enc_num_head=(2, 4, 8, 16, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(2, 2, 2, 2), + dec_channels=(64, 64, 128, 256), + dec_num_head=(4, 4, 8, 16), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=True, + pdnorm_ln=True, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "S3DIS", "Structured3D"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + backbone_out_channels=64, + context_channels=256, + conditions=("Structured3D", "ScanNet", "S3DIS"), + template="[x]", + clip_model="ViT-B/16", + # fmt: off + class_name=( + "wall", "floor", "cabinet", "bed", "chair", "sofa", "table", "door", + "window", "bookshelf", "bookcase", "picture", "counter", "desk", "shelves", "curtain", + "dresser", "pillow", "mirror", "ceiling", "refrigerator", "television", "shower curtain", "nightstand", + "toilet", "sink", "lamp", "bathtub", "garbagebin", "board", "beam", "column", + "clutter", "otherstructure", "otherfurniture", "otherprop", + ), + valid_index=( + (0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 33, 34, 35), + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 20, 22, 24, 25, 27, 34), + (0, 1, 4, 5, 6, 7, 8, 10, 19, 29, 30, 31, 32), + ), + # fmt: on + backbone_mode=False, +) + +# scheduler settings +epoch = 100 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.005, 0.0005], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="block", lr=0.0005)] + +# dataset settings +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type="ConcatDataset", + datasets=[ + # Structured3D + dict( + type="Structured3DDataset", + split=["train", "val", "test"], + data_root="data/structured3d", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict( + # type="ElasticDistortion", + # distortion_params=[[0.2, 0.4], [0.8, 1.6]], + # ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "Structured3D"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=4, # sampling weight + ), + # ScanNet + dict( + type="ScanNetDataset", + split="train", + data_root="data/scannet", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict( + # type="ElasticDistortion", + # distortion_params=[[0.2, 0.4], [0.8, 1.6]], + # ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=102400, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=2, # sampling weight + ), + # S3DIS + dict( + type="S3DISDataset", + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root="data/s3dis", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict( + # type="ElasticDistortion", + # distortion_params=[[0.2, 0.4], [0.8, 1.6]], + # ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.6, mode="random"), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "S3DIS"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=1, # sampling weight + ), + ], + ), + val=dict( + type="S3DISDataset", + split="Area_5", + data_root="data/s3dis", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={"coord": "origin_coord", "segment": "origin_segment"}, + ), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict(type="Add", keys_dict={"condition": "S3DIS"}), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "origin_coord", + "segment", + "origin_segment", + "condition", + ), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type="S3DISDataset", + split="Area_5", + data_root="data/s3dis", + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "S3DIS"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-spunet-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-spunet-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..6545ca151ef59f6ea6e151a90de5e188db7e8ea7 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-spunet-v1m1-0-base.py @@ -0,0 +1,168 @@ +_base_ = ["../_base_/default_runtime.py"] +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=13, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict(type="PolyLR") + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=["coord", "color"], + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=["coord", "color"], + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[dict(type="CenterShift", apply_z=True), dict(type="NormalizeColor")], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + keys=("coord", "color"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-spunet-v1m1-0-cn-base.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-spunet-v1m1-0-cn-base.py new file mode 100644 index 0000000000000000000000000000000000000000..ee037115e69b6d7086109a31f7046520b42243f8 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-spunet-v1m1-0-cn-base.py @@ -0,0 +1,181 @@ +# spconv is too fast, data loading speed is bottleneck. Cache data is a better choice. + + +_base_ = ["../_base_/default_runtime.py"] +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=13, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict(type="PolyLR") + + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=["color", "normal"], + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={"coord": "origin_coord", "segment": "origin_segment"}, + ), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "origin_coord", + "segment", + "origin_segment", + ), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + feat_keys=["color", "normal"], + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[dict(type="CenterShift", apply_z=True), dict(type="NormalizeColor")], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-spunet-v1m2-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-spunet-v1m2-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..98511c5919b5f40271e4d500a30ed0bd42a6d2c9 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-spunet-v1m2-0-base.py @@ -0,0 +1,184 @@ +# spconv is too fast, data loading speed is bottleneck. Cache data is a better choice. + + +_base_ = ["../_base_/default_runtime.py"] +# misc custom setting +batch_size = 48 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m2", + in_channels=3, + num_classes=13, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + bn_momentum=0.1, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict(type="PolyLR") + + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=["color"], + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={"coord": "origin_coord", "segment": "origin_segment"}, + ), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "color", "segment"), + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "origin_coord", + "segment", + "origin_segment", + ), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + feat_keys=["color"], + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[dict(type="CenterShift", apply_z=True), dict(type="NormalizeColor")], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + keys=("coord", "color"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-swin3d-v1m1-0-small.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-swin3d-v1m1-0-small.py new file mode 100644 index 0000000000000000000000000000000000000000..119775214ad1e5e0f508d166e9aa9edfc987b76b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-swin3d-v1m1-0-small.py @@ -0,0 +1,184 @@ +_base_ = ["../_base_/default_runtime.py"] +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="Swin3D-v1m1", + in_channels=9, + num_classes=13, + base_grid_size=0.02, + depths=[2, 4, 9, 4, 4], + channels=[48, 96, 192, 384, 384], + num_heads=[6, 6, 12, 24, 24], + window_sizes=[5, 7, 7, 7, 7], + quant_size=4, + drop_path_rate=0.3, + up_k=3, + num_layers=5, + stem_transformer=True, + down_stride=3, + upsample="linear_attn", + knn_down=True, + cRSE="XYZ_RGB_NORM", + fp16_mode=1, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="AdamW", lr=0.001, weight_decay=0.05) +scheduler = dict(type="MultiStepLR", milestones=[0.6, 0.8], gamma=0.1) +param_dicts = [dict(keyword="blocks", lr=0.0001)] + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.8, 1.2]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="train", + return_grid_coord=True, + return_displacement=True, + ), + dict(type="SphereCrop", point_max=80000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="train", + return_grid_coord=True, + return_displacement=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[dict(type="CenterShift", apply_z=True), dict(type="NormalizeColor")], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + return_displacement=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-swin3d-v1m1-1-large.py b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-swin3d-v1m1-1-large.py new file mode 100644 index 0000000000000000000000000000000000000000..02c43d2debf08f26305d5d5636b857b5f34f17c4 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/s3dis/semseg-swin3d-v1m1-1-large.py @@ -0,0 +1,191 @@ +_base_ = ["../_base_/default_runtime.py"] +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="Swin3D-v1m1", + in_channels=9, + num_classes=13, + base_grid_size=0.02, + depths=[2, 4, 9, 4, 4], + channels=[80, 160, 320, 640, 640], + num_heads=[10, 10, 20, 40, 40], + window_sizes=[5, 7, 7, 7, 7], + quant_size=4, + drop_path_rate=0.3, + up_k=3, + num_layers=5, + stem_transformer=True, + down_stride=3, + upsample="linear_attn", + knn_down=True, + cRSE="XYZ_RGB_NORM", + fp16_mode=1, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 3000 +optimizer = dict(type="AdamW", lr=0.001, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.001, 0.0001], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="blocks", lr=0.0001)] + +# dataset settings +dataset_type = "S3DISDataset" +data_root = "data/s3dis" + +data = dict( + num_classes=13, + ignore_index=-1, + names=[ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ], + train=dict( + type=dataset_type, + split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.8, 1.2]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="train", + return_grid_coord=True, + return_displacement=True, + ), + dict(type="SphereCrop", point_max=80000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="train", + return_grid_coord=True, + return_displacement=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="Area_5", + data_root=data_root, + transform=[dict(type="CenterShift", apply_z=True), dict(type="NormalizeColor")], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.04, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + return_displacement=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [dict(type="RandomScale", scale=[0.9, 0.9])], + [dict(type="RandomScale", scale=[0.95, 0.95])], + [dict(type="RandomScale", scale=[1, 1])], + [dict(type="RandomScale", scale=[1.05, 1.05])], + [dict(type="RandomScale", scale=[1.1, 1.1])], + [ + dict(type="RandomScale", scale=[0.9, 0.9]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[0.95, 0.95]), + dict(type="RandomFlip", p=1), + ], + [dict(type="RandomScale", scale=[1, 1]), dict(type="RandomFlip", p=1)], + [ + dict(type="RandomScale", scale=[1.05, 1.05]), + dict(type="RandomFlip", p=1), + ], + [ + dict(type="RandomScale", scale=[1.1, 1.1]), + dict(type="RandomFlip", p=1), + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/insseg-pointgroup-v1m1-0-spunet-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/insseg-pointgroup-v1m1-0-spunet-base.py new file mode 100644 index 0000000000000000000000000000000000000000..3dec6d47c51545b55b543f80ac0c13556bacc84f --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/insseg-pointgroup-v1m1-0-spunet-base.py @@ -0,0 +1,187 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 12 +mix_prob = 0 +empty_cache = False +enable_amp = True +evaluate = True + +class_names = [ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", +] +num_classes = 20 +segment_ignore_index = (-1, 0, 1) + +# model settings +model = dict( + type="PG-v1m1", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=0, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + backbone_out_channels=96, + semantic_num_classes=num_classes, + semantic_ignore_index=-1, + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + cluster_thresh=1.5, + cluster_closed_points=300, + cluster_propose_points=100, + cluster_min_points=50, +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict(type="PolyLR") + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=num_classes, + ignore_index=-1, + names=class_names, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.5 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.1), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "instance_centroid", + "bbox", + ), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={ + "coord": "origin_coord", + "segment": "origin_segment", + "instance": "origin_instance", + }, + ), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + # dict(type="SphereCrop", point_max=1000000, mode='center'), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "origin_coord", + "origin_segment", + "origin_instance", + "instance_centroid", + "bbox", + ), + feat_keys=("color", "normal"), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + ), + ], + test_mode=False, + ), + test=dict(), # currently not available +) + +hooks = [ + dict(type="CheckpointLoader", keywords="module.", replacement="module."), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict( + type="InsSegEvaluator", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="CheckpointSaver", save_freq=None), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/insseg-ppt-v1m1-0-pointgroup-spunet-ft.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/insseg-ppt-v1m1-0-pointgroup-spunet-ft.py new file mode 100644 index 0000000000000000000000000000000000000000..09789883e5ae3c79f1cd3f3282380fdb30c21782 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/insseg-ppt-v1m1-0-pointgroup-spunet-ft.py @@ -0,0 +1,279 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0 +empty_cache = False +enable_amp = True +evaluate = True +find_unused_parameters = True + +class_names = [ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", +] +num_classes = 20 +segment_ignore_index = (-1, 0, 1) + +# model settings +model = dict( + type="PG-v1m1", + backbone=dict( + type="PPT-v1m1", + backbone=dict( + type="SpUNet-v1m3", + in_channels=6, + num_classes=0, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("ScanNet", "S3DIS", "Structured3D"), + zero_init=False, + norm_decouple=True, + norm_adaptive=True, + norm_affine=True, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], + backbone_out_channels=96, + context_channels=256, + conditions=("Structured3D", "ScanNet", "S3DIS"), + template="[x]", + clip_model="ViT-B/16", + class_name=( + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "bookcase", + "picture", + "counter", + "desk", + "shelves", + "curtain", + "dresser", + "pillow", + "mirror", + "ceiling", + "refrigerator", + "television", + "shower curtain", + "nightstand", + "toilet", + "sink", + "lamp", + "bathtub", + "garbagebin", + "board", + "beam", + "column", + "clutter", + "otherstructure", + "otherfurniture", + "otherprop", + ), + valid_index=( + ( + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 11, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 23, + 25, + 26, + 33, + 34, + 35, + ), + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 20, 22, 24, 25, 27, 34), + (0, 1, 4, 5, 6, 7, 8, 10, 19, 29, 30, 31, 32), + ), + backbone_mode=True, + ), + backbone_out_channels=96, + semantic_num_classes=num_classes, + semantic_ignore_index=-1, + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + cluster_thresh=1.5, + cluster_closed_points=300, + cluster_propose_points=100, + cluster_min_points=50, +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict(type="PolyLR") + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=num_classes, + ignore_index=-1, + names=class_names, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.5 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.1), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "instance_centroid", + "bbox", + "condition", + ), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={ + "coord": "origin_coord", + "segment": "origin_segment", + "instance": "origin_instance", + }, + ), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + # dict(type="SphereCrop", point_max=1000000, mode='center'), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "origin_coord", + "origin_segment", + "origin_instance", + "instance_centroid", + "bbox", + "condition", + ), + feat_keys=("color", "normal"), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + ), + ], + test_mode=False, + ), + test=dict(), # currently not available +) + +hooks = [ + dict(type="CheckpointLoader", keywords="module.", replacement="module.backbone."), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict( + type="InsSegEvaluator", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="CheckpointSaver", save_freq=None), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/objdet-cagroup3d-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/objdet-cagroup3d-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..8e31e32bf369b651f361135614953ae4fcaf1047 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/objdet-cagroup3d-v1m1-0-base.py @@ -0,0 +1,183 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 16 # bs: total bs in all gpus +num_worker = 32 +mix_prob = 0 +empty_cache = False +enable_amp = False +evaluate = True + +class_names = [ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", +] +num_classes = 20 +segment_ignore_index = (-1, 0, 1) + +# model settings +model = dict( + type="PG-v1m1", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=0, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + backbone_out_channels=96, + semantic_num_classes=num_classes, + semantic_ignore_index=-1, + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + cluster_thresh=1.5, + cluster_closed_points=300, + cluster_propose_points=100, + cluster_min_points=50, +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict(type="PolyLR") + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=num_classes, + ignore_index=-1, + names=class_names, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + # dict(type="CenterShift", apply_z=True), + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.5), + # # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + # dict(type="RandomRotate", angle=[-1, 1], axis='z', center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis='x', p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis='y', p=0.5), + # dict(type="RandomScale", scale=[0.9, 1.1]), + # # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + # dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + # dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + # dict(type="ChromaticTranslation", p=0.95, ratio=0.1), + # dict(type="ChromaticJitter", p=0.95, std=0.05), + # # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + # dict(type="GridSample", + # grid_size=0.02, + # hash_type='fnv', + # mode='train', + # return_grid_coord=True, + # keys=("coord", "color", "normal", "segment", "instance")), + # dict(type="SphereCrop", sample_rate=0.8, mode='random'), + # dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "instance_centroid", + "bbox", + ), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={ + "coord": "origin_coord", + "segment": "origin_segment", + "instance": "origin_instance", + }, + ), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + # dict(type="SphereCrop", point_max=1000000, mode='center'), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "origin_coord", + "origin_segment", + "origin_instance", + "instance_centroid", + "bbox", + ), + feat_keys=("color", "normal"), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + ), + ], + test_mode=False, + ), + test=dict(), # currently not available +) + +hooks = [ + dict(type="CheckpointLoader", keywords="module.", replacement="module."), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict( + type="InsSegEvaluator", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="CheckpointSaver", save_freq=None), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/pretrain-msc-v1m1-0-spunet-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/pretrain-msc-v1m1-0-spunet-base.py new file mode 100644 index 0000000000000000000000000000000000000000..3f56a96af8c57a8ebe05380c53c03245420d0d93 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/pretrain-msc-v1m1-0-spunet-base.py @@ -0,0 +1,155 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 32 # bs: total bs in all gpus +num_worker = 32 +mix_prob = 0 +empty_cache = False +enable_amp = False +evaluate = False +find_unused_parameters = False + +# model settings +model = dict( + type="MSC-v1m1", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=0, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + backbone_in_channels=6, + backbone_out_channels=96, + mask_grid_size=0.1, + mask_rate=0.4, + view1_mix_prob=0.8, + view2_mix_prob=0, + matching_max_k=8, + matching_max_radius=0.03, + matching_max_pair=8192, + nce_t=0.4, + contrast_weight=1, + reconstruct_weight=1, + reconstruct_color=True, + reconstruct_normal=False, +) + +# scheduler settings +epoch = 600 +optimizer = dict(type="SGD", lr=0.1, momentum=0.8, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.01, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split=["train", "val", "test"], + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="RandomScale", scale=[0.9, 1.1]), + dict(type="Copy", keys_dict={"coord": "origin_coord"}), + dict( + type="ContrastiveViewsGenerator", + view_keys=("coord", "color", "normal", "origin_coord"), + view_trans_cfg=[ + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=1), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=1), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="RandomColorJitter", + brightness=0.4, + contrast=0.4, + saturation=0.2, + hue=0.02, + p=0.8, + ), + dict(type="ChromaticJitter", p=0.95, std=0.05), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + keys=("origin_coord", "coord", "color", "normal"), + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.6, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + ], + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "view1_origin_coord", + "view1_grid_coord", + "view1_coord", + "view1_color", + "view1_normal", + "view2_origin_coord", + "view2_grid_coord", + "view2_coord", + "view2_color", + "view2_normal", + ), + offset_keys_dict=dict( + view1_offset="view1_coord", view2_offset="view2_coord" + ), + view1_feat_keys=("view1_color", "view1_normal"), + view2_feat_keys=("view2_color", "view2_normal"), + ), + ], + test_mode=False, + ), +) + +hooks = [ + dict(type="CheckpointLoader"), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict(type="CheckpointSaver", save_freq=None), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/pretrain-msc-v1m1-1-spunet-pointcontrast.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/pretrain-msc-v1m1-1-spunet-pointcontrast.py new file mode 100644 index 0000000000000000000000000000000000000000..9ff9061f2fb93a2e72343e8f066893fbae4a897a --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/pretrain-msc-v1m1-1-spunet-pointcontrast.py @@ -0,0 +1,162 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 32 # bs: total bs in all gpus +num_worker = 32 +mix_prob = 0 +empty_cache = False +enable_amp = False +evaluate = False +find_unused_parameters = False + +# model settings +model = dict( + type="MSC-v1m1", + backbone=dict( + type="SpUNet-v1m1", + in_channels=3, + num_classes=0, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + backbone_in_channels=3, + backbone_out_channels=96, + mask_grid_size=0.1, + mask_rate=0, + view1_mix_prob=0, + view2_mix_prob=0, + matching_max_k=8, + matching_max_radius=0.03, + matching_max_pair=4096, + nce_t=0.07, + contrast_weight=1, + reconstruct_weight=1, + reconstruct_color=False, + reconstruct_normal=False, +) + +# scheduler settings +epoch = 10 +eval_epoch = 10 +optimizer = dict(type="SGD", lr=0.1, momentum=0.8, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.01, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetPairDataset" +data_root = "data/scannet_pair" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + data_root=data_root, + view1_transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="Copy", keys_dict={"coord": "origin_coord"}), + # dict(type="RandomScale", scale=[0.9, 1.1]), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=1), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=1), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=1), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="RandomColorJitter", + brightness=0.4, + contrast=0.4, + saturation=0.2, + hue=0.02, + p=0.8, + ), + dict(type="ChromaticJitter", p=0.95, std=0.05), + dict( + type="GridSample", + grid_size=0.025, + hash_type="fnv", + mode="train", + keys=("origin_coord", "coord", "color"), + return_grid_coord=True, + ), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("origin_coord", "grid_coord", "coord", "color"), + offset_keys_dict=dict(offset="coord"), + feat_keys=["color"], + ), + ], + view2_transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="Copy", keys_dict={"coord": "origin_coord"}), + # dict(type="RandomScale", scale=[0.9, 1.1]), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=1), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=1), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=1), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="RandomColorJitter", + brightness=0.4, + contrast=0.4, + saturation=0.2, + hue=0.02, + p=0.8, + ), + dict(type="ChromaticJitter", p=0.95, std=0.05), + dict( + type="GridSample", + grid_size=0.025, + hash_type="fnv", + mode="train", + keys=("origin_coord", "coord", "color"), + return_grid_coord=True, + ), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("origin_coord", "grid_coord", "coord", "color"), + offset_keys_dict=dict(offset="coord"), + feat_keys=["color"], + ), + ], + test_mode=False, + ), +) + +hooks = [ + dict(type="CheckpointLoader"), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict(type="CheckpointSaver", save_freq=None), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/pretrain-msc-v1m2-0-spunet-csc.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/pretrain-msc-v1m2-0-spunet-csc.py new file mode 100644 index 0000000000000000000000000000000000000000..def70881496c3c04d1bf6a32df260fe9cbdac612 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/pretrain-msc-v1m2-0-spunet-csc.py @@ -0,0 +1,165 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 32 # bs: total bs in all gpus +num_worker = 32 +mix_prob = 0 +empty_cache = False +enable_amp = False +evaluate = False +find_unused_parameters = False + +# model settings +model = dict( + type="MSC-v1m2", + backbone=dict( + type="SpUNet-v1m1", + in_channels=3, + num_classes=0, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + backbone_in_channels=3, + backbone_out_channels=96, + mask_grid_size=0.1, + mask_rate=0, + view1_mix_prob=0, + view2_mix_prob=0, + matching_max_k=8, + matching_max_radius=0.03, + matching_max_pair=8192, + nce_t=0.4, + contrast_weight=1, + reconstruct_weight=1, + reconstruct_color=False, + reconstruct_normal=False, + partitions=4, + r1=2, + r2=20, +) + +# scheduler settings +epoch = 10 +eval_epoch = 10 +optimizer = dict(type="SGD", lr=0.1, momentum=0.8, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.01, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetPairDataset" +data_root = "data/scannet_pair" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + data_root=data_root, + view1_transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="Copy", keys_dict={"coord": "origin_coord"}), + # dict(type="RandomScale", scale=[0.9, 1.1]), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=1), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=1), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=1), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="RandomColorJitter", + brightness=0.4, + contrast=0.4, + saturation=0.2, + hue=0.02, + p=0.8, + ), + dict(type="ChromaticJitter", p=0.95, std=0.05), + dict( + type="GridSample", + grid_size=0.025, + hash_type="fnv", + mode="train", + keys=("origin_coord", "coord", "color"), + return_grid_coord=True, + ), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("origin_coord", "grid_coord", "coord", "color"), + offset_keys_dict=dict(offset="coord"), + feat_keys=["color"], + ), + ], + view2_transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="Copy", keys_dict={"coord": "origin_coord"}), + # dict(type="RandomScale", scale=[0.9, 1.1]), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=1), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=1), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=1), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="RandomColorJitter", + brightness=0.4, + contrast=0.4, + saturation=0.2, + hue=0.02, + p=0.8, + ), + dict(type="ChromaticJitter", p=0.95, std=0.05), + dict( + type="GridSample", + grid_size=0.025, + hash_type="fnv", + mode="train", + keys=("origin_coord", "coord", "color"), + return_grid_coord=True, + ), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("origin_coord", "grid_coord", "coord", "color"), + offset_keys_dict=dict(offset="coord"), + feat_keys=["color"], + ), + ], + test_mode=False, + ), +) + +hooks = [ + dict(type="CheckpointLoader"), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict(type="CheckpointSaver", save_freq=None), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-cac-v1m1-0-spunet-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-cac-v1m1-0-spunet-base.py new file mode 100644 index 0000000000000000000000000000000000000000..3968225e55db565a1e675605aa91d8a7a0353010 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-cac-v1m1-0-spunet-base.py @@ -0,0 +1,292 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="CAC-v1m1", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=0, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + num_classes=20, + backbone_out_channels=96, + cos_temp=15, + main_weight=1, + pre_weight=1, + pre_self_weight=1, + kl_weight=1, + conf_thresh=0.75, + detach_pre_logits=True, +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-cac-v1m1-1-spunet-lovasz.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-cac-v1m1-1-spunet-lovasz.py new file mode 100644 index 0000000000000000000000000000000000000000..3968225e55db565a1e675605aa91d8a7a0353010 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-cac-v1m1-1-spunet-lovasz.py @@ -0,0 +1,292 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="CAC-v1m1", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=0, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + num_classes=20, + backbone_out_channels=96, + cos_temp=15, + main_weight=1, + pre_weight=1, + pre_self_weight=1, + kl_weight=1, + conf_thresh=0.75, + detach_pre_logits=True, +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-cac-v1m1-2-ptv2-lovasz.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-cac-v1m1-2-ptv2-lovasz.py new file mode 100644 index 0000000000000000000000000000000000000000..f36a0c3a9426a13217cbefcd21b20f798b02b6a7 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-cac-v1m1-2-ptv2-lovasz.py @@ -0,0 +1,309 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="CAC-v1m1", + backbone=dict( + type="PT-v2m2", + in_channels=9, + num_classes=0, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.15, 0.375, 0.9375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + num_classes=20, + backbone_out_channels=48, + cos_temp=15, + main_weight=1, + pre_weight=1, + pre_self_weight=1, + kl_weight=1, + conf_thresh=0.75, + detach_pre_logits=True, +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-minkunet34c-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-minkunet34c-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..795998f368308c830fcdac760df67252b87ebab5 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-minkunet34c-0-base.py @@ -0,0 +1,193 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict(type="MinkUNet34C", in_channels=9, out_channels=20), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 600 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-oacnns-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-oacnns-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..57ef37e6aa35f153288571bceb76448463772365 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-oacnns-v1m1-0-base.py @@ -0,0 +1,290 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True +sync_bn = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="OACNNs", + in_channels=9, + num_classes=20, + embed_channels=64, + enc_channels=[64, 64, 128, 256], + groups=[4, 4, 8, 16], + enc_depth=[3, 3, 9, 8], + dec_channels=[256, 256, 256, 256], + point_grid_size=[[8, 12, 16, 16], [6, 9, 12, 12], [4, 6, 8, 8], [3, 4, 6, 6]], + dec_depth=[2, 2, 2, 2], + enc_num_ref=[16, 16, 16, 16], + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + + +epoch = 900 +optimizer = dict(type="AdamW", lr=0.001, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + return_min_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "normal", "color"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + return_min_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode='center'), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "normal", "color"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "normal", "color"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "normal", "color"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-octformer-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-octformer-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..cc0bafadc332cfa374e522699d314f721a9b57e8 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-octformer-v1m1-0-base.py @@ -0,0 +1,296 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = False + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="OctFormer-v1m1", + in_channels=10, + num_classes=20, + fpn_channels=168, + channels=(96, 192, 384, 384), + num_blocks=(2, 2, 18, 2), + num_heads=(6, 12, 24, 24), + patch_size=26, + stem_down=2, + head_up=2, + dilation=4, + drop_path=0.5, + nempty=True, + octree_depth=11, + octree_full_depth=2, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 600 +optimizer = dict(type="AdamW", lr=0.0015, weight_decay=0.05) +scheduler = dict( + type="MultiStepWithWarmupLR", + milestones=[0.6, 0.9], + gamma=0.1, + warmup_rate=0.05, + warmup_scale=1e-5, +) +param_dicts = [dict(keyword="blocks", lr=0.00015)] + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.1), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + return_min_coord=True, + return_displacement=True, + project_displacement=True, + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="SphereCrop", point_max=120000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "normal", "segment"), + feat_keys=("coord", "color", "normal", "displacement"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + return_min_coord=True, + return_displacement=True, + project_displacement=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "normal", "segment"), + feat_keys=("coord", "color", "normal", "displacement"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_displacement=True, + project_displacement=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "normal", "index"), + feat_keys=("coord", "color", "normal", "displacement"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-ppt-v1m1-0-sc-st-spunet.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-ppt-v1m1-0-sc-st-spunet.py new file mode 100644 index 0000000000000000000000000000000000000000..7fe0c7512b04aa874e32be04dc77ab35e706b67a --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-ppt-v1m1-0-sc-st-spunet.py @@ -0,0 +1,391 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 24 # bs: total bs in all gpus +num_worker = 48 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +find_unused_parameters = True + +# trainer +train = dict( + type="MultiDatasetTrainer", +) + +# model settings +model = dict( + type="PPT-v1m1", + backbone=dict( + type="SpUNet-v1m3", + in_channels=6, + num_classes=0, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("ScanNet", "S3DIS", "Structured3D"), + zero_init=False, + norm_decouple=True, + norm_adaptive=True, + norm_affine=True, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], + backbone_out_channels=96, + context_channels=256, + conditions=("Structured3D", "ScanNet", "S3DIS"), + template="[x]", + clip_model="ViT-B/16", + # fmt: off + class_name=( + "wall", "floor", "cabinet", "bed", "chair", "sofa", "table", "door", + "window", "bookshelf", "bookcase", "picture", "counter", "desk", "shelves", "curtain", + "dresser", "pillow", "mirror", "ceiling", "refrigerator", "television", "shower curtain", "nightstand", + "toilet", "sink", "lamp", "bathtub", "garbagebin", "board", "beam", "column", + "clutter", "otherstructure", "otherfurniture", "otherprop", + ), + valid_index=( + (0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 33, 34, 35), + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 20, 22, 24, 25, 27, 34), + (0, 1, 4, 5, 6, 7, 8, 10, 19, 29, 30, 31, 32), + ), + # fmt: on + backbone_mode=False, +) + +# scheduler settings +epoch = 100 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) +# param_dicts = [dict(keyword="modulation", lr=0.005)] + +# dataset settings +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type="ConcatDataset", + datasets=[ + # Structured3D + dict( + type="Structured3DDataset", + split="train", + data_root="data/structured3d", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "Structured3D"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=2, # sampling weight + ), + # ScanNet + dict( + type="ScanNetDataset", + split="train", + data_root="data/scannet", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=1, # sampling weight + ), + ], + ), + val=dict( + type="ScanNetDataset", + split="val", + data_root="data/scannet", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type="ScanNetDataset", + split="val", + data_root="data/scannet", + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-ppt-v1m1-1-sc-st-spunet-submit.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-ppt-v1m1-1-sc-st-spunet-submit.py new file mode 100644 index 0000000000000000000000000000000000000000..d503080e21ff7d921dd29ca5dd77e95ccfd51c72 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-ppt-v1m1-1-sc-st-spunet-submit.py @@ -0,0 +1,366 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 24 # bs: total bs in all gpus +num_worker = 48 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +find_unused_parameters = True +evaluate = False + +# trainer +train = dict( + type="MultiDatasetTrainer", +) + +# model settings +model = dict( + type="PPT-v1m1", + backbone=dict( + type="SpUNet-v1m3", + in_channels=6, + num_classes=0, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("ScanNet", "S3DIS", "Structured3D"), + zero_init=False, + norm_decouple=True, + norm_adaptive=True, + norm_affine=True, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], + backbone_out_channels=96, + context_channels=256, + conditions=("Structured3D", "ScanNet", "S3DIS"), + template="[x]", + clip_model="ViT-B/16", + # fmt: off + class_name=( + "wall", "floor", "cabinet", "bed", "chair", "sofa", "table", "door", + "window", "bookshelf", "bookcase", "picture", "counter", "desk", "shelves", "curtain", + "dresser", "pillow", "mirror", "ceiling", "refrigerator", "television", "shower curtain", "nightstand", + "toilet", "sink", "lamp", "bathtub", "garbagebin", "board", "beam", "column", + "clutter", "otherstructure", "otherfurniture", "otherprop", + ), + valid_index=( + (0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 33, 34, 35), + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 20, 22, 24, 25, 27, 34), + (0, 1, 4, 5, 6, 7, 8, 10, 19, 29, 30, 31, 32), + ), + # fmt: on + backbone_mode=False, +) + +# scheduler settings +epoch = 100 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) +# param_dicts = [dict(keyword="modulation", lr=0.005)] + +# dataset settings +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type="ConcatDataset", + datasets=[ + # Structured3D + dict( + type="Structured3DDataset", + split=["train", "val"], + data_root="data/structured3d", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "Structured3D"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=2, # sampling weight + ), + # ScanNet + dict( + type="ScanNetDataset", + split=["train", "val"], + data_root="data/scannet", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=1, # sampling weight + ), + ], + ), + test=dict( + type="ScanNetDataset", + split="test", + data_root="data/scannet", + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..f7b56590ad284eaf9cc2c3b616316627120bdb7b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v1-0-base.py @@ -0,0 +1,277 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PointTransformer-Seg50", + in_channels=9, + num_classes=20, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m1-0-origin.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m1-0-origin.py new file mode 100644 index 0000000000000000000000000000000000000000..fd1f61afea1657cbd3d8db536bdbf78865fce368 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m1-0-origin.py @@ -0,0 +1,297 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = False + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m1", + in_channels=9, + num_classes=20, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.15, 0.375, 0.9375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=True, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m2-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m2-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..3ec72b0177556edfb6bc2f93d286cf04b0a2b31e --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m2-0-base.py @@ -0,0 +1,297 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=9, + num_classes=20, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.15, 0.375, 0.9375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m2-1-submit.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m2-1-submit.py new file mode 100644 index 0000000000000000000000000000000000000000..daf9c9d6218d5f54f4a528d70c5a05c031c41910 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m2-1-submit.py @@ -0,0 +1,273 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True +evaluate = False + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=9, + num_classes=20, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.15, 0.375, 0.9375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split=["train", "val"], + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="test", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m2-2-precise-evaluate.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m2-2-precise-evaluate.py new file mode 100644 index 0000000000000000000000000000000000000000..c01cf0e9f21ed7aefa05a39c75a5b261e800faf4 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m2-2-precise-evaluate.py @@ -0,0 +1,307 @@ +""" +An example for enabling precise evaluation validation dataset during training. +PLease compare with semseg-pt-v2m2-0-base.py to lean the mechanism. +""" + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=9, + num_classes=20, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.15, 0.375, 0.9375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={"coord": "origin_coord", "segment": "origin_segment"}, + ), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "origin_coord", "segment", "origin_segment"), + feat_keys=("coord", "color", "normal"), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m2-3-lovasz.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m2-3-lovasz.py new file mode 100644 index 0000000000000000000000000000000000000000..ed46ff221baac62afcc03628f2d7aac399fa6e24 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v2m2-3-lovasz.py @@ -0,0 +1,300 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=9, + num_classes=20, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.15, 0.375, 0.9375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v3m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v3m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..47cc7e010a9ab3d9e7b4307d1a0d24da4d13f0f3 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v3m1-0-base.py @@ -0,0 +1,312 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentorV2", + num_classes=20, + backbone_out_channels=64, + backbone=dict( + type="PT-v3m1", + in_channels=6, + order=("z", "z-trans", "hilbert", "hilbert-trans"), + stride=(2, 2, 2, 2), + enc_depths=(2, 2, 2, 6, 2), + enc_channels=(32, 64, 128, 256, 512), + enc_num_head=(2, 4, 8, 16, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(2, 2, 2, 2), + dec_channels=(64, 64, 128, 256), + dec_num_head=(4, 4, 8, 16), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=False, + pdnorm_ln=False, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "S3DIS", "Structured3D"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.006, 0.0006], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="block", lr=0.0006)] + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=102400, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v3m1-1-ppt-extreme.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v3m1-1-ppt-extreme.py new file mode 100644 index 0000000000000000000000000000000000000000..4e4804eaa9510bde3235c72cb2a34d2a1907ac14 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-pt-v3m1-1-ppt-extreme.py @@ -0,0 +1,483 @@ +""" +PTv3 + PPT +Pre-trained on ScanNet + Structured3D +(S3DIS is commented by default as a long data time issue of S3DIS: https://github.com/Pointcept/Pointcept/issues/103) +In the original PPT paper, 3 datasets are jointly trained and validated on the three datasets jointly with +one shared weight model. In PTv3, we trained on multi-dataset but only validated on one single dataset to +achieve extreme performance on one single dataset. + +To enable joint training on three datasets, uncomment config for the S3DIS dataset and change the "loop" of + Structured3D and ScanNet to 4 and 2 respectively. +""" + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 24 # bs: total bs in all gpus +num_worker = 48 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +find_unused_parameters = True +clip_grad = 3.0 + +# trainer +train = dict( + type="MultiDatasetTrainer", +) + +# model settings +model = dict( + type="PPT-v1m1", + backbone=dict( + type="PT-v3m1", + in_channels=6, + order=("z", "z-trans", "hilbert", "hilbert-trans"), + stride=(2, 2, 2, 2), + enc_depths=(3, 3, 3, 6, 3), + enc_channels=(48, 96, 192, 384, 512), + enc_num_head=(3, 6, 12, 24, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(3, 3, 3, 3), + dec_channels=(64, 96, 192, 384), + dec_num_head=(4, 6, 12, 24), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=True, + pdnorm_ln=True, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "S3DIS", "Structured3D"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + backbone_out_channels=64, + context_channels=256, + conditions=("Structured3D", "ScanNet", "S3DIS"), + template="[x]", + clip_model="ViT-B/16", + # fmt: off + class_name=( + "wall", "floor", "cabinet", "bed", "chair", "sofa", "table", "door", + "window", "bookshelf", "bookcase", "picture", "counter", "desk", "shelves", "curtain", + "dresser", "pillow", "mirror", "ceiling", "refrigerator", "television", "shower curtain", "nightstand", + "toilet", "sink", "lamp", "bathtub", "garbagebin", "board", "beam", "column", + "clutter", "otherstructure", "otherfurniture", "otherprop", + ), + valid_index=( + (0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 33, 34, 35), + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 20, 22, 24, 25, 27, 34), + (0, 1, 4, 5, 6, 7, 8, 10, 19, 29, 30, 31, 32), + ), + # fmt: on + backbone_mode=False, +) + +# scheduler settings +epoch = 100 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.005, 0.0005], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="block", lr=0.0005)] + +# dataset settings +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type="ConcatDataset", + datasets=[ + # Structured3D + dict( + type="Structured3DDataset", + split=["train", "val", "test"], + data_root="data/structured3d", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "Structured3D"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=2, # sampling weight + ), + # ScanNet + dict( + type="ScanNetDataset", + split="train", + data_root="data/scannet", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=1, # sampling weight + ), + # S3DIS + # dict( + # type="S3DISDataset", + # split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + # data_root="data/s3dis", + # transform=[ + # dict(type="CenterShift", apply_z=True), + # dict( + # type="RandomDropout", + # dropout_ratio=0.2, + # dropout_application_ratio=0.2, + # ), + # # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + # dict( + # type="RandomRotate", + # angle=[-1, 1], + # axis="z", + # center=[0, 0, 0], + # p=0.5, + # ), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + # dict(type="RandomScale", scale=[0.9, 1.1]), + # # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + # dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict( + # type="ElasticDistortion", + # distortion_params=[[0.2, 0.4], [0.8, 1.6]], + # ), + # dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + # dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + # dict(type="ChromaticJitter", p=0.95, std=0.05), + # # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + # dict( + # type="GridSample", + # grid_size=0.02, + # hash_type="fnv", + # mode="train", + # return_grid_coord=True, + # ), + # dict(type="SphereCrop", sample_rate=0.6, mode="random"), + # dict(type="SphereCrop", point_max=204800, mode="random"), + # dict(type="CenterShift", apply_z=False), + # dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + # dict(type="Add", keys_dict={"condition": "S3DIS"}), + # dict(type="ToTensor"), + # dict( + # type="Collect", + # keys=("coord", "grid_coord", "segment", "condition"), + # feat_keys=("color", "normal"), + # ), + # ], + # test_mode=False, + # loop=1, # sampling weight + # ), + ], + ), + val=dict( + type="ScanNetDataset", + split="val", + data_root="data/scannet", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type="ScanNetDataset", + split="val", + data_root="data/scannet", + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..0e2e41aa5d949cad7428f722f017db73a565571a --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-0-base.py @@ -0,0 +1,281 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=20, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-1-interp-eval.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-1-interp-eval.py new file mode 100644 index 0000000000000000000000000000000000000000..9adfb382ad61433c7c73c3e502adbeaf411ae946 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-1-interp-eval.py @@ -0,0 +1,285 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=20, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={"coord": "origin_coord", "segment": "origin_segment"}, + ), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-1-precise-eval.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-1-precise-eval.py new file mode 100644 index 0000000000000000000000000000000000000000..7afb3aef572bc5e22b13f95fe70dbe43bdbb8d5f --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-1-precise-eval.py @@ -0,0 +1,289 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +hooks = [ + dict(type="CheckpointLoader"), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict(type="SemSegEvaluator"), + dict(type="CheckpointSaver", save_freq=None), + dict(type="PreciseEvaluator", test_last=False), +] + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=20, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-la100.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-la100.py new file mode 100644 index 0000000000000000000000000000000000000000..d8774321618c7d84e8a26911cfa418e777798c9c --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-la100.py @@ -0,0 +1,282 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=20, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + la_file="data/scannet/tasks/points/points100", + ignore_index=-1, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-la20.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-la20.py new file mode 100644 index 0000000000000000000000000000000000000000..1171c51184cab3372798d6bf94036719571880f0 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-la20.py @@ -0,0 +1,282 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=20, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + la_file="data/scannet/tasks/points/points20", + ignore_index=-1, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-la200.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-la200.py new file mode 100644 index 0000000000000000000000000000000000000000..158b0873af0db67a2877d966fb9dafdf417872ad --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-la200.py @@ -0,0 +1,282 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=20, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + la_file="data/scannet/tasks/points/points200", + ignore_index=-1, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-la50.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-la50.py new file mode 100644 index 0000000000000000000000000000000000000000..6eb906429e1209881b75f223ff85ff67d4d55594 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-la50.py @@ -0,0 +1,282 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=20, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + la_file="data/scannet/tasks/points/points50", + ignore_index=-1, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-lr1.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-lr1.py new file mode 100644 index 0000000000000000000000000000000000000000..6f5b2267f31b2e68306e414d46486a27081a7f1a --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-lr1.py @@ -0,0 +1,281 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=20, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + lr_file="data/scannet/tasks/scenes/1.txt", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-lr10.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-lr10.py new file mode 100644 index 0000000000000000000000000000000000000000..cff1df75e21a22e49486beb712e1443959f84447 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-lr10.py @@ -0,0 +1,281 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=20, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + lr_file="data/scannet/tasks/scenes/10.txt", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-lr20.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-lr20.py new file mode 100644 index 0000000000000000000000000000000000000000..0d1891fe6e0a984089d5b7ff465bd16e0a6c3ae8 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-lr20.py @@ -0,0 +1,281 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=20, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + lr_file="data/scannet/tasks/scenes/20.txt", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-lr5.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-lr5.py new file mode 100644 index 0000000000000000000000000000000000000000..e5a420a349a1ef0e12a58b3b4ea969092081a274 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-2-efficient-lr5.py @@ -0,0 +1,281 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=20, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + lr_file="data/scannet/tasks/scenes/5.txt", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-3-enable-profiler.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-3-enable-profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..32acb2be05092162561c66822e8cfe0758871147 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-3-enable-profiler.py @@ -0,0 +1,296 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = False + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=20, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) + +hooks = [ + dict(type="CheckpointLoader"), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict(type="SemSegEvaluator"), + dict(type="CheckpointSaver", save_freq=None), + dict( + type="RuntimeProfiler", + forward=True, + backward=True, + interrupt=True, + warm_up=2, + row_limit=30, + ), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-4-ft.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-4-ft.py new file mode 100644 index 0000000000000000000000000000000000000000..f90564e23b3422550c7fe209b977aa428c779b0e --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-4-ft.py @@ -0,0 +1,280 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 48 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=20, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-5-lovasz.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-5-lovasz.py new file mode 100644 index 0000000000000000000000000000000000000000..fb976abbd6570196e589ece51566a12e679c0364 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m1-5-lovasz.py @@ -0,0 +1,283 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=20, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m3-0-pdnorm-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m3-0-pdnorm-base.py new file mode 100644 index 0000000000000000000000000000000000000000..c6aed1fb9f5f4d7d7dd3b8608ac8be972931fbc3 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-spunet-v1m3-0-pdnorm-base.py @@ -0,0 +1,291 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m3", + in_channels=6, + num_classes=20, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("ScanNet", "S3DIS", "Structured3D"), + zero_init=False, + norm_decouple=True, + norm_adaptive=False, + norm_affine=True, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="Add", keys_dict=dict(condition="ScanNet")), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="Add", keys_dict=dict(condition="ScanNet")), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict=dict(condition="ScanNet")), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-st-v1m1-0-origin.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-st-v1m1-0-origin.py new file mode 100644 index 0000000000000000000000000000000000000000..4c05848c5212616f68db4ec70f7dfdebd1ef7d35 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-st-v1m1-0-origin.py @@ -0,0 +1,286 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 8 # bs: total bs in all gpus +mix_prob = 0 +empty_cache = False +enable_amp = True +find_unused_parameters = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="ST-v1m1", + downsample_scale=4, + depths=[3, 3, 9, 3, 3], + channels=[48, 96, 192, 384, 384], + num_heads=[3, 6, 12, 24, 24], + window_size=[0.1, 0.2, 0.4, 0.8, 1.6], + up_k=3, + grid_sizes=[0.02, 0.04, 0.08, 0.16, 0.32], + quant_sizes=[0.005, 0.01, 0.02, 0.04, 0.08], + rel_query=True, + rel_key=True, + rel_value=True, + drop_path_rate=0.3, + num_layers=5, + concat_xyz=True, + num_classes=20, + ratio=0.25, + k=16, + prev_grid_size=0.02, + sigma=1.0, + stem_transformer=False, + kp_ball_radius=0.02 * 2.5, + kp_max_neighbor=34, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 600 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict(type="MultiStepLR", milestones=[0.6, 0.8], gamma=0.1) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", keys=("coord", "segment"), feat_keys=("coord", "color") + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", keys=("coord", "segment"), feat_keys=("coord", "color") + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-st-v1m2-0-refined.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-st-v1m2-0-refined.py new file mode 100644 index 0000000000000000000000000000000000000000..7b9963eedc36be41897e041bc67c245cab0c1e5b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-st-v1m2-0-refined.py @@ -0,0 +1,287 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 8 # bs: total bs in all gpus +mix_prob = 0 +empty_cache = False +enable_amp = True +find_unused_parameters = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="ST-v1m2", + in_channels=9, + num_classes=20, + channels=(48, 96, 192, 384, 384), + num_heads=(6, 12, 24, 24), + depths=(3, 9, 3, 3), + window_size=(0.2, 0.4, 0.8, 1.6), + quant_size=(0.01, 0.02, 0.04, 0.08), + mlp_expend_ratio=4.0, + down_ratio=0.25, + down_num_sample=16, + kp_ball_radius=2.5 * 0.02, + kp_max_neighbor=34, + kp_grid_size=0.02, + kp_sigma=1.0, + drop_path_rate=0.2, + rel_query=True, + rel_key=True, + rel_value=True, + qkv_bias=True, + stem=True, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) +# scheduler settings +epoch = 600 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict(type="MultiStepLR", milestones=[0.6, 0.8], gamma=0.1) + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-swin3d-v1m1-0-small.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-swin3d-v1m1-0-small.py new file mode 100644 index 0000000000000000000000000000000000000000..e8d8308de55cd42ba4ea0bea8a3b951bfda8d6df --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-swin3d-v1m1-0-small.py @@ -0,0 +1,219 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="Swin3D-v1m1", + in_channels=9, + num_classes=20, + base_grid_size=0.02, + depths=[2, 4, 9, 4, 4], + channels=[48, 96, 192, 384, 384], + num_heads=[6, 6, 12, 24, 24], + window_sizes=[5, 7, 7, 7, 7], + quant_size=4, + drop_path_rate=0.3, + up_k=3, + num_layers=5, + stem_transformer=True, + down_stride=3, + upsample="linear_attn", + knn_down=True, + cRSE="XYZ_RGB_NORM", + fp16_mode=1, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 600 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.006, 0.0006], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="blocks", lr=0.0006)] + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.8, 1.2]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + return_displacement=True, + ), + dict(type="SphereCrop", point_max=120000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + return_displacement=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + return_displacement=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-swin3d-v1m1-1-large.py b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-swin3d-v1m1-1-large.py new file mode 100644 index 0000000000000000000000000000000000000000..0957ff85e1902220e0d13676b6814e5420a1776d --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet/semseg-swin3d-v1m1-1-large.py @@ -0,0 +1,219 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="Swin3D-v1m1", + in_channels=9, + num_classes=20, + base_grid_size=0.02, + depths=[2, 4, 9, 4, 4], + channels=[80, 160, 320, 640, 640], + num_heads=[10, 10, 20, 40, 40], + window_sizes=[5, 7, 7, 7, 7], + quant_size=4, + drop_path_rate=0.3, + up_k=3, + num_layers=5, + stem_transformer=True, + down_stride=3, + upsample="linear_attn", + knn_down=True, + cRSE="XYZ_RGB_NORM", + fp16_mode=1, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 600 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.006, 0.0006], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="blocks", lr=0.0006)] + +# dataset settings +dataset_type = "ScanNetDataset" +data_root = "data/scannet" + +data = dict( + num_classes=20, + ignore_index=-1, + names=[ + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refridgerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", + ], + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.8, 1.2]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + return_displacement=True, + ), + dict(type="SphereCrop", point_max=120000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + return_displacement=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + return_displacement=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/insseg-pointgroup-spunet-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/insseg-pointgroup-spunet-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..90a8d5e384b59fd5e6a004d57006861de9ca921a --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/insseg-pointgroup-spunet-0-base.py @@ -0,0 +1,170 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 12 +mix_prob = 0 +empty_cache = False +enable_amp = True +evaluate = True + +class_names = CLASS_LABELS_200 +num_classes = 200 +segment_ignore_index = (-1, 0, 2) + +# model settings +model = dict( + type="PG-v1m1", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=0, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + backbone_out_channels=96, + semantic_num_classes=num_classes, + semantic_ignore_index=-1, + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + cluster_thresh=1.5, + cluster_closed_points=300, + cluster_propose_points=100, + cluster_min_points=50, +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict(type="PolyLR") + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=num_classes, + ignore_index=-1, + names=class_names, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.5 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.1), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "instance_centroid", + "bbox", + ), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="Copy", + keys_dict={ + "coord": "origin_coord", + "segment": "origin_segment", + "instance": "origin_instance", + }, + ), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + keys=("coord", "color", "normal", "segment", "instance"), + ), + # dict(type="SphereCrop", point_max=1000000, mode='center'), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict( + type="InstanceParser", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=( + "coord", + "grid_coord", + "segment", + "instance", + "origin_coord", + "origin_segment", + "origin_instance", + "instance_centroid", + "bbox", + ), + feat_keys=("color", "normal"), + offset_keys_dict=dict(offset="coord", origin_offset="origin_coord"), + ), + ], + test_mode=False, + ), + test=dict(), # currently not available +) + +hooks = [ + dict(type="CheckpointLoader", keywords="module.", replacement="module."), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict( + type="InsSegEvaluator", + segment_ignore_index=segment_ignore_index, + instance_ignore_index=-1, + ), + dict(type="CheckpointSaver", save_freq=None), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-cac-v1m1-0-spunet-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-cac-v1m1-0-spunet-base.py new file mode 100644 index 0000000000000000000000000000000000000000..736bc767abdcba23de5676c5321cef01e19707c4 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-cac-v1m1-0-spunet-base.py @@ -0,0 +1,192 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="CAC-v1m1", + backbone=dict( + type="SpUNet-v1m1", + in_channels=9, + num_classes=0, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], + num_classes=200, + backbone_out_channels=96, + cos_temp=15, + main_weight=1, + pre_weight=1, + pre_self_weight=1, + kl_weight=1, + conf_thresh=0, + detach_pre_logits=True, +) + + +# scheduler settings +epoch = 600 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=200, + ignore_index=-1, + names=CLASS_LABELS_200, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-cac-v1m1-1-spunet-lovasz.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-cac-v1m1-1-spunet-lovasz.py new file mode 100644 index 0000000000000000000000000000000000000000..e5584a8b763e911a896d4c948cc42f4d0da880e1 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-cac-v1m1-1-spunet-lovasz.py @@ -0,0 +1,195 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="CAC-v1m1", + backbone=dict( + type="SpUNet-v1m1", + in_channels=9, + num_classes=0, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + num_classes=200, + backbone_out_channels=96, + cos_temp=15, + main_weight=1, + pre_weight=1, + pre_self_weight=1, + kl_weight=1, + conf_thresh=0, + detach_pre_logits=True, +) + + +# scheduler settings +epoch = 600 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=200, + ignore_index=-1, + names=CLASS_LABELS_200, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-cac-v1m1-2-ptv2-lovasz.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-cac-v1m1-2-ptv2-lovasz.py new file mode 100644 index 0000000000000000000000000000000000000000..bbd49a65d32cac158ece4cf06c641d08b452207c --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-cac-v1m1-2-ptv2-lovasz.py @@ -0,0 +1,292 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="CAC-v1m1", + backbone=dict( + type="PT-v2m2", + in_channels=9, + num_classes=0, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.15, 0.375, 0.9375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + num_classes=200, + backbone_out_channels=48, + cos_temp=15, + main_weight=1, + pre_weight=1, + pre_self_weight=1, + kl_weight=1, + conf_thresh=0, + detach_pre_logits=True, +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=200, + ignore_index=-1, + names=CLASS_LABELS_200, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-minkunet34c-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-minkunet34c-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..dd8479414cc4a8306c1914490c6621cb6debaeab --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-minkunet34c-0-base.py @@ -0,0 +1,176 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict(type="MinkUNet34C", in_channels=9, out_channels=200), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 600 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=200, + ignore_index=-1, + names=CLASS_LABELS_200, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 6, 1 / 6], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 6, 1 / 6], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..60c1e2fbd99f8cfac633545d265bfc6cbc7219f8 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v1-0-base.py @@ -0,0 +1,260 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PointTransformer-Seg50", + in_channels=9, + num_classes=200, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=200, + ignore_index=-1, + names=CLASS_LABELS_200, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v2m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v2m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..b454f1de6563aec984578c8b7bff52bab7010218 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v2m1-0-base.py @@ -0,0 +1,280 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = False + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m1", + in_channels=9, + num_classes=200, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.15, 0.375, 0.9375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=True, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=200, + ignore_index=-1, + names=CLASS_LABELS_200, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v2m2-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v2m2-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..51287dd96eb4d2af1cbe01cbf29e029918dc21b8 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v2m2-0-base.py @@ -0,0 +1,280 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=9, + num_classes=200, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.15, 0.375, 0.9375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=200, + ignore_index=-1, + names=CLASS_LABELS_200, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v2m2-1-benchmark-submit.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v2m2-1-benchmark-submit.py new file mode 100644 index 0000000000000000000000000000000000000000..e8d3d1e449b688dc04d225d22626b2009285e52d --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v2m2-1-benchmark-submit.py @@ -0,0 +1,256 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True +evaluate = False + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=9, + num_classes=200, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.15, 0.375, 0.9375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=200, + ignore_index=-1, + names=CLASS_LABELS_200, + train=dict( + type=dataset_type, + split=["train", "val"], + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="test", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v2m2-2-lovasz.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v2m2-2-lovasz.py new file mode 100644 index 0000000000000000000000000000000000000000..c3ab0f65f6e56a5faa876ab4ec52f531cd9e2453 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v2m2-2-lovasz.py @@ -0,0 +1,283 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=9, + num_classes=200, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.15, 0.375, 0.9375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=200, + ignore_index=-1, + names=CLASS_LABELS_200, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v3m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v3m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..ed73ca90ec80dec027906fb69b1678a006840f36 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v3m1-0-base.py @@ -0,0 +1,295 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentorV2", + num_classes=200, + backbone_out_channels=64, + backbone=dict( + type="PT-v3m1", + in_channels=6, + order=["z", "z-trans", "hilbert", "hilbert-trans"], + stride=(2, 2, 2, 2), + enc_depths=(2, 2, 2, 6, 2), + enc_channels=(32, 64, 128, 256, 512), + enc_num_head=(2, 4, 8, 16, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(2, 2, 2, 2), + dec_channels=(64, 64, 128, 256), + dec_num_head=(4, 4, 8, 16), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=False, + pdnorm_ln=False, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "S3DIS", "Structured3D"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.006, 0.0006], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="block", lr=0.0006)] + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=200, + ignore_index=-1, + names=CLASS_LABELS_200, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=102400, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v3m1-1-ppt-ft.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v3m1-1-ppt-ft.py new file mode 100644 index 0000000000000000000000000000000000000000..fa92b10fe935193e4333b19e78ad8997aab1b102 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-pt-v3m1-1-ppt-ft.py @@ -0,0 +1,299 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +find_unused_parameters = True + +# model settings +model = dict( + type="DefaultSegmentorV2", + num_classes=200, + backbone_out_channels=64, + backbone=dict( + type="PT-v3m1", + in_channels=6, + order=("z", "z-trans", "hilbert", "hilbert-trans"), + stride=(2, 2, 2, 2), + enc_depths=(3, 3, 3, 6, 3), + enc_channels=(48, 96, 192, 384, 512), + enc_num_head=(3, 6, 12, 24, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(3, 3, 3, 3), + dec_channels=(64, 96, 192, 384), + dec_num_head=(4, 6, 12, 24), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=True, + pdnorm_ln=True, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "S3DIS", "Structured3D"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.006, 0.0006], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="block", lr=0.0006)] + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=200, + ignore_index=-1, + names=CLASS_LABELS_200, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=102400, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-spunet-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-spunet-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..1fcd0fecf93c058b0f68c251ebb0210a64da976b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-spunet-v1m1-0-base.py @@ -0,0 +1,182 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=9, + num_classes=200, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 600 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=200, + ignore_index=-1, + names=CLASS_LABELS_200, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-spunet-v1m1-1-lovasz.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-spunet-v1m1-1-lovasz.py new file mode 100644 index 0000000000000000000000000000000000000000..12c17df2be7bf2895c4e2018c124c196e8297f80 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-spunet-v1m1-1-lovasz.py @@ -0,0 +1,185 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=9, + num_classes=200, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 600 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=200, + ignore_index=-1, + names=CLASS_LABELS_200, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-st-v1m2-0-refined.py b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-st-v1m2-0-refined.py new file mode 100644 index 0000000000000000000000000000000000000000..98363e34c91f344faf410493066f6291d8b75eaf --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannet200/semseg-st-v1m2-0-refined.py @@ -0,0 +1,270 @@ +from pointcept.datasets.preprocessing.scannet.meta_data.scannet200_constants import ( + CLASS_LABELS_200, +) + +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 8 # bs: total bs in all gpus +mix_prob = 0 +empty_cache = False +enable_amp = True +find_unused_parameters = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="ST-v1m2", + in_channels=9, + num_classes=200, + channels=(48, 96, 192, 384, 384), + num_heads=(6, 12, 24, 24), + depths=(3, 9, 3, 3), + window_size=(0.2, 0.4, 0.8, 1.6), + quant_size=(0.01, 0.02, 0.04, 0.08), + mlp_expend_ratio=4.0, + down_ratio=0.25, + down_num_sample=16, + kp_ball_radius=2.5 * 0.02, + kp_max_neighbor=34, + kp_grid_size=0.02, + kp_sigma=1.0, + drop_path_rate=0.2, + rel_query=True, + rel_key=True, + rel_value=True, + qkv_bias=True, + stem=True, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) +# scheduler settings +epoch = 600 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict(type="MultiStepLR", milestones=[0.6, 0.8], gamma=0.1) + +# dataset settings +dataset_type = "ScanNet200Dataset" +data_root = "data/scannet" + +data = dict( + num_classes=200, + ignore_index=-1, + names=CLASS_LABELS_200, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="SphereCrop", point_max=100000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_min_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v2m2-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v2m2-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..81baa7c600fbcfce1121899897879fd13f9ed0eb --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v2m2-0-base.py @@ -0,0 +1,291 @@ +_base_ = [ + "../_base_/default_runtime.py", + "../_base_/dataset/scannetpp.py", +] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=9, + num_classes=100, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.15, 0.375, 0.9375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNetPPDataset" +data_root = "data/scannetpp" + +data = dict( + num_classes=100, + ignore_index=-1, + train=dict( + type=dataset_type, + split="train_grid1mm_chunk6x6_stride3x3", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + dict(type="Copy", keys_dict={"segment": "origin_segment"}), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + keys=("coord", "color", "normal", "segment"), + return_inverse=True, + ), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v2m2-1-submit.py b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v2m2-1-submit.py new file mode 100644 index 0000000000000000000000000000000000000000..e068997c5158ab4d1e84f3d2e75454198ee6897f --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v2m2-1-submit.py @@ -0,0 +1,278 @@ +_base_ = [ + "../_base_/default_runtime.py", + "../_base_/dataset/scannetpp.py", +] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +evaluate = False + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=9, + num_classes=100, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.15, 0.375, 0.9375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 900 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.02) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) + +# dataset settings +dataset_type = "ScanNetPPDataset" +data_root = "data/scannetpp" + +data = dict( + num_classes=100, + ignore_index=-1, + train=dict( + type=dataset_type, + split=["train_grid1mm_chunk6x6_stride3x3", "val_grid1mm_chunk6x6_stride3x3"], + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="test", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + dict(type="Copy", keys_dict={"segment": "origin_segment"}), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + keys=("coord", "color", "normal", "segment"), + return_inverse=True, + ), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) + +# hook +hooks = [ + dict(type="CheckpointLoader"), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict(type="SemSegEvaluator"), + dict(type="CheckpointSaver", save_freq=None), + dict(type="PreciseEvaluator", test_last=True), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v3m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v3m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..efd95ad6190fb3a14234a9da293338ec67140658 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v3m1-0-base.py @@ -0,0 +1,302 @@ +_base_ = [ + "../_base_/default_runtime.py", + "../_base_/dataset/scannetpp.py", +] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentorV2", + num_classes=100, + backbone_out_channels=64, + backbone=dict( + type="PT-v3m1", + in_channels=6, + order=("z", "z-trans", "hilbert", "hilbert-trans"), + stride=(2, 2, 2, 2), + enc_depths=(2, 2, 2, 6, 2), + enc_channels=(32, 64, 128, 256, 512), + enc_num_head=(2, 4, 8, 16, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(2, 2, 2, 2), + dec_channels=(64, 64, 128, 256), + dec_num_head=(4, 4, 8, 16), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=False, + pdnorm_ln=False, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "S3DIS", "Structured3D"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.006, 0.0006], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="block", lr=0.0006)] + +# dataset settings +dataset_type = "ScanNetPPDataset" +data_root = "data/scannetpp" + +data = dict( + num_classes=100, + ignore_index=-1, + train=dict( + type=dataset_type, + split="train_grid1mm_chunk6x6_stride3x3", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + dict(type="Copy", keys_dict={"segment": "origin_segment"}), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + keys=("coord", "color", "normal", "segment"), + return_inverse=True, + ), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v3m1-1-submit.py b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v3m1-1-submit.py new file mode 100644 index 0000000000000000000000000000000000000000..9b91ca6d93037bf502566afd3a704b2ef7aa010b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v3m1-1-submit.py @@ -0,0 +1,289 @@ +_base_ = [ + "../_base_/default_runtime.py", + "../_base_/dataset/scannetpp.py", +] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +evaluate = False + +# model settings +model = dict( + type="DefaultSegmentorV2", + num_classes=100, + backbone_out_channels=64, + backbone=dict( + type="PT-v3m1", + in_channels=6, + order=("z", "z-trans", "hilbert", "hilbert-trans"), + stride=(2, 2, 2, 2), + enc_depths=(2, 2, 2, 6, 2), + enc_channels=(32, 64, 128, 256, 512), + enc_num_head=(2, 4, 8, 16, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(2, 2, 2, 2), + dec_channels=(64, 64, 128, 256), + dec_num_head=(4, 4, 8, 16), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=False, + pdnorm_ln=False, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "S3DIS", "Structured3D"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 800 +optimizer = dict(type="AdamW", lr=0.006, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.006, 0.0006], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="block", lr=0.0006)] + +# dataset settings +dataset_type = "ScanNetPPDataset" +data_root = "data/scannetpp" + +data = dict( + num_classes=100, + ignore_index=-1, + train=dict( + type=dataset_type, + split=["train_grid1mm_chunk6x6_stride3x3", "val_grid1mm_chunk6x6_stride3x3"], + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="test", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + dict(type="Copy", keys_dict={"segment": "origin_segment"}), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + keys=("coord", "color", "normal", "segment"), + return_inverse=True, + ), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) + +# hook +hooks = [ + dict(type="CheckpointLoader"), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict(type="SemSegEvaluator"), + dict(type="CheckpointSaver", save_freq=None), + dict(type="PreciseEvaluator", test_last=True), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v3m1-2-ppt-extreme.py b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v3m1-2-ppt-extreme.py new file mode 100644 index 0000000000000000000000000000000000000000..5b0c756223bd65cafca57ba077ff2d6830887403 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v3m1-2-ppt-extreme.py @@ -0,0 +1,499 @@ +_base_ = [ + "../_base_/default_runtime.py", + "../_base_/dataset/scannetpp.py", +] + +# misc custom setting +batch_size = 24 # bs: total bs in all gpus +num_worker = 48 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +find_unused_parameters = True + +# trainer +train = dict( + type="MultiDatasetTrainer", +) + +# model settings +model = dict( + type="PPT-v1m2", + backbone=dict( + type="PT-v3m1", + in_channels=6, + order=("z", "z-trans", "hilbert", "hilbert-trans"), + stride=(2, 2, 2, 2), + enc_depths=(3, 3, 3, 6, 3), + enc_channels=(48, 96, 192, 384, 512), + enc_num_head=(3, 6, 12, 24, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(3, 3, 3, 3), + dec_channels=(64, 96, 192, 384), + dec_num_head=(4, 6, 12, 24), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=True, + pdnorm_ln=True, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "ScanNet++", "S3DIS", "Structured3D"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + backbone_out_channels=64, + context_channels=256, + conditions=("ScanNet", "ScanNet++", "S3DIS", "Structured3D"), + num_classes=(200, 100, 13, 25), +) + +# scheduler settings +epoch = 100 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.005, 0.0005], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="block", lr=0.0005)] + +# dataset settings +data = dict( + num_classes=100, + ignore_index=-1, + train=dict( + type="ConcatDataset", + datasets=[ + # Structured3D + dict( + type="Structured3DDataset", + split=["train", "val", "test"], + data_root="data/structured3d", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "Structured3D"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=2, # sampling weight + ), + # ScanNet + dict( + type="ScanNet200Dataset", + split=["train", "val"], + data_root="data/scannet", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=1, # sampling weight + ), + # S3DIS + # dict( + # type="S3DISDataset", + # split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + # data_root="data/s3dis", + # transform=[ + # dict(type="CenterShift", apply_z=True), + # dict( + # type="RandomDropout", + # dropout_ratio=0.2, + # dropout_application_ratio=0.2, + # ), + # # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + # dict( + # type="RandomRotate", + # angle=[-1, 1], + # axis="z", + # center=[0, 0, 0], + # p=0.5, + # ), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + # dict(type="RandomScale", scale=[0.9, 1.1]), + # # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + # dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict( + # type="ElasticDistortion", + # distortion_params=[[0.2, 0.4], [0.8, 1.6]], + # ), + # dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + # dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + # dict(type="ChromaticJitter", p=0.95, std=0.05), + # # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + # dict( + # type="GridSample", + # grid_size=0.02, + # hash_type="fnv", + # mode="train", + # return_grid_coord=True, + # ), + # dict(type="SphereCrop", sample_rate=0.6, mode="random"), + # dict(type="SphereCrop", point_max=204800, mode="random"), + # dict(type="CenterShift", apply_z=False), + # dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + # dict(type="Add", keys_dict={"condition": "S3DIS"}), + # dict(type="ToTensor"), + # dict( + # type="Collect", + # keys=("coord", "grid_coord", "segment", "condition"), + # feat_keys=("color", "normal"), + # ), + # ], + # test_mode=False, + # loop=1, # sampling weight + # ), + dict( + type="ScanNetPPDataset", + split="train_grid1mm_chunk6x6_stride3x3", + data_root="data/scannetpp", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "ScanNet++"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + ], + ), + val=dict( + type="ScanNetPPDataset", + split="val", + data_root="data/scannetpp", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict(type="Add", keys_dict={"condition": "ScanNet++"}), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type="ScanNetPPDataset", + split="val", + data_root="data/scannetpp", + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + dict(type="Copy", keys_dict={"segment": "origin_segment"}), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + keys=("coord", "color", "normal", "segment"), + return_inverse=True, + ), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "ScanNet++"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v3m1-3-ppt-extreme-submit.py b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v3m1-3-ppt-extreme-submit.py new file mode 100644 index 0000000000000000000000000000000000000000..eb48cf2db8af6d55dd85945e92be33df1e49b5fc --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-pt-v3m1-3-ppt-extreme-submit.py @@ -0,0 +1,488 @@ +_base_ = [ + "../_base_/default_runtime.py", + "../_base_/dataset/scannetpp.py", +] + +# misc custom setting +batch_size = 24 # bs: total bs in all gpus +num_worker = 48 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +evaluate = False +find_unused_parameters = True + +# trainer +train = dict( + type="MultiDatasetTrainer", +) + +# model settings +model = dict( + type="PPT-v1m2", + backbone=dict( + type="PT-v3m1", + in_channels=6, + order=("z", "z-trans", "hilbert", "hilbert-trans"), + stride=(2, 2, 2, 2), + enc_depths=(3, 3, 3, 6, 3), + enc_channels=(48, 96, 192, 384, 512), + enc_num_head=(3, 6, 12, 24, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(3, 3, 3, 3), + dec_channels=(64, 96, 192, 384), + dec_num_head=(4, 6, 12, 24), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=True, + pdnorm_ln=True, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "ScanNet++", "S3DIS", "Structured3D"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + backbone_out_channels=64, + context_channels=256, + conditions=("ScanNet", "ScanNet++", "S3DIS", "Structured3D"), + num_classes=(200, 100, 13, 25), +) + +# scheduler settings +epoch = 100 +optimizer = dict(type="AdamW", lr=0.005, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.005, 0.0005], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="block", lr=0.0005)] + +# dataset settings +data = dict( + num_classes=100, + ignore_index=-1, + train=dict( + type="ConcatDataset", + datasets=[ + # Structured3D + dict( + type="Structured3DDataset", + split=["train", "val", "test"], + data_root="data/structured3d", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "Structured3D"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=2, # sampling weight + ), + # ScanNet + dict( + type="ScanNet200Dataset", + split=["train", "val"], + data_root="data/scannet", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=1, # sampling weight + ), + # S3DIS + # dict( + # type="S3DISDataset", + # split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + # data_root="data/s3dis", + # transform=[ + # dict(type="CenterShift", apply_z=True), + # dict( + # type="RandomDropout", + # dropout_ratio=0.2, + # dropout_application_ratio=0.2, + # ), + # # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + # dict( + # type="RandomRotate", + # angle=[-1, 1], + # axis="z", + # center=[0, 0, 0], + # p=0.5, + # ), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + # dict(type="RandomScale", scale=[0.9, 1.1]), + # # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + # dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict( + # type="ElasticDistortion", + # distortion_params=[[0.2, 0.4], [0.8, 1.6]], + # ), + # dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + # dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + # dict(type="ChromaticJitter", p=0.95, std=0.05), + # # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + # dict( + # type="GridSample", + # grid_size=0.02, + # hash_type="fnv", + # mode="train", + # return_grid_coord=True, + # ), + # dict(type="SphereCrop", sample_rate=0.6, mode="random"), + # dict(type="SphereCrop", point_max=204800, mode="random"), + # dict(type="CenterShift", apply_z=False), + # dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + # dict(type="Add", keys_dict={"condition": "S3DIS"}), + # dict(type="ToTensor"), + # dict( + # type="Collect", + # keys=("coord", "grid_coord", "segment", "condition"), + # feat_keys=("color", "normal"), + # ), + # ], + # test_mode=False, + # loop=1, # sampling weight + # ), + dict( + type="ScanNetPPDataset", + split=[ + "train_grid1mm_chunk6x6_stride3x3", + "val_grid1mm_chunk6x6_stride3x3", + ], + data_root="data/scannetpp", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "ScanNet++"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + ], + ), + test=dict( + type="ScanNetPPDataset", + split="test", + data_root="data/scannetpp", + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + dict(type="Copy", keys_dict={"segment": "origin_segment"}), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + keys=("coord", "color", "normal", "segment"), + return_inverse=True, + ), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "ScanNet++"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) + +# hook +hooks = [ + dict(type="CheckpointLoader"), + dict(type="IterationTimer", warmup_iter=2), + dict(type="InformationWriter"), + dict(type="SemSegEvaluator"), + dict(type="CheckpointSaver", save_freq=None), + dict(type="PreciseEvaluator", test_last=True), +] diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-spunet-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-spunet-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..871aa42954765ea6091c328910e40ee6c31e6173 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-spunet-v1m1-0-base.py @@ -0,0 +1,271 @@ +_base_ = [ + "../_base_/default_runtime.py", + "../_base_/dataset/scannetpp.py", +] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=100, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + + +# scheduler settings +epoch = 800 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "ScanNetPPDataset" +data_root = "data/scannetpp" + +data = dict( + num_classes=100, + ignore_index=-1, + train=dict( + type=dataset_type, + split="train_grid1mm_chunk6x6_stride3x3", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + dict(type="Copy", keys_dict={"segment": "origin_segment"}), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + keys=("coord", "color", "normal", "segment"), + return_inverse=True, + ), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-spunet-v1m1-1-ppt-extreme.py b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-spunet-v1m1-1-ppt-extreme.py new file mode 100644 index 0000000000000000000000000000000000000000..55ed0fe2be2b23404980d7c19633458f843506bd --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/scannetpp/semseg-spunet-v1m1-1-ppt-extreme.py @@ -0,0 +1,480 @@ +_base_ = [ + "../_base_/default_runtime.py", + "../_base_/dataset/scannetpp.py", +] + +# misc custom setting +batch_size = 24 # bs: total bs in all gpus +num_worker = 48 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +find_unused_parameters = True + +# trainer +train = dict( + type="MultiDatasetTrainer", +) + +# model settings +model = dict( + type="PPT-v1m2", + backbone=dict( + type="SpUNet-v1m3", + in_channels=6, + num_classes=0, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("ScanNet", "ScanNet++", "S3DIS", "Structured3D"), + zero_init=False, + norm_decouple=True, + norm_adaptive=True, + norm_affine=True, + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + backbone_out_channels=96, + context_channels=256, + conditions=("ScanNet", "ScanNet++", "S3DIS", "Structured3D"), + num_classes=(200, 100, 13, 25), +) + +# scheduler settings +epoch = 100 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +data = dict( + num_classes=100, + ignore_index=-1, + train=dict( + type="ConcatDataset", + datasets=[ + # Structured3D + dict( + type="Structured3DDataset", + split=["train", "val", "test"], + data_root="data/structured3d", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "Structured3D"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=2, # sampling weight + ), + # ScanNet + dict( + type="ScanNet200Dataset", + split=["train", "val"], + data_root="data/scannet", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "ScanNet"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + loop=1, # sampling weight + ), + # S3DIS + # dict( + # type="S3DISDataset", + # split=("Area_1", "Area_2", "Area_3", "Area_4", "Area_6"), + # data_root="data/s3dis", + # transform=[ + # dict(type="CenterShift", apply_z=True), + # dict( + # type="RandomDropout", + # dropout_ratio=0.2, + # dropout_application_ratio=0.2, + # ), + # # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + # dict( + # type="RandomRotate", + # angle=[-1, 1], + # axis="z", + # center=[0, 0, 0], + # p=0.5, + # ), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + # dict(type="RandomScale", scale=[0.9, 1.1]), + # # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + # dict(type="RandomFlip", p=0.5), + # dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict( + # type="ElasticDistortion", + # distortion_params=[[0.2, 0.4], [0.8, 1.6]], + # ), + # dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + # dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + # dict(type="ChromaticJitter", p=0.95, std=0.05), + # # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + # dict( + # type="GridSample", + # grid_size=0.02, + # hash_type="fnv", + # mode="train", + # return_grid_coord=True, + # ), + # dict(type="SphereCrop", sample_rate=0.6, mode="random"), + # dict(type="SphereCrop", point_max=204800, mode="random"), + # dict(type="CenterShift", apply_z=False), + # dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + # dict(type="Add", keys_dict={"condition": "S3DIS"}), + # dict(type="ToTensor"), + # dict( + # type="Collect", + # keys=("coord", "grid_coord", "segment", "condition"), + # feat_keys=("color", "normal"), + # ), + # ], + # test_mode=False, + # loop=1, # sampling weight + # ), + dict( + type="ScanNetPPDataset", + split="train_grid1mm_chunk6x6_stride3x3", + data_root="data/scannetpp", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", + dropout_ratio=0.2, + dropout_application_ratio=0.2, + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict( + type="ElasticDistortion", + distortion_params=[[0.2, 0.4], [0.8, 1.6]], + ), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", point_max=204800, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + # dict(type="ShufflePoint"), + dict(type="Add", keys_dict={"condition": "ScanNet++"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + ], + ), + val=dict( + type="ScanNetPPDataset", + split="val", + data_root="data/scannetpp", + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict(type="Add", keys_dict={"condition": "ScanNet++"}), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type="ScanNetPPDataset", + split="val", + data_root="data/scannetpp", + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + dict(type="Copy", keys_dict={"segment": "origin_segment"}), + dict( + type="GridSample", + grid_size=0.01, + hash_type="fnv", + mode="train", + keys=("coord", "color", "normal", "segment"), + return_inverse=True, + ), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + keys=("coord", "color", "normal"), + return_grid_coord=True, + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "ScanNet++"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-minkunet34c-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-minkunet34c-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..6b22906b26f70255b437c1f8e50e71e24dde3f6d --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-minkunet34c-0-base.py @@ -0,0 +1,213 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 8 # bs: total bs in all gpus +mix_prob = 0 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict(type="MinkUNet34C", in_channels=4, out_channels=19), + criteria=[ + dict( + type="CrossEntropyLoss", + weight=[ + 3.1557, + 8.7029, + 7.8281, + 6.1354, + 6.3161, + 7.9937, + 8.9704, + 10.1922, + 1.6155, + 4.2187, + 1.9385, + 5.5455, + 2.0198, + 2.6261, + 1.3212, + 5.1102, + 2.5492, + 5.8585, + 7.3929, + ], + loss_weight=1.0, + ignore_index=-1, + ) + ], +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) + +# dataset settings +dataset_type = "SemanticKITTIDataset" +data_root = "data/semantic_kitti" +ignore_index = -1 +names = [ + "car", + "bicycle", + "motorcycle", + "truck", + "other-vehicle", + "person", + "bicyclist", + "motorcyclist", + "road", + "parking", + "sidewalk", + "other-ground", + "building", + "fence", + "vegetation", + "trunk", + "terrain", + "pole", + "traffic-sign", +] + +data = dict( + num_classes=19, + ignore_index=ignore_index, + names=names, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ignore_index=ignore_index, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-ppt-v1m1-0-sk-nu-wa-spunet.py b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-ppt-v1m1-0-sk-nu-wa-spunet.py new file mode 100644 index 0000000000000000000000000000000000000000..2ea85111fdc0eb44096367ce5c50df3c2683da68 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-ppt-v1m1-0-sk-nu-wa-spunet.py @@ -0,0 +1,351 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +find_unused_parameters = True + +# trainer +train = dict( + type="MultiDatasetTrainer", +) + +# model settings +model = dict( + type="PPT-v1m1", + backbone=dict( + type="SpUNet-v1m3", + in_channels=4, + num_classes=0, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("SemanticKITTI", "nuScenes", "Waymo"), + zero_init=False, + norm_decouple=True, + norm_adaptive=False, + norm_affine=True, + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + backbone_out_channels=96, + context_channels=256, + conditions=("SemanticKITTI", "nuScenes", "Waymo"), + template="[x]", + clip_model="ViT-B/16", + # fmt: off + class_name=( + # SemanticKITTI + "car", "bicycle", "motorcycle", "truck", "other vehicle", + "person", "person who rides a bicycle", "person who rides a motorcycle", "road", "parking", + "path for pedestrians at the side of a road", "other ground", "building", "fence", "vegetation", + "trunk", "terrain", "pole", "traffic sign", + # nuScenes + "barrier", "bicycle", "bus", "car", "construction vehicle", + "motorcycle", "pedestrian", "traffic cone", "trailer", "truck", + "path suitable or safe for driving", "other flat", "sidewalk", "terrain", "man made", "vegetation", + # waymo + "car", "truck", "bus", "other vehicle", "person who rides a motorcycle", + "person who rides a bicycle", "pedestrian", "sign", "traffic light", "pole", + "construction cone", "bicycle", "motorcycle", "building", "vegetation", + "tree trunk", "curb", "road", "lane marker", "other ground", "horizontal surface that can not drive", + "surface when pedestrians most likely to walk on", + ), + valid_index=( + [i for i in range(19)], + [i for i in range(19, 19 + 16)], + [i for i in range(19 + 16, 19 + 16 + 22)], + ), + # fmt: on + backbone_mode=False, +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) +# param_dicts = [dict(keyword="modulation", lr=0.0002)] + +# dataset settings +data = dict( + num_classes=19, + ignore_index=-1, + names=[ + "car", + "bicycle", + "motorcycle", + "truck", + "other-vehicle", + "person", + "bicyclist", + "motorcyclist", + "road", + "parking", + "sidewalk", + "other-ground", + "building", + "fence", + "vegetation", + "trunk", + "terrain", + "pole", + "traffic-sign", + ], + train=dict( + type="ConcatDataset", + datasets=[ + # nuScenes + dict( + type="NuScenesDataset", + split="train", + data_root="data/nuscenes", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='x', p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='y', p=0.5), + dict( + type="PointClip", + point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "nuScenes"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + # SemanticKITTI + dict( + type="SemanticKITTIDataset", + split="train", + data_root="data/semantic_kitti", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict( + type="PointClip", + point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "SemanticKITTI"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + # Waymo + dict( + type="WaymoDataset", + split="training", + data_root="data/waymo", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict( + type="PointClip", + point_cloud_range=(-75.2, -75.2, -4, 75.2, 75.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "Waymo"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + ], + ), + val=dict( + type="SemanticKITTIDataset", + split="val", + data_root="data/semantic_kitti", + transform=[ + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + dict(type="Add", keys_dict={"condition": "SemanticKITTI"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + ), + test=dict( + type="SemanticKITTIDataset", + split="val", + data_root="data/semantic_kitti", + transform=[ + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict(type="Add", keys_dict={"condition": "SemanticKITTI"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ignore_index=-1, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-ppt-v1m2-0-sk-nu-wa-spunet-submit.py b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-ppt-v1m2-0-sk-nu-wa-spunet-submit.py new file mode 100644 index 0000000000000000000000000000000000000000..1f1c21cb9e0a05067de2862d918ce39490cfe4c2 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-ppt-v1m2-0-sk-nu-wa-spunet-submit.py @@ -0,0 +1,301 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +find_unused_parameters = True +evaluate = False + +# trainer +train = dict( + type="MultiDatasetTrainer", +) + +# model settings +model = dict( + type="PPT-v1m2", + backbone=dict( + type="SpUNet-v1m3", + in_channels=4, + num_classes=0, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("SemanticKITTI", "nuScenes", "Waymo"), + zero_init=False, + norm_decouple=True, + norm_adaptive=False, + norm_affine=True, + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + backbone_out_channels=96, + context_channels=256, + conditions=("SemanticKITTI", "nuScenes", "Waymo"), + num_classes=(19, 16, 22), +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) +# param_dicts = [dict(keyword="modulation", lr=0.0002)] + +# dataset settings +data = dict( + num_classes=19, + ignore_index=-1, + names=[ + "car", + "bicycle", + "motorcycle", + "truck", + "other-vehicle", + "person", + "bicyclist", + "motorcyclist", + "road", + "parking", + "sidewalk", + "other-ground", + "building", + "fence", + "vegetation", + "trunk", + "terrain", + "pole", + "traffic-sign", + ], + train=dict( + type="ConcatDataset", + datasets=[ + # nuScenes + dict( + type="NuScenesDataset", + split=["train", "val"], + data_root="data/nuscenes", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='x', p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='y', p=0.5), + dict( + type="PointClip", + point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "nuScenes"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + # SemanticKITTI + dict( + type="SemanticKITTIDataset", + split=["train", "val"], + data_root="data/semantic_kitti", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict( + type="PointClip", + point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "SemanticKITTI"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + # Waymo + dict( + type="WaymoDataset", + split=["training", "validation"], + data_root="data/waymo", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict( + type="PointClip", + point_cloud_range=(-75.2, -75.2, -4, 75.2, 75.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "Waymo"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + ], + ), + test=dict( + type="SemanticKITTIDataset", + split="test", + data_root="data/semantic_kitti", + transform=[ + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict(type="Add", keys_dict={"condition": "SemanticKITTI"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ignore_index=-1, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-ppt-v1m2-0-sk-nu-wa-spunet.py b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-ppt-v1m2-0-sk-nu-wa-spunet.py new file mode 100644 index 0000000000000000000000000000000000000000..eb5cd428b7e6856a130e7bc30c1c5e9ea7c58428 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-ppt-v1m2-0-sk-nu-wa-spunet.py @@ -0,0 +1,325 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +num_worker = 24 +mix_prob = 0.8 +empty_cache = False +enable_amp = True +find_unused_parameters = True + +# trainer +train = dict( + type="MultiDatasetTrainer", +) + +# model settings +model = dict( + type="PPT-v1m2", + backbone=dict( + type="SpUNet-v1m3", + in_channels=4, + num_classes=0, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("SemanticKITTI", "nuScenes", "Waymo"), + zero_init=False, + norm_decouple=True, + norm_adaptive=False, + norm_affine=True, + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + backbone_out_channels=96, + context_channels=256, + conditions=("SemanticKITTI", "nuScenes", "Waymo"), + num_classes=(19, 16, 22), +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) +# param_dicts = [dict(keyword="modulation", lr=0.0002)] + +# dataset settings +data = dict( + num_classes=19, + ignore_index=-1, + names=[ + "car", + "bicycle", + "motorcycle", + "truck", + "other-vehicle", + "person", + "bicyclist", + "motorcyclist", + "road", + "parking", + "sidewalk", + "other-ground", + "building", + "fence", + "vegetation", + "trunk", + "terrain", + "pole", + "traffic-sign", + ], + train=dict( + type="ConcatDataset", + datasets=[ + # nuScenes + dict( + type="NuScenesDataset", + split="train", + data_root="data/nuscenes", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis='z', p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='x', p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis='y', p=0.5), + dict( + type="PointClip", + point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "nuScenes"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + # SemanticKITTI + dict( + type="SemanticKITTIDataset", + split="train", + data_root="data/semantic_kitti", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict( + type="PointClip", + point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "SemanticKITTI"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + # Waymo + dict( + type="WaymoDataset", + split="training", + data_root="data/waymo", + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict( + type="RandomRotate", + angle=[-1, 1], + axis="z", + center=[0, 0, 0], + p=0.5, + ), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict( + type="PointClip", + point_cloud_range=(-75.2, -75.2, -4, 75.2, 75.2, 2), + ), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="Add", keys_dict={"condition": "Waymo"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + loop=1, + ), + ], + ), + val=dict( + type="SemanticKITTIDataset", + split="val", + data_root="data/semantic_kitti", + transform=[ + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + dict(type="Add", keys_dict={"condition": "SemanticKITTI"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment", "condition"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=-1, + ), + test=dict( + type="SemanticKITTIDataset", + split="val", + data_root="data/semantic_kitti", + transform=[ + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict(type="Add", keys_dict={"condition": "SemanticKITTI"}), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index", "condition"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ignore_index=-1, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-pt-v2m2-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-pt-v2m2-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..7d1670a65c4e06b06caeef456bd0626310acbc40 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-pt-v2m2-0-base.py @@ -0,0 +1,222 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 8 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=4, + num_classes=19, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.15, 0.375, 0.9375, 2.34375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + # fmt: off + criteria=[ + dict(type="CrossEntropyLoss", + weight=[3.1557, 8.7029, 7.8281, 6.1354, 6.3161, 7.9937, 8.9704, 10.1922, 1.6155, 4.2187, + 1.9385, 5.5455, 2.0198, 2.6261, 1.3212, 5.1102, 2.5492, 5.8585, 7.3929], + loss_weight=1.0, + ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], + # fmt: on +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) + +# dataset settings +dataset_type = "SemanticKITTIDataset" +data_root = "data/semantic_kitti" +ignore_index = -1 +names = [ + "car", + "bicycle", + "motorcycle", + "truck", + "other-vehicle", + "person", + "bicyclist", + "motorcyclist", + "road", + "parking", + "sidewalk", + "other-ground", + "building", + "fence", + "vegetation", + "trunk", + "terrain", + "pole", + "traffic-sign", +] + +data = dict( + num_classes=19, + ignore_index=ignore_index, + names=names, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="SphereCrop", point_max=120000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict( + type="PointClip", + point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2), + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ignore_index=ignore_index, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-pt-v2m2-1-benchmark-submit.py b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-pt-v2m2-1-benchmark-submit.py new file mode 100644 index 0000000000000000000000000000000000000000..d65cda64e4ee3f485c11ada2c1eeccb30ca9e8ef --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-pt-v2m2-1-benchmark-submit.py @@ -0,0 +1,218 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 8 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True +evaluate = False + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=4, + num_classes=19, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.15, 0.375, 0.9375, 2.34375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[ + dict( + type="CrossEntropyLoss", + weight=[ + 3.1557, + 8.7029, + 7.8281, + 6.1354, + 6.3161, + 7.9937, + 8.9704, + 10.1922, + 1.6155, + 4.2187, + 1.9385, + 5.5455, + 2.0198, + 2.6261, + 1.3212, + 5.1102, + 2.5492, + 5.8585, + 7.3929, + ], + loss_weight=1.0, + ignore_index=-1, + ), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) + +# dataset settings +dataset_type = "SemanticKITTIDataset" +data_root = "data/semantic_kitti" +ignore_index = -1 +names = [ + "car", + "bicycle", + "motorcycle", + "truck", + "other-vehicle", + "person", + "bicyclist", + "motorcyclist", + "road", + "parking", + "sidewalk", + "other-ground", + "building", + "fence", + "vegetation", + "trunk", + "terrain", + "pole", + "traffic-sign", +] + +data = dict( + num_classes=19, + ignore_index=ignore_index, + names=names, + train=dict( + type=dataset_type, + split=["train", "val"], + data_root=data_root, + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="SphereCrop", point_max=120000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + test=dict( + type=dataset_type, + split="test", + data_root=data_root, + transform=[], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict( + type="PointClip", + point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2), + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ignore_index=ignore_index, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-spunet-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-spunet-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..7be9cb0979f431a137768849c0328ada0601fbd7 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-spunet-v1m1-0-base.py @@ -0,0 +1,219 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=4, + num_classes=19, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[ + dict( + type="CrossEntropyLoss", + weight=[ + 3.1557, + 8.7029, + 7.8281, + 6.1354, + 6.3161, + 7.9937, + 8.9704, + 10.1922, + 1.6155, + 4.2187, + 1.9385, + 5.5455, + 2.0198, + 2.6261, + 1.3212, + 5.1102, + 2.5492, + 5.8585, + 7.3929, + ], + loss_weight=1.0, + ignore_index=-1, + ) + ], +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) + +# dataset settings +dataset_type = "SemanticKITTIDataset" +data_root = "data/semantic_kitti" +ignore_index = -1 +names = [ + "car", + "bicycle", + "motorcycle", + "truck", + "other-vehicle", + "person", + "bicyclist", + "motorcyclist", + "road", + "parking", + "sidewalk", + "other-ground", + "building", + "fence", + "vegetation", + "trunk", + "terrain", + "pole", + "traffic-sign", +] + +data = dict( + num_classes=19, + ignore_index=ignore_index, + names=names, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ignore_index=ignore_index, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-spvcnn-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-spvcnn-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..599ff4d538e1e2bc706abc9c00a41dc71cd5e007 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/semantic_kitti/semseg-spvcnn-v1m1-0-base.py @@ -0,0 +1,219 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 8 # bs: total bs in all gpus +mix_prob = 0 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SPVCNN", + in_channels=4, + out_channels=19, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 2, 2, 2, 2, 2, 2, 2), + ), + criteria=[ + dict( + type="CrossEntropyLoss", + weight=[ + 3.1557, + 8.7029, + 7.8281, + 6.1354, + 6.3161, + 7.9937, + 8.9704, + 10.1922, + 1.6155, + 4.2187, + 1.9385, + 5.5455, + 2.0198, + 2.6261, + 1.3212, + 5.1102, + 2.5492, + 5.8585, + 7.3929, + ], + loss_weight=1.0, + ignore_index=-1, + ) + ], +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) + +# dataset settings +dataset_type = "SemanticKITTIDataset" +data_root = "data/semantic_kitti" +ignore_index = -1 +names = [ + "car", + "bicycle", + "motorcycle", + "truck", + "other-vehicle", + "person", + "bicyclist", + "motorcyclist", + "road", + "parking", + "sidewalk", + "other-ground", + "building", + "fence", + "vegetation", + "trunk", + "terrain", + "pole", + "traffic-sign", +] + +data = dict( + num_classes=19, + ignore_index=ignore_index, + names=names, + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="PointClip", point_cloud_range=(-35.2, -35.2, -4, 35.2, 35.2, 2)), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ignore_index=ignore_index, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/structured3d/semseg-pt-v2m2-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/structured3d/semseg-pt-v2m2-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..5c67cd6a103d6b3176482c6710938d8e23f2feea --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/structured3d/semseg-pt-v2m2-0-base.py @@ -0,0 +1,304 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="PT-v2m2", + in_channels=9, + num_classes=25, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.15, 0.375, 0.9375), # x3, x2.5, x2.5, x2.5 + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.3, + enable_checkpoint=False, + unpool_backend="map", # map / interp + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 100 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "Structured3DDataset" +data_root = "data/structured3d" + +data = dict( + num_classes=25, + ignore_index=-1, + names=( + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "picture", + "desk", + "shelves", + "curtain", + "dresser", + "pillow", + "mirror", + "ceiling", + "refrigerator", + "television", + "nightstand", + "sink", + "lamp", + "otherstructure", + "otherfurniture", + "otherprop", + ), + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="SphereCrop", point_max=120000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/structured3d/semseg-spunet-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/structured3d/semseg-spunet-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..1a2fa48320601baed972856a20afcba21991df0c --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/structured3d/semseg-spunet-v1m1-0-base.py @@ -0,0 +1,285 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=6, + num_classes=25, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 100 +optimizer = dict(type="SGD", lr=0.05, momentum=0.9, weight_decay=0.0001, nesterov=True) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=10000.0, +) + +# dataset settings +dataset_type = "Structured3DDataset" +data_root = "data/structured3d" + +data = dict( + num_classes=25, + ignore_index=-1, + names=( + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "picture", + "desk", + "shelves", + "curtain", + "dresser", + "pillow", + "mirror", + "ceiling", + "refrigerator", + "television", + "nightstand", + "sink", + "lamp", + "otherstructure", + "otherfurniture", + "otherprop", + ), + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + dict(type="SphereCrop", sample_rate=0.6, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/structured3d/semseg-swin3d-v1m1-0-small.py b/submodules/PointTransformerV3/Pointcept/configs/structured3d/semseg-swin3d-v1m1-0-small.py new file mode 100644 index 0000000000000000000000000000000000000000..e52bb1ee7a4da43ca523c1979129ec2fa4c5ecbe --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/structured3d/semseg-swin3d-v1m1-0-small.py @@ -0,0 +1,306 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="Swin3D-v1m1", + in_channels=9, + num_classes=25, + base_grid_size=0.02, + depths=[2, 4, 9, 4, 4], + channels=[48, 96, 192, 384, 384], + num_heads=[6, 6, 12, 24, 24], + window_sizes=[5, 7, 7, 7, 7], + quant_size=4, + drop_path_rate=0.3, + up_k=3, + num_layers=5, + stem_transformer=True, + down_stride=3, + upsample="linear_attn", + knn_down=True, + cRSE="XYZ_RGB_NORM", + fp16_mode=1, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 100 +optimizer = dict(type="AdamW", lr=0.008, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.008, 0.0008], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="blocks", lr=0.0008)] + +# dataset settings +dataset_type = "Structured3DDataset" +data_root = "data/structured3d" + +data = dict( + num_classes=25, + ignore_index=-1, + names=( + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "picture", + "desk", + "shelves", + "curtain", + "dresser", + "pillow", + "mirror", + "ceiling", + "refrigerator", + "television", + "nightstand", + "sink", + "lamp", + "otherstructure", + "otherfurniture", + "otherprop", + ), + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + return_displacement=True, + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="SphereCrop", point_max=120000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + return_displacement=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + return_displacement=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/structured3d/semseg-swin3d-v1m1-1-large.py b/submodules/PointTransformerV3/Pointcept/configs/structured3d/semseg-swin3d-v1m1-1-large.py new file mode 100644 index 0000000000000000000000000000000000000000..de62b4234473f34648201507a6b7d37d11674df6 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/structured3d/semseg-swin3d-v1m1-1-large.py @@ -0,0 +1,306 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="Swin3D-v1m1", + in_channels=9, + num_classes=25, + base_grid_size=0.02, + depths=[2, 4, 9, 4, 4], + channels=[80, 160, 320, 640, 640], + num_heads=[10, 10, 20, 40, 40], + window_sizes=[5, 7, 7, 7, 7], + quant_size=4, + drop_path_rate=0.3, + up_k=3, + num_layers=5, + stem_transformer=True, + down_stride=3, + upsample="linear_attn", + knn_down=True, + cRSE="XYZ_RGB_NORM", + fp16_mode=1, + ), + criteria=[dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1)], +) + +# scheduler settings +epoch = 100 +optimizer = dict(type="AdamW", lr=0.008, weight_decay=0.05) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.008, 0.0008], + pct_start=0.05, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=1000.0, +) +param_dicts = [dict(keyword="blocks", lr=0.0008)] + +# dataset settings +dataset_type = "Structured3DDataset" +data_root = "data/structured3d" + +data = dict( + num_classes=25, + ignore_index=-1, + names=( + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "picture", + "desk", + "shelves", + "curtain", + "dresser", + "pillow", + "mirror", + "ceiling", + "refrigerator", + "television", + "nightstand", + "sink", + "lamp", + "otherstructure", + "otherfurniture", + "otherprop", + ), + train=dict( + type=dataset_type, + split="train", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2 + ), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="x", p=0.5), + dict(type="RandomRotate", angle=[-1 / 64, 1 / 64], axis="y", p=0.5), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict(type="ChromaticAutoContrast", p=0.2, blend_factor=None), + dict(type="ChromaticTranslation", p=0.95, ratio=0.05), + dict(type="ChromaticJitter", p=0.95, std=0.05), + # dict(type="HueSaturationTranslation", hue_max=0.2, saturation_max=0.2), + # dict(type="RandomColorDrop", p=0.2, color_augment=0.0), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + return_displacement=True, + ), + dict(type="SphereCrop", sample_rate=0.8, mode="random"), + dict(type="SphereCrop", point_max=120000, mode="random"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ShufflePoint"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + val=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="train", + return_grid_coord=True, + return_displacement=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="center"), + dict(type="CenterShift", apply_z=False), + dict(type="NormalizeColor"), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + test_mode=False, + ), + test=dict( + type=dataset_type, + split="val", + data_root=data_root, + transform=[ + dict(type="CenterShift", apply_z=True), + dict(type="NormalizeColor"), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.02, + hash_type="fnv", + mode="test", + return_grid_coord=True, + return_displacement=True, + keys=("coord", "color", "normal"), + ), + crop=None, + post_transform=[ + dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("color", "normal", "displacement"), + coord_feat_keys=("color", "normal"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[0.95, 0.95]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ), + dict(type="RandomScale", scale=[1.05, 1.05]), + ], + [dict(type="RandomFlip", p=1)], + ], + ), + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/waymo/semseg-pt-v3m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/waymo/semseg-pt-v3m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..89cb3bc588ce2ec662d76c81dad3fbcea8373f8a --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/waymo/semseg-pt-v3m1-0-base.py @@ -0,0 +1,248 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentorV2", + num_classes=22, + backbone_out_channels=64, + backbone=dict( + type="PT-v3m1", + in_channels=4, + order=["z", "z-trans", "hilbert", "hilbert-trans"], + stride=(2, 2, 2, 2), + enc_depths=(2, 2, 2, 6, 2), + enc_channels=(32, 64, 128, 256, 512), + enc_num_head=(2, 4, 8, 16, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(2, 2, 2, 2), + dec_channels=(64, 64, 128, 256), + dec_num_head=(4, 4, 8, 16), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + shuffle_orders=True, + pre_norm=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=False, + pdnorm_ln=False, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("nuScenes", "SemanticKITTI", "Waymo"), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=[0.002, 0.0002], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) +param_dicts = [dict(keyword="block", lr=0.0002)] + +# dataset settings +dataset_type = "WaymoDataset" +data_root = "data/waymo" +ignore_index = -1 +names = [ + "Car", + "Truck", + "Bus", + # Other small vehicles (e.g. pedicab) and large vehicles (e.g. construction vehicles, RV, limo, tram). + "Other Vehicle", + "Motorcyclist", + "Bicyclist", + "Pedestrian", + "Sign", + "Traffic Light", + # Lamp post, traffic sign pole etc. + "Pole", + # Construction cone/pole. + "Construction Cone", + "Bicycle", + "Motorcycle", + "Building", + # Bushes, tree branches, tall grasses, flowers etc. + "Vegetation", + "Tree Trunk", + # Curb on the edge of roads. This does not include road boundaries if there’s no curb. + "Curb", + # Surface a vehicle could drive on. This includes the driveway connecting + # parking lot and road over a section of sidewalk. + "Road", + # Marking on the road that’s specifically for defining lanes such as + # single/double white/yellow lines. + "Lane Marker", + # Marking on the road other than lane markers, bumps, cateyes, railtracks etc. + "Other Ground", + # Most horizontal surface that’s not drivable, e.g. grassy hill, pedestrian walkway stairs etc. + "Walkable", + # Nicely paved walkable surface when pedestrians most likely to walk on. + "Sidewalk", +] + +data = dict( + num_classes=22, + ignore_index=ignore_index, + names=names, + train=dict( + type=dataset_type, + split="training", + data_root=data_root, + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict(type="PointClip", point_cloud_range=(-75.2, -75.2, -4, 75.2, 75.2, 2)), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + val=dict( + type=dataset_type, + split="validation", + data_root=data_root, + transform=[ + dict(type="PointClip", point_cloud_range=(-75.2, -75.2, -4, 75.2, 75.2, 2)), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + test=dict( + type=dataset_type, + split="validation", + data_root=data_root, + transform=[ + dict(type="PointClip", point_cloud_range=(-75.2, -75.2, -4, 75.2, 75.2, 2)), + dict(type="Copy", keys_dict={"segment": "origin_segment"}), + dict( + type="GridSample", + grid_size=0.025, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_inverse=True, + ), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ignore_index=ignore_index, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/configs/waymo/semseg-spunet-v1m1-0-base.py b/submodules/PointTransformerV3/Pointcept/configs/waymo/semseg-spunet-v1m1-0-base.py new file mode 100644 index 0000000000000000000000000000000000000000..67d8011d2ff4df45757749facdb44d2f1c175b10 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/configs/waymo/semseg-spunet-v1m1-0-base.py @@ -0,0 +1,210 @@ +_base_ = ["../_base_/default_runtime.py"] + +# misc custom setting +batch_size = 12 # bs: total bs in all gpus +mix_prob = 0.8 +empty_cache = False +enable_amp = True + +# model settings +model = dict( + type="DefaultSegmentor", + backbone=dict( + type="SpUNet-v1m1", + in_channels=4, + num_classes=22, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ), + criteria=[ + dict(type="CrossEntropyLoss", loss_weight=1.0, ignore_index=-1), + dict(type="LovaszLoss", mode="multiclass", loss_weight=1.0, ignore_index=-1), + ], +) + +# scheduler settings +epoch = 50 +eval_epoch = 50 +optimizer = dict(type="AdamW", lr=0.002, weight_decay=0.005) +scheduler = dict( + type="OneCycleLR", + max_lr=optimizer["lr"], + pct_start=0.04, + anneal_strategy="cos", + div_factor=10.0, + final_div_factor=100.0, +) + +# dataset settings +dataset_type = "WaymoDataset" +data_root = "data/waymo" +ignore_index = -1 +names = [ + "Car", + "Truck", + "Bus", + # Other small vehicles (e.g. pedicab) and large vehicles (e.g. construction vehicles, RV, limo, tram). + "Other Vehicle", + "Motorcyclist", + "Bicyclist", + "Pedestrian", + "Sign", + "Traffic Light", + # Lamp post, traffic sign pole etc. + "Pole", + # Construction cone/pole. + "Construction Cone", + "Bicycle", + "Motorcycle", + "Building", + # Bushes, tree branches, tall grasses, flowers etc. + "Vegetation", + "Tree Trunk", + # Curb on the edge of roads. This does not include road boundaries if there’s no curb. + "Curb", + # Surface a vehicle could drive on. This includes the driveway connecting + # parking lot and road over a section of sidewalk. + "Road", + # Marking on the road that’s specifically for defining lanes such as + # single/double white/yellow lines. + "Lane Marker", + # Marking on the road other than lane markers, bumps, cateyes, railtracks etc. + "Other Ground", + # Most horizontal surface that’s not drivable, e.g. grassy hill, pedestrian walkway stairs etc. + "Walkable", + # Nicely paved walkable surface when pedestrians most likely to walk on. + "Sidewalk", +] + +data = dict( + num_classes=22, + ignore_index=ignore_index, + names=names, + train=dict( + type=dataset_type, + split="training", + data_root=data_root, + transform=[ + # dict(type="RandomDropout", dropout_ratio=0.2, dropout_application_ratio=0.2), + # dict(type="RandomRotateTargetAngle", angle=(1/2, 1, 3/2), center=[0, 0, 0], axis="z", p=0.75), + dict(type="RandomRotate", angle=[-1, 1], axis="z", center=[0, 0, 0], p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="x", p=0.5), + # dict(type="RandomRotate", angle=[-1/6, 1/6], axis="y", p=0.5), + dict(type="PointClip", point_cloud_range=(-75.2, -75.2, -4, 75.2, 75.2, 2)), + dict(type="RandomScale", scale=[0.9, 1.1]), + # dict(type="RandomShift", shift=[0.2, 0.2, 0.2]), + dict(type="RandomFlip", p=0.5), + dict(type="RandomJitter", sigma=0.005, clip=0.02), + # dict(type="ElasticDistortion", distortion_params=[[0.2, 0.4], [0.8, 1.6]]), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + # dict(type="SphereCrop", point_max=1000000, mode="random"), + # dict(type="CenterShift", apply_z=False), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + val=dict( + type=dataset_type, + split="validation", + data_root=data_root, + transform=[ + dict(type="PointClip", point_cloud_range=(-75.2, -75.2, -4, 75.2, 75.2, 2)), + dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "strength", "segment"), + return_grid_coord=True, + ), + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "segment"), + feat_keys=("coord", "strength"), + ), + ], + test_mode=False, + ignore_index=ignore_index, + ), + test=dict( + type=dataset_type, + split="validation", + data_root=data_root, + transform=[ + dict(type="PointClip", point_cloud_range=(-75.2, -75.2, -4, 75.2, 75.2, 2)), + ], + test_mode=True, + test_cfg=dict( + voxelize=dict( + type="GridSample", + grid_size=0.05, + hash_type="fnv", + mode="test", + return_grid_coord=True, + keys=("coord", "strength"), + ), + crop=None, + post_transform=[ + dict(type="ToTensor"), + dict( + type="Collect", + keys=("coord", "grid_coord", "index"), + feat_keys=("coord", "strength"), + ), + ], + aug_transform=[ + [ + dict( + type="RandomRotateTargetAngle", + angle=[0], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[1], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + [ + dict( + type="RandomRotateTargetAngle", + angle=[3 / 2], + axis="z", + center=[0, 0, 0], + p=1, + ) + ], + ], + ), + ignore_index=ignore_index, + ), +) diff --git a/submodules/PointTransformerV3/Pointcept/docs/logo.png b/submodules/PointTransformerV3/Pointcept/docs/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..cdd04d6aa6cc7a31e29e3970977427e7edde2c17 Binary files /dev/null and b/submodules/PointTransformerV3/Pointcept/docs/logo.png differ diff --git a/submodules/PointTransformerV3/Pointcept/docs/logo_dark.png b/submodules/PointTransformerV3/Pointcept/docs/logo_dark.png new file mode 100644 index 0000000000000000000000000000000000000000..32af83bd9e42d65251637fb4a90a9e3a4d2e0f83 Binary files /dev/null and b/submodules/PointTransformerV3/Pointcept/docs/logo_dark.png differ diff --git a/submodules/PointTransformerV3/Pointcept/docs/offset.png b/submodules/PointTransformerV3/Pointcept/docs/offset.png new file mode 100644 index 0000000000000000000000000000000000000000..e66df4f29321312d700f2bb7960e502fe233c5d6 Binary files /dev/null and b/submodules/PointTransformerV3/Pointcept/docs/offset.png differ diff --git a/submodules/PointTransformerV3/Pointcept/docs/offset_dark.png b/submodules/PointTransformerV3/Pointcept/docs/offset_dark.png new file mode 100755 index 0000000000000000000000000000000000000000..4db79e0d94e7669b6653c0e9abad96ec1cc86364 Binary files /dev/null and b/submodules/PointTransformerV3/Pointcept/docs/offset_dark.png differ diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/functions/__init__.py b/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/functions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1d8120292a327a400e15fa9a72cd08b16a68794f --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/functions/__init__.py @@ -0,0 +1 @@ +from .functions import bfs_cluster, ballquery_batch_p, Clustering diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/functions/functions.py b/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/functions/functions.py new file mode 100644 index 0000000000000000000000000000000000000000..c8ed62b10c2e88236d814ab34cece5a335e16930 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/functions/functions.py @@ -0,0 +1,176 @@ +import torch +from torch.autograd import Function +import pointgroup_ops_cuda + + +class BallQueryBatchP(Function): + @staticmethod + def forward(ctx, coords, batch_idxs, batch_offsets, radius, meanActive): + """ + :param ctx: + :param coords: (n, 3) float + :param batch_idxs: (n) int + :param batch_offsets: (B+1) int + :param radius: float + :param meanActive: int + :return: idx (nActive), int + :return: start_len (n, 2), int + """ + + n = coords.size(0) + + assert coords.is_contiguous() and coords.is_cuda + assert batch_idxs.is_contiguous() and batch_idxs.is_cuda + assert batch_offsets.is_contiguous() and batch_offsets.is_cuda + + while True: + idx = torch.cuda.IntTensor(n * meanActive).zero_() + start_len = torch.cuda.IntTensor(n, 2).zero_() + nActive = pointgroup_ops_cuda.ballquery_batch_p( + coords, batch_idxs, batch_offsets, idx, start_len, n, meanActive, radius + ) + if nActive <= n * meanActive: + break + meanActive = int(nActive // n + 1) + idx = idx[:nActive] + + return idx, start_len + + @staticmethod + def backward(ctx, a=None, b=None): + return None, None, None + + +ballquery_batch_p = BallQueryBatchP.apply + + +class Clustering: + def __init__( + self, + ignored_labels, + class_mapping, + thresh=0.03, + closed_points=300, + min_points=50, + propose_points=100, + score_func=torch.max, + ) -> None: + self.ignored_labels = ignored_labels + self.thresh = thresh + self.closed_points = closed_points + self.min_points = min_points + self.class_mapping = class_mapping + self.propose_points = propose_points + self.score_func = score_func + + def cluster(self, vertices, scores): + labels = torch.max(scores, 1)[1] # (N) long, cuda + proposals_idx, proposals_offset = self.cluster_(vertices, labels) + + ## debug + # import ipdb; ipdb.set_trace() + # colors = np.array(create_color_palette())[labels.cpu()] + # write_triangle_mesh(vertices, colors, None, 'semantics.ply') + + # scatter + proposals_pred = torch.zeros( + (proposals_offset.shape[0] - 1, vertices.shape[0]), dtype=torch.int + ) # (nProposal, N), int, cuda + proposals_pred[proposals_idx[:, 0].long(), proposals_idx[:, 1].long()] = 1 + labels = labels[proposals_idx[:, 1][proposals_offset[:-1].long()].long()] + + proposals_pointnum = proposals_pred.sum(1) + npoint_mask = proposals_pointnum > self.propose_points + + proposals_pred = proposals_pred[npoint_mask] + labels = labels[npoint_mask] + return proposals_pred, labels + + def cluster_(self, vertices, labels): + """ + :param batch_idxs: (N), int, cuda + :labels: 0-19 + """ + batch_idxs = torch.zeros_like(labels) + + mask_non_ignored = torch.ones_like(labels).bool() + for ignored_label in self.ignored_labels: + mask_non_ignored = mask_non_ignored & ( + self.class_mapping[labels] != ignored_label + ) + object_idxs = mask_non_ignored.nonzero().view(-1) + + vertices_ = vertices[object_idxs].float() + labels_ = labels[object_idxs].int() + + if vertices_.numel() == 0: + return torch.zeros((0, 2)).int(), torch.zeros(1).int() + + batch_idxs_ = batch_idxs[object_idxs].int() + batch_offsets_ = torch.FloatTensor([0, object_idxs.shape[0]]).int().cuda() + + idx, start_len = ballquery_batch_p( + vertices_, batch_idxs_, batch_offsets_, self.thresh, self.closed_points + ) + proposals_idx, proposals_offset = bfs_cluster( + labels_.cpu(), idx.cpu(), start_len.cpu(), self.min_points + ) + proposals_idx[:, 1] = object_idxs[proposals_idx[:, 1].long()].int() + + return proposals_idx, proposals_offset + + def get_instances(self, vertices, scores): + proposals_pred, labels = self.cluster(vertices, scores) + instances = {} + for proposal_id in range(len(proposals_pred)): + clusters_i = proposals_pred[proposal_id] + score = scores[clusters_i.bool(), labels[proposal_id]] + score = self.score_func(score) + instances[proposal_id] = {} + instances[proposal_id]["conf"] = score.cpu().numpy() + instances[proposal_id]["label_id"] = self.class_mapping.cpu()[ + labels[proposal_id] + ] + instances[proposal_id]["pred_mask"] = clusters_i.cpu().numpy() + return instances + + +class BFSCluster(Function): + @staticmethod + def forward(ctx, semantic_label, ball_query_idxs, start_len, threshold): + """ + :param ctx: + :param semantic_label: (N), int + :param ball_query_idxs: (nActive), int + :param start_len: (N, 2), int + :return: cluster_idxs: int (sumNPoint, 2), dim 0 for cluster_id, dim 1 for corresponding point idxs in N + :return: cluster_offsets: int (nCluster + 1) + """ + + N = start_len.size(0) + + assert semantic_label.is_contiguous() + assert ball_query_idxs.is_contiguous() + assert start_len.is_contiguous() + + cluster_idxs = semantic_label.new() + cluster_offsets = semantic_label.new() + + pointgroup_ops_cuda.bfs_cluster( + semantic_label, + ball_query_idxs, + start_len, + cluster_idxs, + cluster_offsets, + N, + threshold, + ) + + return cluster_idxs, cluster_offsets + + @staticmethod + def backward(ctx, a=None): + return None + + +bfs_cluster = BFSCluster.apply diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/setup.py b/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..162b68258920afcd46c3b49764236beecfbf35cb --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/setup.py @@ -0,0 +1,59 @@ +import os +from sys import argv +from setuptools import setup +from torch.utils.cpp_extension import BuildExtension, CUDAExtension +from distutils.sysconfig import get_config_vars + +(opt,) = get_config_vars("OPT") +os.environ["OPT"] = " ".join( + flag for flag in opt.split() if flag != "-Wstrict-prototypes" +) + + +def _argparse(pattern, argv, is_flag=True, is_list=False): + if is_flag: + found = pattern in argv + if found: + argv.remove(pattern) + return found, argv + else: + arr = [arg for arg in argv if pattern == arg.split("=")[0]] + if is_list: + if len(arr) == 0: # not found + return False, argv + else: + assert "=" in arr[0], f"{arr[0]} requires a value." + argv.remove(arr[0]) + val = arr[0].split("=")[1] + if "," in val: + return val.split(","), argv + else: + return [val], argv + else: + if len(arr) == 0: # not found + return False, argv + else: + assert "=" in arr[0], f"{arr[0]} requires a value." + argv.remove(arr[0]) + return arr[0].split("=")[1], argv + + +INCLUDE_DIRS, argv = _argparse("--include_dirs", argv, False, is_list=True) +include_dirs = [] +if not (INCLUDE_DIRS is False): + include_dirs += INCLUDE_DIRS + +setup( + name="pointgroup_ops", + packages=["pointgroup_ops"], + package_dir={"pointgroup_ops": "functions"}, + ext_modules=[ + CUDAExtension( + name="pointgroup_ops_cuda", + sources=["src/bfs_cluster.cpp", "src/bfs_cluster_kernel.cu"], + extra_compile_args={"cxx": ["-g"], "nvcc": ["-O2"]}, + ) + ], + include_dirs=[*include_dirs], + cmdclass={"build_ext": BuildExtension}, +) diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/src/bfs_cluster.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/src/bfs_cluster.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d0298aae5bf1f184eb4e923d8f9f8893168c8e19 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/src/bfs_cluster.cpp @@ -0,0 +1,145 @@ +/* +Ball Query with BatchIdx & Clustering Algorithm +Written by Li Jiang +All Rights Reserved 2020. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int ballquery_batch_p_cuda(int n, int meanActive, float radius, const float *xyz, const int *batch_idxs, const int *batch_offsets, int *idx, int *start_len, cudaStream_t stream); + + +using Int = int32_t; +class ConnectedComponent{ +public: + std::vector pt_idxs {}; + + ConnectedComponent(){}; + void addPoint(Int pt_idx) + { + pt_idxs.push_back(pt_idx); + + } +}; +using ConnectedComponents = std::vector; + +/* ================================== ballquery_batch_p ================================== */ +// input xyz: (n, 3) float +// input batch_idxs: (n) int +// input batch_offsets: (B+1) int, batch_offsets[-1] +// output idx: (n * meanActive) dim 0 for number of points in the ball, idx in n +// output start_len: (n, 2), int +int ballquery_batch_p(at::Tensor xyz_tensor, at::Tensor batch_idxs_tensor, at::Tensor batch_offsets_tensor, at::Tensor idx_tensor, at::Tensor start_len_tensor, int n, int meanActive, float radius){ + const float *xyz = xyz_tensor.data(); + const int *batch_idxs = batch_idxs_tensor.data(); + const int *batch_offsets = batch_offsets_tensor.data(); + int *idx = idx_tensor.data(); + int *start_len = start_len_tensor.data(); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + int cumsum = ballquery_batch_p_cuda(n, meanActive, radius, xyz, batch_idxs, batch_offsets, idx, start_len, stream); + return cumsum; +} + +/* ================================== bfs_cluster ================================== */ +ConnectedComponent find_cc(Int idx, int *semantic_label, Int *ball_query_idxs, int *start_len, int *visited){ + ConnectedComponent cc; + cc.addPoint(idx); + visited[idx] = 1; + + std::queue Q; + assert(Q.empty()); + Q.push(idx); + + while(!Q.empty()){ + Int cur = Q.front(); Q.pop(); + int start = start_len[cur * 2]; + int len = start_len[cur * 2 + 1]; + int label_cur = semantic_label[cur]; + for(Int i = start; i < start + len; i++){ + Int idx_i = ball_query_idxs[i]; + if(semantic_label[idx_i] != label_cur) continue; + if(visited[idx_i] == 1) continue; + + cc.addPoint(idx_i); + visited[idx_i] = 1; + + Q.push(idx_i); + } + } + return cc; +} + +//input: semantic_label, int, N +//input: ball_query_idxs, Int, (nActive) +//input: start_len, int, (N, 2) +//output: clusters, CCs +int get_clusters(int *semantic_label, Int *ball_query_idxs, int *start_len, const Int nPoint, int threshold, ConnectedComponents &clusters){ + int visited[nPoint] = {0}; + + int sumNPoint = 0; + for(Int i = 0; i < nPoint; i++){ + if(visited[i] == 0){ + ConnectedComponent CC = find_cc(i, semantic_label, ball_query_idxs, start_len, visited); + if((int)CC.pt_idxs.size() >= threshold){ + clusters.push_back(CC); + sumNPoint += (int)CC.pt_idxs.size(); + } + } + } + + return sumNPoint; +} + +void fill_cluster_idxs_(ConnectedComponents &CCs, int *cluster_idxs, int *cluster_offsets){ + for(int i = 0; i < (int)CCs.size(); i++){ + cluster_offsets[i + 1] = cluster_offsets[i] + (int)CCs[i].pt_idxs.size(); + for(int j = 0; j < (int)CCs[i].pt_idxs.size(); j++){ + int idx = CCs[i].pt_idxs[j]; + cluster_idxs[(cluster_offsets[i] + j) * 2 + 0] = i; + cluster_idxs[(cluster_offsets[i] + j) * 2 + 1] = idx; + } + } +} + +//input: semantic_label, int, N +//input: ball_query_idxs, int, (nActive) +//input: start_len, int, (N, 2) +//output: cluster_idxs, int (sumNPoint, 2), dim 0 for cluster_id, dim 1 for corresponding point idxs in N +//output: cluster_offsets, int (nCluster + 1) +void bfs_cluster(at::Tensor semantic_label_tensor, at::Tensor ball_query_idxs_tensor, at::Tensor start_len_tensor, +at::Tensor cluster_idxs_tensor, at::Tensor cluster_offsets_tensor, const int N, int threshold){ + int *semantic_label = semantic_label_tensor.data(); + Int *ball_query_idxs = ball_query_idxs_tensor.data(); + int *start_len = start_len_tensor.data(); + + ConnectedComponents CCs; + int sumNPoint = get_clusters(semantic_label, ball_query_idxs, start_len, N, threshold, CCs); + + int nCluster = (int)CCs.size(); + cluster_idxs_tensor.resize_({sumNPoint, 2}); + cluster_offsets_tensor.resize_({nCluster + 1}); + cluster_idxs_tensor.zero_(); + cluster_offsets_tensor.zero_(); + + int *cluster_idxs = cluster_idxs_tensor.data(); + int *cluster_offsets = cluster_offsets_tensor.data(); + + fill_cluster_idxs_(CCs, cluster_idxs, cluster_offsets); +} + +//------------------------------------API------------------------------------------ +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m){ + + m.def("ballquery_batch_p", &ballquery_batch_p, "ballquery_batch_p"); + m.def("bfs_cluster", &bfs_cluster, "bfs_cluster"); + +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/src/bfs_cluster_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/src/bfs_cluster_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..99a31842d605588b214826223143f2669475f402 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointgroup_ops/src/bfs_cluster_kernel.cu @@ -0,0 +1,91 @@ +/* +Ball Query with BatchIdx +Written by Li Jiang +All Rights Reserved 2020. +*/ +#include +#include +#include + +#define TOTAL_THREADS 1024 +#define THREADS_PER_BLOCK 512 +#define DIVUP(m,n) ((m) / (n) + ((m) % (n) > 0)) + + +/* ================================== ballquery_batch_p ================================== */ +__global__ void ballquery_batch_p_cuda_(int n, int meanActive, float radius, const float *xyz, const int *batch_idxs, const int *batch_offsets, int *idx, int *start_len, int *cumsum) { + int pt_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (pt_idx >= n) return; + + start_len += (pt_idx * 2); + int idx_temp[1000]; + + float radius2 = radius * radius; + float o_x = xyz[pt_idx * 3 + 0]; + float o_y = xyz[pt_idx * 3 + 1]; + float o_z = xyz[pt_idx * 3 + 2]; + + int batch_idx = batch_idxs[pt_idx]; + int start = batch_offsets[batch_idx]; + int end = batch_offsets[batch_idx + 1]; + + int cnt = 0; + for(int k = start; k < end; k++){ + float x = xyz[k * 3 + 0]; + float y = xyz[k * 3 + 1]; + float z = xyz[k * 3 + 2]; + float d2 = (o_x - x) * (o_x - x) + (o_y - y) * (o_y - y) + (o_z - z) * (o_z - z); + if(d2 < radius2){ + if(cnt < 1000){ + idx_temp[cnt] = k; + } + else{ + break; + } + ++cnt; + } + } + + start_len[0] = atomicAdd(cumsum, cnt); + start_len[1] = cnt; + + int thre = n * meanActive; + if(start_len[0] >= thre) return; + + idx += start_len[0]; + if(start_len[0] + cnt >= thre) cnt = thre - start_len[0]; + + for(int k = 0; k < cnt; k++){ + idx[k] = idx_temp[k]; + } +} + + +int ballquery_batch_p_cuda(int n, int meanActive, float radius, const float *xyz, const int *batch_idxs, const int *batch_offsets, int *idx, int *start_len, cudaStream_t stream) { + // param xyz: (n, 3) + // param batch_idxs: (n) + // param batch_offsets: (B + 1) + // output idx: (n * meanActive) dim 0 for number of points in the ball, idx in n + // output start_len: (n, 2), int + + cudaError_t err; + + dim3 blocks(DIVUP(n, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + + int cumsum = 0; + int* p_cumsum; + cudaMalloc((void**)&p_cumsum, sizeof(int)); + cudaMemcpy(p_cumsum, &cumsum, sizeof(int), cudaMemcpyHostToDevice); + + ballquery_batch_p_cuda_<<>>(n, meanActive, radius, xyz, batch_idxs, batch_offsets, idx, start_len, p_cumsum); + + err = cudaGetLastError(); + if (cudaSuccess != err) { + fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err)); + exit(-1); + } + + cudaMemcpy(&cumsum, p_cumsum, sizeof(int), cudaMemcpyDeviceToHost); + return cumsum; +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/__init__.py b/submodules/PointTransformerV3/Pointcept/libs/pointops/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c8f75488366c12e144febe3adccd63b40820cdfa --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/__init__.py @@ -0,0 +1 @@ +from .functions import * diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/__init__.py b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8c05f2f4b74f1ae4343daf9b38b4576d75f13e81 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/__init__.py @@ -0,0 +1,14 @@ +from .query import knn_query, ball_query, random_ball_query +from .sampling import farthest_point_sampling +from .grouping import grouping, grouping2 +from .interpolation import interpolation, interpolation2 +from .subtraction import subtraction +from .aggregation import aggregation +from .attention import attention_relation_step, attention_fusion_step +from .utils import ( + query_and_group, + knn_query_and_group, + ball_query_and_group, + batch2offset, + offset2batch, +) diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/aggregation.py b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/aggregation.py new file mode 100644 index 0000000000000000000000000000000000000000..f0f62444a70d317dfb8df4adc1167bba5dd19ef1 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/aggregation.py @@ -0,0 +1,57 @@ +import torch +from torch.autograd import Function + +from pointops._C import aggregation_forward_cuda, aggregation_backward_cuda + + +class Aggregation(Function): + @staticmethod + def forward(ctx, input, position, weight, idx): + """ + input: input: (n, c), position: (n, nsample, c), weight : (n, nsample, c'), idx: (n, nsample) + output: (n, c) + """ + assert ( + input.is_contiguous() + and position.is_contiguous() + and weight.is_contiguous() + ) + n, nsample, c = position.shape + w_c = weight.shape[-1] + output = torch.cuda.FloatTensor(n, c).zero_() + aggregation_forward_cuda( + n, nsample, c, w_c, input, position, weight, idx, output + ) + ctx.save_for_backward(input, position, weight, idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_out: (n, c) + output: grad_input: (n, c), grad_position: (n, nsample, c), grad_weight : (n, nsample, c') + """ + input, position, weight, idx = ctx.saved_tensors + n, nsample, c = position.shape + w_c = weight.shape[-1] + grad_input = torch.cuda.FloatTensor(n, c).zero_() + grad_position = torch.cuda.FloatTensor(n, nsample, c).zero_() + grad_weight = torch.cuda.FloatTensor(n, nsample, w_c).zero_() + aggregation_backward_cuda( + n, + nsample, + c, + w_c, + input, + position, + weight, + idx, + grad_output, + grad_input, + grad_position, + grad_weight, + ) + return grad_input, grad_position, grad_weight, None + + +aggregation = Aggregation.apply diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/attention.py b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..4e774ff67051d6272f7de3fd751bf3b712431249 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/attention.py @@ -0,0 +1,120 @@ +import torch +from torch.autograd import Function + +from pointops._C import ( + attention_relation_step_forward_cuda, + attention_relation_step_backward_cuda, + attention_fusion_step_forward_cuda, + attention_fusion_step_backward_cuda, +) + + +class AttentionRelationStep(Function): + @staticmethod + def forward(ctx, query, key, weight, index_target, index_refer): + """ + input - query: (n, g, c), key: (n, g, c), weight: (c) 1_c for scatter attention, + index_target: (m), index_refer: (m) + output - relation: (M, g) + """ + + assert ( + query.is_contiguous() + and key.is_contiguous() + and index_target.is_contiguous() + and index_refer.is_contiguous() + and weight.is_contiguous() + ) + + assert index_target.shape[0] == index_refer.shape[0] + + _, g, c = query.shape + m = index_target.shape[0] + output = torch.cuda.FloatTensor(m, g).zero_() + attention_relation_step_forward_cuda( + m, g, c, query, key, weight, index_target.int(), index_refer.int(), output + ) + ctx.save_for_backward(query, key, weight, index_target, index_refer) + return output + + @staticmethod + def backward(ctx, grad_output): + query, key, weight, index_target, index_refer = ctx.saved_tensors + n, g, c = query.shape + m = index_target.shape[0] + grad_query = torch.cuda.FloatTensor(n, g, c).zero_() + grad_key = torch.cuda.FloatTensor(n, g, c).zero_() + grad_weight = torch.cuda.FloatTensor(c).zero_() + attention_relation_step_backward_cuda( + m, + g, + c, + query, + grad_query, + key, + grad_key, + weight, + grad_weight, + index_target.int(), + index_refer.int(), + grad_output, + ) + return grad_query, grad_key, None, None, None + + +class AttentionFusionStep(Function): + @staticmethod + def forward(ctx, weight, value, index_target, index_refer): + """ + input - weight: (m, g), value: (n, g, c) + index_target: (m), index_value: (m) + output - output: (n, g, c) + """ + + assert ( + weight.is_contiguous() + and value.is_contiguous() + and index_target.is_contiguous() + and index_refer.is_contiguous() + and weight.is_contiguous() + ) + + assert index_target.shape[0] == index_refer.shape[0] + + n, g, c = value.shape + m = index_refer.shape[0] + output = torch.cuda.FloatTensor(n, g, c).zero_() + attention_fusion_step_forward_cuda( + m, g, c, weight, value, index_target.int(), index_refer.int(), output + ) + ctx.save_for_backward(weight, value, index_target, index_refer) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_output: (n, g, c) + output: grad_weight: (m, g), grad_value: (n, g, c), none, none + """ + weight, value, index_target, index_refer = ctx.saved_tensors + n, g, c = value.shape + m = index_target.shape[0] + grad_weight = torch.cuda.FloatTensor(m, g).zero_() + grad_value = torch.cuda.FloatTensor(n, g, c).zero_() + attention_fusion_step_backward_cuda( + m, + g, + c, + weight, + grad_weight, + value, + grad_value, + index_target.int(), + index_refer.int(), + grad_output, + ) + return grad_weight, grad_value, None, None + + +attention_relation_step = AttentionRelationStep.apply +attention_fusion_step = AttentionFusionStep.apply diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/grouping.py b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/grouping.py new file mode 100644 index 0000000000000000000000000000000000000000..c22d1e827f82331a4287362a368ccf93927493e6 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/grouping.py @@ -0,0 +1,63 @@ +import torch +from torch.autograd import Function + +from pointops._C import grouping_forward_cuda, grouping_backward_cuda + + +class Grouping(Function): + @staticmethod + def forward(ctx, input, idx): + """ + input: input: (n, c), idx : (m, nsample) + output: (m, nsample, c) + """ + assert input.is_contiguous() and idx.is_contiguous() + m, nsample, n, c = idx.shape[0], idx.shape[1], input.shape[0], input.shape[1] + output = torch.cuda.FloatTensor(m, nsample, c) + grouping_forward_cuda(m, nsample, c, input, idx, output) + ctx.n = n + ctx.save_for_backward(idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_out: (m, c, nsample) + output: (n, c), None + """ + n = ctx.n + (idx,) = ctx.saved_tensors + m, nsample, c = grad_output.shape + grad_input = torch.cuda.FloatTensor(n, c).zero_() + grouping_backward_cuda(m, nsample, c, grad_output, idx, grad_input) + return grad_input, None + + +def grouping(idx, feat, xyz, new_xyz=None, with_xyz=False): + if new_xyz is None: + new_xyz = xyz + assert xyz.is_contiguous() and feat.is_contiguous() + m, nsample, c = idx.shape[0], idx.shape[1], feat.shape[1] + xyz = torch.cat([xyz, torch.zeros([1, 3]).to(xyz.device)], dim=0) + feat = torch.cat([feat, torch.zeros([1, c]).to(feat.device)], dim=0) + grouped_feat = feat[idx.view(-1).long(), :].view( + m, nsample, c + ) # (m, num_sample, c) + + if with_xyz: + assert new_xyz.is_contiguous() + mask = torch.sign(idx + 1) + grouped_xyz = xyz[idx.view(-1).long(), :].view( + m, nsample, 3 + ) - new_xyz.unsqueeze( + 1 + ) # (m, num_sample, 3) + grouped_xyz = torch.einsum( + "n s c, n s -> n s c", grouped_xyz, mask + ) # (m, num_sample, 3) + return torch.cat((grouped_xyz, grouped_feat), -1) + else: + return grouped_feat + + +grouping2 = Grouping.apply diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/interpolation.py b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/interpolation.py new file mode 100644 index 0000000000000000000000000000000000000000..4a5c861f272f89421fa097505d9882b2c473a060 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/interpolation.py @@ -0,0 +1,59 @@ +import torch +from torch.autograd import Function + +from pointops._C import interpolation_forward_cuda, interpolation_backward_cuda +from .query import knn_query + + +def interpolation(xyz, new_xyz, feat, offset, new_offset, k=3): + """ + input: coords: (m, 3), new_xyz: (n, 3), color: (m, c), offset: (b), new_offset: (b) + output: (n, c) + """ + assert xyz.is_contiguous() and new_xyz.is_contiguous() and feat.is_contiguous() + idx, dist = knn_query(k, xyz, offset, new_xyz, new_offset) # (n, 3), (n, 3) + dist_recip = 1.0 / (dist + 1e-8) # (n, 3) + norm = torch.sum(dist_recip, dim=1, keepdim=True) + weight = dist_recip / norm # (n, 3) + + new_feat = torch.cuda.FloatTensor(new_xyz.shape[0], feat.shape[1]).zero_() + for i in range(k): + new_feat += feat[idx[:, i].long(), :] * weight[:, i].unsqueeze(-1) + return new_feat + + +class Interpolation(Function): + @staticmethod + def forward(ctx, xyz, new_xyz, input, offset, new_offset, k=3): + """ + input: coords: (m, 3), new_xyz: (n, 3), input: (m, c), offset: (b), new_offset: (b) + output: (n, c) + """ + assert xyz.is_contiguous() and new_xyz.is_contiguous() and input.is_contiguous() + idx, dist = knn_query(k, xyz, offset, new_xyz, new_offset) # (n, k), (n, k) + dist_recip = 1.0 / (dist + 1e-8) # (n, k) + norm = torch.sum(dist_recip, dim=1, keepdim=True) + weight = dist_recip / norm # (n, k) + + n, c, m = new_xyz.shape[0], input.shape[1], input.shape[0] + output = torch.cuda.FloatTensor(n, c).zero_() + interpolation_forward_cuda(n, c, k, input, idx, weight, output) + ctx.m, ctx.k = m, k + ctx.save_for_backward(idx, weight) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: coords: (m, 3), new_xyz: (n, 3), input: (m, c), offset: (b), new_offset: (b) + output: (n, c) + """ + m, k = ctx.m, ctx.k + idx, weight = ctx.saved_tensors + n, c = grad_output.shape + grad_input = torch.cuda.FloatTensor(m, c).zero_() + interpolation_backward_cuda(n, c, k, grad_output, idx, weight, grad_input) + return None, None, grad_input, None, None, None + + +interpolation2 = Interpolation.apply diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/query.py b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/query.py new file mode 100644 index 0000000000000000000000000000000000000000..c1294b6125e00ae1d1dec21ed52a803c164c4810 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/query.py @@ -0,0 +1,113 @@ +import torch +from torch.autograd import Function + +from pointops._C import knn_query_cuda, random_ball_query_cuda, ball_query_cuda + + +class KNNQuery(Function): + @staticmethod + def forward(ctx, nsample, xyz, offset, new_xyz=None, new_offset=None): + """ + input: coords: (n, 3), new_xyz: (m, 3), offset: (b), new_offset: (b) + output: idx: (m, nsample) -1 is placeholder, dist2: (m, nsample) + """ + if new_xyz is None or new_offset is None: + new_xyz = xyz + new_offset = offset + assert xyz.is_contiguous() and new_xyz.is_contiguous() + m = new_xyz.shape[0] + idx = torch.cuda.IntTensor(m, nsample).zero_() + dist2 = torch.cuda.FloatTensor(m, nsample).zero_() + knn_query_cuda( + m, nsample, xyz, new_xyz, offset.int(), new_offset.int(), idx, dist2 + ) + return idx, torch.sqrt(dist2) + + +class RandomBallQuery(Function): + """Random Ball Query. + + Find nearby points in spherical space. + """ + + @staticmethod + def forward( + ctx, nsample, max_radius, min_radius, xyz, offset, new_xyz=None, new_offset=None + ): + """ + input: coords: (n, 3), new_xyz: (m, 3), offset: (b), new_offset: (b) + output: idx: (m, nsample), dist2: (m, nsample) + """ + if new_xyz is None or new_offset is None: + new_xyz = xyz + new_offset = offset + assert xyz.is_contiguous() and new_xyz.is_contiguous() + assert min_radius < max_radius + + m = new_xyz.shape[0] + order = [] + for k in range(offset.shape[0]): + s_k, e_k = (0, offset[0]) if k == 0 else (offset[k - 1], offset[k]) + order.append( + torch.randperm(e_k - s_k, dtype=torch.int32, device=offset.device) + s_k + ) + order = torch.cat(order, dim=0) + idx = torch.cuda.IntTensor(m, nsample).zero_() + dist2 = torch.cuda.FloatTensor(m, nsample).zero_() + random_ball_query_cuda( + m, + nsample, + min_radius, + max_radius, + order, + xyz, + new_xyz, + offset.int(), + new_offset.int(), + idx, + dist2, + ) + return idx, torch.sqrt(dist2) + + +class BallQuery(Function): + """Ball Query. + + Find nearby points in spherical space. + """ + + @staticmethod + def forward( + ctx, nsample, max_radius, min_radius, xyz, offset, new_xyz=None, new_offset=None + ): + """ + input: coords: (n, 3), new_xyz: (m, 3), offset: (b), new_offset: (b) + output: idx: (m, nsample), dist2: (m, nsample) + """ + if new_xyz is None or new_offset is None: + new_xyz = xyz + new_offset = offset + assert xyz.is_contiguous() and new_xyz.is_contiguous() + assert min_radius < max_radius + + m = new_xyz.shape[0] + idx = torch.cuda.IntTensor(m, nsample).zero_() + dist2 = torch.cuda.FloatTensor(m, nsample).zero_() + ball_query_cuda( + m, + nsample, + min_radius, + max_radius, + xyz, + new_xyz, + offset.int(), + new_offset.int(), + idx, + dist2, + ) + return idx, torch.sqrt(dist2) + + +knn_query = KNNQuery.apply +ball_query = BallQuery.apply +random_ball_query = RandomBallQuery.apply diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/sampling.py b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/sampling.py new file mode 100644 index 0000000000000000000000000000000000000000..9f233d4afe02e43a6a390ca465f7108a01b98541 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/sampling.py @@ -0,0 +1,27 @@ +import torch +from torch.autograd import Function + +from pointops._C import farthest_point_sampling_cuda + + +class FarthestPointSampling(Function): + @staticmethod + def forward(ctx, xyz, offset, new_offset): + """ + input: coords: (n, 3), offset: (b), new_offset: (b) + output: idx: (m) + """ + assert xyz.is_contiguous() + n, b, n_max = xyz.shape[0], offset.shape[0], offset[0] + for i in range(1, b): + n_max = max(offset[i] - offset[i - 1], n_max) + idx = torch.cuda.IntTensor(new_offset[b - 1].item()).zero_() + tmp = torch.cuda.FloatTensor(n).fill_(1e10) + farthest_point_sampling_cuda( + b, n_max, xyz, offset.int(), new_offset.int(), tmp, idx + ) + del tmp + return idx + + +farthest_point_sampling = FarthestPointSampling.apply diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/subtraction.py b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/subtraction.py new file mode 100644 index 0000000000000000000000000000000000000000..bc683ce3d75901777e57886adc077d570230e027 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/subtraction.py @@ -0,0 +1,38 @@ +import torch +from torch.autograd import Function + +from pointops._C import subtraction_forward_cuda, subtraction_backward_cuda + + +class Subtraction(Function): + @staticmethod + def forward(ctx, input1, input2, idx): + """ + input: input1: (n, c), input2: (n, c), idx: (n, nsample) + output: (n, nsample, c) + """ + assert input1.is_contiguous() and input2.is_contiguous() + n, c = input1.shape + nsample = idx.shape[-1] + output = torch.cuda.FloatTensor(n, nsample, c).zero_() + subtraction_forward_cuda(n, nsample, c, input1, input2, idx, output) + ctx.save_for_backward(idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_out: (n, nsample, c) + output: grad_input1: (n, c), grad_input2: (n, c) + """ + (idx,) = ctx.saved_tensors + n, nsample, c = grad_output.shape + grad_input1 = torch.cuda.FloatTensor(n, c).zero_() + grad_input2 = torch.cuda.FloatTensor(n, c).zero_() + subtraction_backward_cuda( + n, nsample, c, idx, grad_output, grad_input1, grad_input2 + ) + return grad_input1, grad_input2, None + + +subtraction = Subtraction.apply diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/utils.py b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..15e3e328bb012bb684787466f3ec2e97d1317b2b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/functions/utils.py @@ -0,0 +1,121 @@ +import torch +from pointops import knn_query, ball_query, grouping + + +def knn_query_and_group( + feat, + xyz, + offset=None, + new_xyz=None, + new_offset=None, + idx=None, + nsample=None, + with_xyz=False, +): + if idx is None: + assert nsample is not None + idx, _ = knn_query(nsample, xyz, offset, new_xyz, new_offset) + return grouping(idx, feat, xyz, new_xyz, with_xyz), idx + + +def ball_query_and_group( + feat, + xyz, + offset=None, + new_xyz=None, + new_offset=None, + idx=None, + max_radio=None, + min_radio=0, + nsample=None, + with_xyz=False, +): + if idx is None: + assert nsample is not None and offset is not None + assert max_radio is not None and min_radio is not None + idx, _ = ball_query( + nsample, max_radio, min_radio, xyz, offset, new_xyz, new_offset + ) + return grouping(idx, feat, xyz, new_xyz, with_xyz), idx + + +def query_and_group( + nsample, + xyz, + new_xyz, + feat, + idx, + offset, + new_offset, + dilation=0, + with_feat=True, + with_xyz=True, +): + """ + input: coords: (n, 3), new_xyz: (m, 3), color: (n, c), idx: (m, nsample), offset: (b), new_offset: (b) + output: new_feat: (m, nsample, c+3), grouped_idx: (m, nsample) + """ + assert xyz.is_contiguous() and new_xyz.is_contiguous() and feat.is_contiguous() + if new_xyz is None: + new_xyz = xyz + + if idx is None: + num_samples_total = 1 + (nsample - 1) * (dilation + 1) + # num points in a batch might < num_samples_total => [n1, n2, ..., nk, ns, ns, ns, ...] + idx_no_dilation, _ = knn_query( + num_samples_total, xyz, offset, new_xyz, new_offset + ) # (m, nsample * (d + 1)) + idx = [] + batch_end = offset.tolist() + batch_start = [0] + batch_end[:-1] + new_batch_end = new_offset.tolist() + new_batch_start = [0] + new_batch_end[:-1] + for i in range(offset.shape[0]): + if batch_end[i] - batch_start[i] < num_samples_total: + soft_dilation = (batch_end[i] - batch_start[i] - 1) / (nsample - 1) - 1 + else: + soft_dilation = dilation + idx.append( + idx_no_dilation[ + new_batch_start[i] : new_batch_end[i], + [int((soft_dilation + 1) * i) for i in range(nsample)], + ] + ) + idx = torch.cat(idx, dim=0) + + if not with_feat: + return idx + + n, m, c = xyz.shape[0], new_xyz.shape[0], feat.shape[1] + grouped_xyz = xyz[idx.view(-1).long(), :].view(m, nsample, 3) # (m, nsample, 3) + # grouped_xyz = grouping(coords, idx) # (m, nsample, 3) + grouped_xyz -= new_xyz.unsqueeze(1) # (m, nsample, 3) + grouped_feat = feat[idx.view(-1).long(), :].view(m, nsample, c) # (m, nsample, c) + # grouped_feat = grouping(color, idx) # (m, nsample, c) + + if with_xyz: + return torch.cat((grouped_xyz, grouped_feat), -1), idx # (m, nsample, 3+c) + else: + return grouped_feat, idx + + +def offset2batch(offset): + return ( + torch.cat( + [ + ( + torch.tensor([i] * (o - offset[i - 1])) + if i > 0 + else torch.tensor([i] * o) + ) + for i, o in enumerate(offset) + ], + dim=0, + ) + .long() + .to(offset.device) + ) + + +def batch2offset(batch): + return torch.cumsum(batch.bincount(), dim=0).int() diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/setup.py b/submodules/PointTransformerV3/Pointcept/libs/pointops/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..0cdf07b6c12bf702b40accbb51fd1825e4050a8b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/setup.py @@ -0,0 +1,33 @@ +import os +from setuptools import setup +from torch.utils.cpp_extension import BuildExtension, CUDAExtension +from distutils.sysconfig import get_config_vars + +(opt,) = get_config_vars("OPT") +os.environ["OPT"] = " ".join( + flag for flag in opt.split() if flag != "-Wstrict-prototypes" +) + +src = "src" +sources = [ + os.path.join(root, file) + for root, dirs, files in os.walk(src) + for file in files + if file.endswith(".cpp") or file.endswith(".cu") +] + +setup( + name="pointops", + version="1.0", + install_requires=["torch", "numpy"], + packages=["pointops"], + package_dir={"pointops": "functions"}, + ext_modules=[ + CUDAExtension( + name="pointops._C", + sources=sources, + extra_compile_args={"cxx": ["-g"], "nvcc": ["-O2"]}, + ) + ], + cmdclass={"build_ext": BuildExtension}, +) diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/__init__.py b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/aggregation/aggregation_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/aggregation/aggregation_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..491b6f41660edf9b5ea5656cc88edba8ed807d71 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/aggregation/aggregation_cuda.cpp @@ -0,0 +1,28 @@ +#include +#include +#include +#include "aggregation_cuda_kernel.h" + + +void aggregation_forward_cuda(int n, int nsample, int c, int w_c, at::Tensor input_tensor, at::Tensor position_tensor, at::Tensor weight_tensor, at::Tensor idx_tensor, at::Tensor output_tensor) +{ + const float *input = input_tensor.data_ptr(); + const float *position = position_tensor.data_ptr(); + const float *weight = weight_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + aggregation_forward_cuda_launcher(n, nsample, c, w_c, input, position, weight, idx, output); +} + +void aggregation_backward_cuda(int n, int nsample, int c, int w_c, at::Tensor input_tensor, at::Tensor position_tensor, at::Tensor weight_tensor, at::Tensor idx_tensor, at::Tensor grad_output_tensor, at::Tensor grad_input_tensor, at::Tensor grad_position_tensor, at::Tensor grad_weight_tensor) +{ + const float *input = input_tensor.data_ptr(); + const float *position = position_tensor.data_ptr(); + const float *weight = weight_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + const float *grad_output = grad_output_tensor.data_ptr(); + float *grad_input = grad_input_tensor.data_ptr(); + float *grad_position = grad_position_tensor.data_ptr(); + float *grad_weight = grad_weight_tensor.data_ptr(); + aggregation_backward_cuda_launcher(n, nsample, c, w_c, input, position, weight, idx, grad_output, grad_input, grad_position, grad_weight); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/aggregation/aggregation_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/aggregation/aggregation_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..8339bb7e2088abffefba02c26b248edafed6cf47 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/aggregation/aggregation_cuda_kernel.cu @@ -0,0 +1,53 @@ +#include "../cuda_utils.h" +#include "aggregation_cuda_kernel.h" + + +__global__ void aggregation_forward_cuda_kernel(int n, int nsample, int c, int w_c, const float *input, const float *position, const float *weight, const int *idx, float *output) { + // input: input: (n, c), position: (n, nsample, c), weight: (n, nsample, w_c), idx: (n, nsample), output: (n, c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= n * c) return; + const int c_idx = index % c; + const int n_idx = index / c; + const int w_c_idx = c_idx % w_c; + for (int nsample_idx = 0; nsample_idx < nsample; nsample_idx++) + { + int idx_idx = n_idx * nsample + nsample_idx; + int input_idx = idx[idx_idx] * c + c_idx; + int position_idx = n_idx * nsample * c + nsample_idx * c + c_idx; + int weight_idx = n_idx * nsample * w_c + nsample_idx * w_c + w_c_idx; + output[index] += (input[input_idx] + position[position_idx]) * weight[weight_idx]; + } +} + +__global__ void aggregation_backward_cuda_kernel(int n, int nsample, int c, int w_c, const float *input, const float *position, const float *weight, const int *idx, const float *grad_output, float *grad_input, float *grad_position, float *grad_weight) { + // input: grad_output: (n, c), output: grad_input: (n, c), grad_position: (n, nsample, c), grad_weight: (n, nsample, w_c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= n * c) return; + const int c_idx = index % c; + const int n_idx = index / c; + const int w_c_idx = c_idx % w_c; + for (int nsample_idx = 0; nsample_idx < nsample; nsample_idx++) + { + int idx_idx = n_idx * nsample + nsample_idx; + int input_idx = idx[idx_idx] * c + c_idx; + int position_idx = n_idx * nsample * c + nsample_idx * c + c_idx; + int weight_idx = n_idx * nsample * w_c + nsample_idx * w_c + w_c_idx; + atomicAdd(grad_input + input_idx, grad_output[index] * weight[weight_idx]); + grad_position[position_idx] = grad_output[index] * weight[weight_idx]; + atomicAdd(grad_weight + weight_idx, grad_output[index] * (input[input_idx] + position[position_idx])); + } +} + +void aggregation_forward_cuda_launcher(int n, int nsample, int c, int w_c, const float *input, const float *position, const float *weight, const int *idx, float *output) { + // input: input: (n, c), position: (n, nsample, c), weight: (n, nsample, w_c), idx: (n, nsample), output: (n, c) + dim3 blocks(DIVUP(n * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + aggregation_forward_cuda_kernel<<>>(n, nsample, c, w_c, input, position, weight, idx, output); +} + +void aggregation_backward_cuda_launcher(int n, int nsample, int c, int w_c, const float *input, const float *position, const float *weight, const int *idx, const float *grad_output, float *grad_input, float *grad_position, float *grad_weight) { + // input: grad_output: (n, c), output: grad_input: (n, c), grad_position: (n, nsample, c), grad_weight: (n, nsample, w_c) + dim3 blocks(DIVUP(n * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + aggregation_backward_cuda_kernel<<>>(n, nsample, c, w_c, input, position, weight, idx, grad_output, grad_input, grad_position, grad_weight); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/aggregation/aggregation_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/aggregation/aggregation_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..5211a96aa2acbe0d9baf32bddc9ab4be87703072 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/aggregation/aggregation_cuda_kernel.h @@ -0,0 +1,20 @@ +#ifndef _AGGREGATION_CUDA_KERNEL +#define _AGGREGATION_CUDA_KERNEL +#include +#include +#include + +void aggregation_forward_cuda(int n, int nsample, int c, int w_c, at::Tensor input_tensor, at::Tensor position_tensor, at::Tensor weight_tensor, at::Tensor idx_tensor, at::Tensor output_tensor); +void aggregation_backward_cuda(int n, int nsample, int c, int w_c, at::Tensor input_tensor, at::Tensor position_tensor, at::Tensor weight_tensor, at::Tensor idx_tensor, at::Tensor grad_output_tensor, at::Tensor grad_input_tensor, at::Tensor grad_position_tensor, at::Tensor grad_weight_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void aggregation_forward_cuda_launcher(int n, int nsample, int c, int w_c, const float *input, const float *position, const float *weight, const int *idx, float *output); +void aggregation_backward_cuda_launcher(int n, int nsample, int c, int w_c, const float *input, const float *position, const float *weight, const int *idx, const float *grad_output, float *grad_input, float *grad_position, float *grad_weight); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/attention/attention_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/attention/attention_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..79b90c7ebc3ed85dc389bc4ae3169a086efc5848 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/attention/attention_cuda.cpp @@ -0,0 +1,76 @@ +#include +#include +#include +#include "attention_cuda_kernel.h" + + +void attention_relation_step_forward_cuda(int m, int g, int c, + at::Tensor query_tensor, at::Tensor key_tensor, at::Tensor weight_tensor, + at::Tensor index_target_tensor, at::Tensor index_refer_tensor, + at::Tensor output_tensor) +{ + const float *query = query_tensor.data_ptr(); + const float *key = key_tensor.data_ptr(); + const float *weight = weight_tensor.data_ptr(); + const int *index_target = index_target_tensor.data_ptr(); + const int *index_refer = index_refer_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + attention_relation_step_forward_cuda_launcher(m, g, c, query, key, weight, index_target, index_refer, output); +} + +void attention_relation_step_backward_cuda(int m, int g, int c, + at::Tensor query_tensor, at::Tensor grad_query_tensor, + at::Tensor key_tensor, at::Tensor grad_key_tensor, + at::Tensor weight_tensor, at::Tensor grad_weight_tensor, + at::Tensor index_target_tensor, at::Tensor index_refer_tensor, + at::Tensor grad_output_tensor) +{ + const float *query = query_tensor.data_ptr(); + float *grad_query = grad_query_tensor.data_ptr(); + const float *key = key_tensor.data_ptr(); + float *grad_key = grad_key_tensor.data_ptr(); + const float *weight = weight_tensor.data_ptr(); + float *grad_weight = grad_weight_tensor.data_ptr(); + const int *index_target = index_target_tensor.data_ptr(); + const int *index_refer = index_refer_tensor.data_ptr(); + const float *grad_output = grad_output_tensor.data_ptr(); + attention_relation_step_backward_cuda_launcher(m, g, c, + query, grad_query, + key, grad_key, + weight, grad_weight, + index_target, index_refer, grad_output); +} + + +void attention_fusion_step_forward_cuda(int m, int g, int c, + at::Tensor weight_tensor, at::Tensor value_tensor, + at::Tensor index_target_tensor, at::Tensor index_refer_tensor, + at::Tensor output_tensor) +{ + const float *weight = weight_tensor.data_ptr(); + const float *value = value_tensor.data_ptr(); + const int *index_target = index_target_tensor.data_ptr(); + const int *index_refer = index_refer_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + attention_fusion_step_forward_cuda_launcher(m, g, c, weight, value, index_target, index_refer, output); +} + + +void attention_fusion_step_backward_cuda(int m, int g, int c, + at::Tensor weight_tensor, at::Tensor grad_weight_tensor, + at::Tensor value_tensor, at::Tensor grad_value_tensor, + at::Tensor index_target_tensor, at::Tensor index_refer_tensor, + at::Tensor grad_output_tensor) +{ + const float *weight = weight_tensor.data_ptr(); + float *grad_weight = grad_weight_tensor.data_ptr(); + const float *value = value_tensor.data_ptr(); + float *grad_value = grad_value_tensor.data_ptr(); + const int *index_target = index_target_tensor.data_ptr(); + const int *index_refer = index_refer_tensor.data_ptr(); + const float *grad_output = grad_output_tensor.data_ptr(); + attention_fusion_step_backward_cuda_launcher(m, g, c, + weight, grad_weight, + value, grad_value, + index_target, index_refer, grad_output); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/attention/attention_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/attention/attention_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..05f4544a4dc4da584ad70eece75265d4845171e7 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/attention/attention_cuda_kernel.cu @@ -0,0 +1,149 @@ +#include "../cuda_utils.h" +#include "attention_cuda_kernel.h" + + +/* +Kernels +*/ + +__global__ void attention_relation_step_forward_cuda_kernel(int m, int g, int c, + const float *query, const float *key, const float *weight, + const int *index_target, const int *index_refer, + float *output) +{ + int r_idx = blockIdx.x * blockDim.x + threadIdx.x; + int g_idx = blockIdx.y; + int c_idx = blockIdx.z; + + if (r_idx >= m || g_idx >= g || c_idx >= c) return; + int q_idx = index_target[r_idx] * g * c + g_idx * c + c_idx; + int k_idx = index_refer[r_idx] * g * c + g_idx * c + c_idx; + + float r = query[q_idx] * key[k_idx] * weight[c_idx]; + atomicAdd(output + r_idx * g + g_idx, r); +} + +__global__ void attention_relation_step_backward_cuda_kernel(int m, int g, int c, + const float *query, float *grad_query, + const float *key, float *grad_key, + const float *weight, float *grad_weight, + const int *index_target, const int *index_refer, + const float *grad_output) +{ + int r_idx = blockIdx.x * blockDim.x + threadIdx.x; + int g_idx = blockIdx.y; + int c_idx = blockIdx.z; + + if (r_idx >= m || g_idx >= g || c_idx >= c) return; + + int q_idx = index_target[r_idx] * g * c + g_idx * c + c_idx; + int k_idx = index_refer[r_idx] * g * c + g_idx * c + c_idx; + int o_idx = r_idx * g + g_idx; + float grad_r = grad_output[o_idx]; + atomicAdd(grad_query + q_idx, grad_r * key[k_idx] * weight[c_idx]); + atomicAdd(grad_key + k_idx, grad_r * query[q_idx] * weight[c_idx]); + atomicAdd(grad_weight + c_idx, grad_r * key[k_idx] * query[q_idx]); +} + + +__global__ void attention_fusion_step_forward_cuda_kernel(int m, int g, int c, + const float *weight, const float *value, + const int *index_target, const int *index_refer, + float *output) +{ + int r_idx = blockIdx.x * blockDim.x + threadIdx.x; + int g_idx = blockIdx.y; + int c_idx = blockIdx.z; + + if (r_idx >= m || g_idx >= g || c_idx >= c) return; + + int o_idx = index_target[r_idx] * g * c + g_idx * c + c_idx; + int v_idx = index_refer[r_idx] * g * c + g_idx * c + c_idx; + + float f = weight[r_idx * g + g_idx] * value[v_idx]; + atomicAdd(output + o_idx, f); +} + + +__global__ void attention_fusion_step_backward_cuda_kernel(int m, int g, int c, + const float *weight, float *grad_weight, + const float *value, float *grad_value, + const int *index_target, const int *index_refer, + const float *grad_output) +{ + int r_idx = blockIdx.x * blockDim.x + threadIdx.x; + int g_idx = blockIdx.y; + int c_idx = blockIdx.z; + + if (r_idx >= m || g_idx >= g || c_idx >= c) return; + + int o_idx = index_target[r_idx] * g * c + g_idx * c + c_idx; + int v_idx = index_refer[r_idx] * g * c + g_idx * c + c_idx; + int w_idx = r_idx * g + g_idx; + float grad = grad_output[o_idx]; + atomicAdd(grad_weight + w_idx, grad * value[v_idx]); + atomicAdd(grad_value + v_idx, grad * weight[w_idx]); +} + +/* +Launchers +*/ + + +void attention_relation_step_forward_cuda_launcher(int m, int g, int c, + const float *query, const float *key, const float *weight, + const int *index_target, const int *index_refer, + float *output) +{ + dim3 blocks(DIVUP(m, THREADS_PER_BLOCK), g, c); + dim3 threads(THREADS_PER_BLOCK); + attention_relation_step_forward_cuda_kernel<<>>(m, g, c, query, key, weight, + index_target, index_refer, output); +} + +void attention_relation_step_backward_cuda_launcher(int m, int g, int c, + const float *query, float *grad_query, + const float *key, float *grad_key, + const float *weight, float *grad_weight, + const int *index_target, const int *index_refer, + const float *grad_output) +{ + dim3 blocks(DIVUP(m, THREADS_PER_BLOCK), g, c); + dim3 threads(THREADS_PER_BLOCK); + attention_relation_step_backward_cuda_kernel<<>>(m, g, c, + query, grad_query, + key, grad_key, + weight, grad_weight, + index_target, index_refer, + grad_output); +} + + +void attention_fusion_step_forward_cuda_launcher(int m, int g, int c, + const float *weight, const float *value, + const int *index_target, const int *index_refer, + float *output) +{ + dim3 blocks(DIVUP(m, THREADS_PER_BLOCK), g, c); + dim3 threads(THREADS_PER_BLOCK); + attention_fusion_step_forward_cuda_kernel<<>>(m, g, c, weight, value, + index_target, index_refer, output); +} + + +void attention_fusion_step_backward_cuda_launcher(int m, int g, int c, + const float *weight, float *grad_weight, + const float *value, float *grad_value, + const int *index_target, const int *index_refer, + const float *grad_output) +{ + dim3 blocks(DIVUP(m, THREADS_PER_BLOCK), g, c); + dim3 threads(THREADS_PER_BLOCK); + attention_fusion_step_backward_cuda_kernel<<>>(m, g, c, + weight, grad_weight, + value, grad_value, + index_target, index_refer, + grad_output); +} + + diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/attention/attention_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/attention/attention_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..fec965c0415c4cb5c64fd10e441b6a4c6a6c9ae9 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/attention/attention_cuda_kernel.h @@ -0,0 +1,54 @@ +#ifndef _ATTENTION_CUDA_KERNEL +#define _ATTENTION_CUDA_KERNEL +#include +#include +#include + +void attention_relation_step_forward_cuda(int m, int g, int c, + at::Tensor query_tensor, at::Tensor key_tensor, at::Tensor weight_tensor, + at::Tensor index_target_tensor, at::Tensor index_refer_tensor, + at::Tensor output_tensor); +void attention_relation_step_backward_cuda(int m, int g, int c, + at::Tensor query_tensor, at::Tensor grad_query_tensor, + at::Tensor key_tensor, at::Tensor grad_key_tensor, + at::Tensor weight_tensor, at::Tensor grad_weight_tensor, + at::Tensor index_target_tensor, at::Tensor index_refer_tensor, + at::Tensor grad_output_tensor); +void attention_fusion_step_forward_cuda(int m, int g, int c, + at::Tensor weight_tensor, at::Tensor value_tensor, + at::Tensor index_target_tensor, at::Tensor index_refer_tensor, + at::Tensor output_tensor); +void attention_fusion_step_backward_cuda(int m, int g, int c, + at::Tensor weight_tensor, at::Tensor grad_weight_tensor, + at::Tensor value_tensor, at::Tensor grad_value_tensor, + at::Tensor index_target_tensor, at::Tensor index_refer_tensor, + at::Tensor grad_output_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void attention_relation_step_forward_cuda_launcher(int m, int g, int c, + const float *query, const float *key, const float *weight, + const int *index_target, const int *index_refer, + float *output); +void attention_relation_step_backward_cuda_launcher(int m, int g, int c, + const float *query, float *grad_query, + const float *key, float *grad_key, + const float *weight, float *grad_weight, + const int *index_target, const int *index_refer, + const float *grad_output); +void attention_fusion_step_forward_cuda_launcher(int m, int g, int c, + const float *weight, const float *value, + const int *index_target, const int *index_refer, + float *output); +void attention_fusion_step_backward_cuda_launcher(int m, int g, int c, + const float *weight, float *grad_weight, + const float *value, float *grad_value, + const int *index_target, const int *index_refer, + const float *grad_output); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/ball_query/ball_query_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/ball_query/ball_query_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..04cd5ff9e8e39c006222d5651f3aae70ce2e35c9 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/ball_query/ball_query_cuda.cpp @@ -0,0 +1,20 @@ +#include +#include +#include +#include "ball_query_cuda_kernel.h" + + +void ball_query_cuda(int m, int nsample, + float min_radius, float max_radius, + at::Tensor xyz_tensor, at::Tensor new_xyz_tensor, + at::Tensor offset_tensor, at::Tensor new_offset_tensor, + at::Tensor idx_tensor, at::Tensor dist2_tensor) +{ + const float *xyz = xyz_tensor.data_ptr(); + const float *new_xyz = new_xyz_tensor.data_ptr(); + const int *offset = offset_tensor.data_ptr(); + const int *new_offset = new_offset_tensor.data_ptr(); + int *idx = idx_tensor.data_ptr(); + float *dist2 = dist2_tensor.data_ptr(); + ball_query_cuda_launcher(m, nsample, min_radius, max_radius, xyz, new_xyz, offset, new_offset, idx, dist2); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/ball_query/ball_query_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/ball_query/ball_query_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..7b3d95a9835f607798f0d63e2b66ddb3af9032da --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/ball_query/ball_query_cuda_kernel.cu @@ -0,0 +1,190 @@ +#include "../cuda_utils.h" +#include "ball_query_cuda_kernel.h" + + +namespace ball_query_utils{ + +template +__device__ void swap(DType *x, DType *y) +{ + DType tmp = *x; + *x = *y; + *y = tmp; +} + +__device__ void reheap(float *dist, int *idx, int k) +{ + int root = 0; + int child = root * 2 + 1; + while (child < k) + { + if(child + 1 < k && dist[child+1] > dist[child]) + child++; + if(dist[root] > dist[child]) + return; + swap(&dist[root], &dist[child]); + swap(&idx[root], &idx[child]); + root = child; + child = root * 2 + 1; + } +} + + +__device__ void heap_sort(float *dist, int *idx, int k) +{ + int i; + for (i = k - 1; i > 0; i--) + { + swap(&dist[0], &dist[i]); + swap(&idx[0], &idx[i]); + reheap(dist, idx, i); + } +} + +__device__ int get_bt_idx(int idx, const int *offset) +{ + int i = 0; + while (1) + { + if (idx < offset[i]) + break; + else + i++; + } + return i; +} +} // namespace ball_query_utils + +__global__ void ball_query_cuda_kernel(int m, int nsample, + float min_radius, float max_radius, + const float *__restrict__ xyz, const float *__restrict__ new_xyz, + const int *__restrict__ offset, const int *__restrict__ new_offset, + int *__restrict__ idx, float *__restrict__ dist2) { + // input: xyz (n, 3) new_xyz (m, 3) + // output: idx (m, nsample) dist (m, nsample) + int pt_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (pt_idx >= m) return; + + new_xyz += pt_idx * 3; + idx += pt_idx * nsample; + dist2 += pt_idx * nsample; + + int bt_idx = ball_query_utils::get_bt_idx(pt_idx, new_offset); + int start; + if (bt_idx == 0) + start = 0; + else + start = offset[bt_idx - 1]; + int end = offset[bt_idx]; + + float max_radius2 = max_radius * max_radius; + float min_radius2 = min_radius * min_radius; + float new_x = new_xyz[0]; + float new_y = new_xyz[1]; + float new_z = new_xyz[2]; + + float candi_dist[2048]; + int candi_idx[2048]; + int candi_num = 0; + + for(int i = start; i < end; i++){ + float x = xyz[i * 3 + 0]; + float y = xyz[i * 3 + 1]; + float z = xyz[i * 3 + 2]; + float d2 = (new_x - x) * (new_x - x) + (new_y - y) * (new_y - y) + (new_z - z) * (new_z - z); + + if (d2 <= 1e-5 || (d2 >= min_radius2 && d2 < max_radius2)){ + // TODO: Check d2 <= 1e-5 + candi_dist[candi_num] = d2; + candi_idx[candi_num] = i; + candi_num += 1; + } + } + ball_query_utils::heap_sort(candi_dist, candi_idx, candi_num); + if(candi_num <= nsample){ + for(int i = 0; i < candi_num; i++){ + idx[i] = candi_idx[i]; + dist2[i] = candi_dist[i]; + } + for(int i = candi_num; i < nsample; i++){ + idx[i] = -1; + dist2[i] = 1e10; + } + } + else{ + float sep = static_cast(candi_num) / nsample; + for(int i = 0; i < nsample; i++) + { + int index = static_cast(sep * i); + idx[i] = candi_idx[index]; + dist2[i] = candi_idx[index]; + } + } +} + +/* Random Sample Mode Ball Query */ + +// __global__ void ball_query_cuda_kernel(int m, int nsample, +// float min_radius, float max_radius, +// const float *__restrict__ xyz, const float *__restrict__ new_xyz, +// const int *__restrict__ offset, const int *__restrict__ new_offset, +// int *__restrict__ idx, float *__restrict__ dist2) { +// // input: xyz (n, 3) new_xyz (m, 3) +// // output: idx (m, nsample) dist (m, nsample) +// int pt_idx = blockIdx.x * blockDim.x + threadIdx.x; +// if (pt_idx >= m) return; +// +// new_xyz += pt_idx * 3; +// idx += pt_idx * nsample; +// dist2 += pt_idx * nsample; +// +// int bt_idx = ball_get_bt_idx(pt_idx, new_offset); +// int start; +// if (bt_idx == 0) +// start = 0; +// else +// start = offset[bt_idx - 1]; +// int end = offset[bt_idx]; +// +// float max_radius2 = max_radius * max_radius; +// float min_radius2 = min_radius * min_radius; +// float new_x = new_xyz[0]; +// float new_y = new_xyz[1]; +// float new_z = new_xyz[2]; +// +// int cnt = 0; +// for(int i = start; i < end; i++){ +// float x = xyz[i * 3 + 0]; +// float y = xyz[i * 3 + 1]; +// float z = xyz[i * 3 + 2]; +// float d2 = (new_x - x) * (new_x - x) + (new_y - y) * (new_y - y) + (new_z - z) * (new_z - z); +// +// if (d2 == 0 || (d2 >= min_radius2 && d2 < max_radius2)) { +// if (cnt == 0) { +// for (int l = 0; l < nsample; ++l) { +// idx[l] = i; +// dist2[l] = d2; +// } +// } +// idx[cnt] = i; +// ++cnt; +// if (cnt >= nsample) break; +// } +// } +// } + + +void ball_query_cuda_launcher(int m, int nsample, + float min_radius, float max_radius, + const float *xyz, const float *new_xyz, + const int *offset, const int *new_offset, + int *idx, float *dist2) { + // input: new_xyz: (m, 3), xyz: (n, 3), idx: (m, nsample) + dim3 blocks(DIVUP(m, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + ball_query_cuda_kernel<<>>(m, nsample, + min_radius, max_radius, + xyz, new_xyz, + offset, new_offset, + idx, dist2); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/ball_query/ball_query_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/ball_query/ball_query_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..03007a285a3559da85d099681f1316915e1d31b1 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/ball_query/ball_query_cuda_kernel.h @@ -0,0 +1,26 @@ +#ifndef _BALL_QUERY_CUDA_KERNEL +#define _BALL_QUERY_CUDA_KERNEL +#include +#include +#include + +void ball_query_cuda(int m, int nsample, + float min_radius, float max_radius, + at::Tensor xyz_tensor, at::Tensor new_xyz_tensor, + at::Tensor offset_tensor, at::Tensor new_offset_tensor, + at::Tensor idx_tensor, at::Tensor dist2_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void ball_query_cuda_launcher(int m, int nsample, + float min_radius, float max_radius, + const float *xyz, const float *new_xyz, + const int *offset, const int *new_offset, + int *idx, float *dist2); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/cuda_utils.h b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/cuda_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..bbfe7a06bf989056c0bd99e3e64fdbe7d15bb093 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/cuda_utils.h @@ -0,0 +1,23 @@ +#ifndef _CUDA_UTILS_H +#define _CUDA_UTILS_H + +#include +#include + +#define TOTAL_THREADS 1024 +#define THREADS_PER_BLOCK 512 +#define DIVUP(m, n) ((m) / (n) + ((m) % (n) > 0)) + +inline int opt_n_threads(int work_size) { + const int pow_2 = std::log(static_cast(work_size)) / std::log(2.0); + return std::max(std::min(1 << pow_2, TOTAL_THREADS), 1); +} + +inline dim3 opt_block_config(int x, int y) { + const int x_threads = opt_n_threads(x); + const int y_threads = std::max(std::min(opt_n_threads(y), TOTAL_THREADS / x_threads), 1); + dim3 block_config(x_threads, y_threads, 1); + return block_config; +} + +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/grouping/grouping_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/grouping/grouping_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6f7990adaf43f0a77050eed0d55adad19f256e10 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/grouping/grouping_cuda.cpp @@ -0,0 +1,21 @@ +#include +#include +#include +#include "grouping_cuda_kernel.h" + + +void grouping_forward_cuda(int m, int nsample, int c, at::Tensor input_tensor, at::Tensor idx_tensor, at::Tensor output_tensor) +{ + const float *input = input_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + grouping_forward_cuda_launcher(m, nsample, c, input, idx, output); +} + +void grouping_backward_cuda(int m, int nsample, int c, at::Tensor grad_output_tensor, at::Tensor idx_tensor, at::Tensor grad_input_tensor) +{ + const float *grad_output = grad_output_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + float *grad_input = grad_input_tensor.data_ptr(); + grouping_backward_cuda_launcher(m, nsample, c, grad_output, idx, grad_input); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/grouping/grouping_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/grouping/grouping_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..58ec0a21a2949f9f82504ccd24597c544c50af40 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/grouping/grouping_cuda_kernel.cu @@ -0,0 +1,40 @@ +#include "../cuda_utils.h" +#include "grouping_cuda_kernel.h" + + +__global__ void grouping_forward_cuda_kernel(int m, int nsample, int c, const float *__restrict__ input, const int *__restrict__ idx, float *__restrict__ output) { + // input: input: (n, c), idx: (m, nsample), output: (m, nsample, c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= m * nsample * c) return; + const int c_idx = index % c; + const int nsample_idx = (index / c) % nsample; + const int m_idx = index / nsample / c; + const int input_idx = idx[m_idx * nsample + nsample_idx] * c + c_idx; + output[index] = input[input_idx]; +} + +__global__ void grouping_backward_cuda_kernel(int m, int nsample, int c, const float *__restrict__ grad_output, const int *__restrict__ idx, float *__restrict__ grad_input) { + // input: grad_output: (m, nsample, c), idx: (m, nsample), output: grad_input: (n, c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= m * nsample * c) return; + const int c_idx = index % c; + const int nsample_idx = (index / c) % nsample; + const int m_idx = index / nsample / c; + const int input_idx = idx[m_idx * nsample + nsample_idx] * c + c_idx; + atomicAdd(grad_input + input_idx, grad_output[index]); +} + +void grouping_forward_cuda_launcher(int m, int nsample, int c, const float *input, const int *idx, float *output) { + // input: input: (n, c), idx: (m, nsample), output: (m, nsample, c) + dim3 blocks(DIVUP(m * nsample * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + grouping_forward_cuda_kernel<<>>(m, nsample, c, input, idx, output); +} + +void grouping_backward_cuda_launcher(int m, int nsample, int c, const float *grad_output, const int *idx, float *grad_input) +{ + // input: grad_output: (m, nsample, c), idx: (m, nsample), output: grad_input: (n, c) + dim3 blocks(DIVUP(m * nsample * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + grouping_backward_cuda_kernel<<>>(m, nsample, c, grad_output, idx, grad_input); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/grouping/grouping_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/grouping/grouping_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..3db4aaa9fad5811d559d47c500e4b00f0165d9b4 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/grouping/grouping_cuda_kernel.h @@ -0,0 +1,20 @@ +#ifndef _GROUPING_CUDA_KERNEL +#define _GROUPING_CUDA_KERNEL +#include +#include +#include + +void grouping_forward_cuda(int m, int nsample, int c, at::Tensor input_tensor, at::Tensor idx_tensor, at::Tensor output_tensor); +void grouping_backward_cuda(int m, int nsample, int c, at::Tensor grad_output_tensor, at::Tensor idx_tensor, at::Tensor grad_input_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void grouping_forward_cuda_launcher(int m, int nsample, int c, const float *input, const int *idx, float *output); +void grouping_backward_cuda_launcher(int m, int nsample, int c, const float *grad_output, const int *idx, float *grad_input); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/interpolation/interpolation_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/interpolation/interpolation_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f2c1b0078f4b70626705d7b3f5d1d65d37ee6de7 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/interpolation/interpolation_cuda.cpp @@ -0,0 +1,23 @@ +#include +#include +#include +#include "interpolation_cuda_kernel.h" + + +void interpolation_forward_cuda(int n, int c, int k, at::Tensor input_tensor, at::Tensor idx_tensor, at::Tensor weight_tensor, at::Tensor output_tensor) +{ + const float *input = input_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + const float *weight = weight_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + interpolation_forward_cuda_launcher(n, c, k, input, idx, weight, output); +} + +void interpolation_backward_cuda(int n, int c, int k, at::Tensor grad_output_tensor, at::Tensor idx_tensor, at::Tensor weight_tensor, at::Tensor grad_input_tensor) +{ + const float *grad_output = grad_output_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + const float *weight = weight_tensor.data_ptr(); + float *grad_input = grad_input_tensor.data_ptr(); + interpolation_backward_cuda_launcher(n, c, k, grad_output, idx, weight, grad_input); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/interpolation/interpolation_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/interpolation/interpolation_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..f560d8c92c6eac865b8c1e1dc27140fe3fcc2250 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/interpolation/interpolation_cuda_kernel.cu @@ -0,0 +1,47 @@ +#include "../cuda_utils.h" +#include "interpolation_cuda_kernel.h" + + +__global__ void interpolation_forward_cuda_kernel(int n, int c, int k, const float *input, const int *idx, const float *weight, float *output) +{ + // input: input: (m, c), idx: (n, k), weight: (n, k), output: output (n, c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= n * c) return; + int c_idx = index % c; + int n_idx = index / c; + for (int i = 0; i < k; i++) + { + int idx_idx = n_idx * k + i; + int input_idx = idx[idx_idx] * c + c_idx; + output[index] += input[input_idx] * weight[idx_idx]; + } +} + +__global__ void interpolation_backward_cuda_kernel(int n, int c, int k, const float *grad_output, const int *idx, const float *weight, float *grad_input) +{ + // input: grad_output: (n, c), idx: (n, k), weight: (n, k), output: grad_input (m, c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= n * c) return; + int c_idx = index % c; + int n_idx = index / c; + for (int i = 0; i < k; i++) + { + int idx_idx = n_idx * k + i; + int input_idx = idx[idx_idx] * c + c_idx; + atomicAdd(grad_input + input_idx, grad_output[index] * weight[idx_idx]); + } +} + +void interpolation_forward_cuda_launcher(int n, int c, int k, const float *input, const int *idx, const float *weight, float *output) { + // input: input: (m, c), idx: (n, k), weight: (n, k), output: output (n, c) + dim3 blocks(DIVUP(n * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + interpolation_forward_cuda_kernel<<>>(n, c, k, input, idx, weight, output); +} + +void interpolation_backward_cuda_launcher(int n, int c, int k, const float *grad_output, const int *idx, const float *weight, float *grad_input) { + // input: grad_output: (n, c), idx: (n, k), weight: (n, k), output: grad_input (m, c) + dim3 blocks(DIVUP(n * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + interpolation_backward_cuda_kernel<<>>(n, c, k, grad_output, idx, weight, grad_input); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/interpolation/interpolation_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/interpolation/interpolation_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..309e5dd0a34ccb58807bbf32389ba65e7ee6961b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/interpolation/interpolation_cuda_kernel.h @@ -0,0 +1,20 @@ +#ifndef _INTERPOLATION_CUDA_KERNEL +#define _INTERPOLATION_CUDA_KERNEL +#include +#include +#include + +void interpolation_forward_cuda(int n, int c, int k, at::Tensor input_tensor, at::Tensor idx_tensor, at::Tensor weight_tensor, at::Tensor output_tensor); +void interpolation_backward_cuda(int n, int c, int k, at::Tensor grad_output_tensor, at::Tensor idx_tensor, at::Tensor weight_tensor, at::Tensor grad_input_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void interpolation_forward_cuda_launcher(int n, int c, int k, const float *input, const int *idx, const float *weight, float *output); +void interpolation_backward_cuda_launcher(int n, int c, int k, const float *grad_output, const int *idx, const float *weight, float *grad_input); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/knn_query/knn_query_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/knn_query/knn_query_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bbe841ce0352fd234143b3b4978ec001522b31dd --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/knn_query/knn_query_cuda.cpp @@ -0,0 +1,16 @@ +#include +#include +#include +#include "knn_query_cuda_kernel.h" + + +void knn_query_cuda(int m, int nsample, at::Tensor xyz_tensor, at::Tensor new_xyz_tensor, at::Tensor offset_tensor, at::Tensor new_offset_tensor, at::Tensor idx_tensor, at::Tensor dist2_tensor) +{ + const float *xyz = xyz_tensor.data_ptr(); + const float *new_xyz = new_xyz_tensor.data_ptr(); + const int *offset = offset_tensor.data_ptr(); + const int *new_offset = new_offset_tensor.data_ptr(); + int *idx = idx_tensor.data_ptr(); + float *dist2 = dist2_tensor.data_ptr(); + knn_query_cuda_launcher(m, nsample, xyz, new_xyz, offset, new_offset, idx, dist2); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/knn_query/knn_query_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/knn_query/knn_query_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..297740237eae98cc4e61421bc261755d79b83142 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/knn_query/knn_query_cuda_kernel.cu @@ -0,0 +1,112 @@ +#include "../cuda_utils.h" +#include "knn_query_cuda_kernel.h" + + +namespace knn_query_utils{ + +template +__device__ void swap(DType *x, DType *y) +{ + DType tmp = *x; + *x = *y; + *y = tmp; +} + +__device__ void reheap(float *dist, int *idx, int k) +{ + int root = 0; + int child = root * 2 + 1; + while (child < k) + { + if(child + 1 < k && dist[child+1] > dist[child]) + child++; + if(dist[root] > dist[child]) + return; + swap(&dist[root], &dist[child]); + swap(&idx[root], &idx[child]); + root = child; + child = root * 2 + 1; + } +} + + +__device__ void heap_sort(float *dist, int *idx, int k) +{ + int i; + for (i = k - 1; i > 0; i--) + { + swap(&dist[0], &dist[i]); + swap(&idx[0], &idx[i]); + reheap(dist, idx, i); + } +} + + +__device__ int get_bt_idx(int idx, const int *offset) +{ + int i = 0; + while (1) + { + if (idx < offset[i]) + break; + else + i++; + } + return i; +} +} // namespace knn_query_utils + + +__global__ void knn_query_cuda_kernel(int m, int nsample, const float *__restrict__ xyz, const float *__restrict__ new_xyz, const int *__restrict__ offset, const int *__restrict__ new_offset, int *__restrict__ idx, float *__restrict__ dist2) { + // input: xyz (n, 3) new_xyz (m, 3) + // output: idx (m, nsample) dist2 (m, nsample) + int pt_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (pt_idx >= m) return; + + new_xyz += pt_idx * 3; + idx += pt_idx * nsample; + dist2 += pt_idx * nsample; + + int bt_idx = knn_query_utils::get_bt_idx(pt_idx, new_offset); + int start; + if (bt_idx == 0) + start = 0; + else + start = offset[bt_idx - 1]; + int end = offset[bt_idx]; + + float new_x = new_xyz[0]; + float new_y = new_xyz[1]; + float new_z = new_xyz[2]; + + float best_dist[128]; + int best_idx[128]; + for(int i = 0; i < nsample; i++){ + best_dist[i] = 1e10; + best_idx[i] = -1; + } + for(int i = start; i < end; i++){ + float x = xyz[i * 3 + 0]; + float y = xyz[i * 3 + 1]; + float z = xyz[i * 3 + 2]; + float d2 = (new_x - x) * (new_x - x) + (new_y - y) * (new_y - y) + (new_z - z) * (new_z - z); + if (d2 < best_dist[0]){ + best_dist[0] = d2; + best_idx[0] = i; + knn_query_utils::reheap(best_dist, best_idx, nsample); + } + } + knn_query_utils::heap_sort(best_dist, best_idx, nsample); + for(int i = 0; i < nsample; i++){ + idx[i] = best_idx[i]; + dist2[i] = best_dist[i]; + } +} + + +void knn_query_cuda_launcher(int m, int nsample, const float *xyz, const float *new_xyz, const int *offset, const int *new_offset, int *idx, float *dist2) { + // input: new_xyz: (m, 3), xyz: (n, 3), idx: (m, nsample) + dim3 blocks(DIVUP(m, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + knn_query_cuda_kernel<<>>(m, nsample, xyz, new_xyz, offset, new_offset, idx, dist2); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/knn_query/knn_query_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/knn_query/knn_query_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..c07c1cb46a56b7a37d55e25fb78816e034a8387e --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/knn_query/knn_query_cuda_kernel.h @@ -0,0 +1,18 @@ +#ifndef _KNN_QUERY_CUDA_KERNEL +#define _KNN_QUERY_CUDA_KERNEL +#include +#include +#include + +void knn_query_cuda(int m, int nsample, at::Tensor xyz_tensor, at::Tensor new_xyz_tensor, at::Tensor offset_tensor, at::Tensor new_offset_tensor, at::Tensor idx_tensor, at::Tensor dist2_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void knn_query_cuda_launcher(int m, int nsample, const float *xyz, const float *new_xyz, const int *offset, const int *new_offset, int *idx, float *dist2); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/pointops_api.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/pointops_api.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5ca4377607eb181d48d458d700f1df876294a848 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/pointops_api.cpp @@ -0,0 +1,32 @@ +#include +#include + +#include "knn_query/knn_query_cuda_kernel.h" +#include "ball_query/ball_query_cuda_kernel.h" +#include "random_ball_query/random_ball_query_cuda_kernel.h" +#include "sampling/sampling_cuda_kernel.h" +#include "grouping/grouping_cuda_kernel.h" +#include "interpolation/interpolation_cuda_kernel.h" +#include "aggregation/aggregation_cuda_kernel.h" +#include "subtraction/subtraction_cuda_kernel.h" +#include "attention/attention_cuda_kernel.h" + + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("knn_query_cuda", &knn_query_cuda, "knn_query_cuda"); + m.def("ball_query_cuda", &ball_query_cuda, "ball_query_cuda"); + m.def("random_ball_query_cuda", &random_ball_query_cuda, "random_ball_query_cuda"); + m.def("farthest_point_sampling_cuda", &farthest_point_sampling_cuda, "farthest_point_sampling_cuda"); + m.def("grouping_forward_cuda", &grouping_forward_cuda, "grouping_forward_cuda"); + m.def("grouping_backward_cuda", &grouping_backward_cuda, "grouping_backward_cuda"); + m.def("interpolation_forward_cuda", &interpolation_forward_cuda, "interpolation_forward_cuda"); + m.def("interpolation_backward_cuda", &interpolation_backward_cuda, "interpolation_backward_cuda"); + m.def("subtraction_forward_cuda", &subtraction_forward_cuda, "subtraction_forward_cuda"); + m.def("subtraction_backward_cuda", &subtraction_backward_cuda, "subtraction_backward_cuda"); + m.def("aggregation_forward_cuda", &aggregation_forward_cuda, "aggregation_forward_cuda"); + m.def("aggregation_backward_cuda", &aggregation_backward_cuda, "aggregation_backward_cuda"); + m.def("attention_relation_step_forward_cuda", &attention_relation_step_forward_cuda, "attention_relation_step_forward_cuda"); + m.def("attention_relation_step_backward_cuda", &attention_relation_step_backward_cuda, "attention_relation_step_backward_cuda"); + m.def("attention_fusion_step_forward_cuda", &attention_fusion_step_forward_cuda, "attention_fusion_step_forward_cuda"); + m.def("attention_fusion_step_backward_cuda", &attention_fusion_step_backward_cuda, "attention_fusion_step_backward_cuda"); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/random_ball_query/random_ball_query_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/random_ball_query/random_ball_query_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c2618c94b6b19175f044131cebeefe8a23152c47 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/random_ball_query/random_ball_query_cuda.cpp @@ -0,0 +1,21 @@ +#include +#include +#include +#include "random_ball_query_cuda_kernel.h" + + +void random_ball_query_cuda(int m, int nsample, + float min_radius, float max_radius, at::Tensor order_tensor, + at::Tensor xyz_tensor, at::Tensor new_xyz_tensor, + at::Tensor offset_tensor, at::Tensor new_offset_tensor, + at::Tensor idx_tensor, at::Tensor dist2_tensor) +{ + const int *order = order_tensor.data_ptr(); + const float *xyz = xyz_tensor.data_ptr(); + const float *new_xyz = new_xyz_tensor.data_ptr(); + const int *offset = offset_tensor.data_ptr(); + const int *new_offset = new_offset_tensor.data_ptr(); + int *idx = idx_tensor.data_ptr(); + float *dist2 = dist2_tensor.data_ptr(); + random_ball_query_cuda_launcher(m, nsample, min_radius, max_radius, order, xyz, new_xyz, offset, new_offset, idx, dist2); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/random_ball_query/random_ball_query_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/random_ball_query/random_ball_query_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..bfafb0f8b731e201783c94144cad9de3e11228ad --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/random_ball_query/random_ball_query_cuda_kernel.cu @@ -0,0 +1,123 @@ +#include "../cuda_utils.h" +#include "random_ball_query_cuda_kernel.h" + + +namespace random_ball_query_utils{ + +template +__device__ void swap(DType *x, DType *y) +{ + DType tmp = *x; + *x = *y; + *y = tmp; +} + +__device__ void reheap(float *dist, int *idx, int k) +{ + int root = 0; + int child = root * 2 + 1; + while (child < k) + { + if(child + 1 < k && dist[child+1] > dist[child]) + child++; + if(dist[root] > dist[child]) + return; + swap(&dist[root], &dist[child]); + swap(&idx[root], &idx[child]); + root = child; + child = root * 2 + 1; + } +} + + +__device__ void heap_sort(float *dist, int *idx, int k) +{ + int i; + for (i = k - 1; i > 0; i--) + { + swap(&dist[0], &dist[i]); + swap(&idx[0], &idx[i]); + reheap(dist, idx, i); + } +} + +__device__ int get_bt_idx(int idx, const int *offset) +{ + int i = 0; + while (1) + { + if (idx < offset[i]) + break; + else + i++; + } + return i; +} +} // namespace ball_query_utils + +__global__ void random_ball_query_cuda_kernel(int m, int nsample, + float min_radius, float max_radius, const int *__restrict__ order, + const float *__restrict__ xyz, const float *__restrict__ new_xyz, + const int *__restrict__ offset, const int *__restrict__ new_offset, + int *__restrict__ idx, float *__restrict__ dist2) { + // input: xyz (n, 3) new_xyz (m, 3) + // output: idx (m, nsample) dist (m, nsample) + int pt_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (pt_idx >= m) return; + + new_xyz += pt_idx * 3; + idx += pt_idx * nsample; + dist2 += pt_idx * nsample; + + int bt_idx = random_ball_query_utils::get_bt_idx(pt_idx, new_offset); + int start; + if (bt_idx == 0) + start = 0; + else + start = offset[bt_idx - 1]; + int end = offset[bt_idx]; + + float max_radius2 = max_radius * max_radius; + float min_radius2 = min_radius * min_radius; + float new_x = new_xyz[0]; + float new_y = new_xyz[1]; + float new_z = new_xyz[2]; + + int cnt = 0; + + for(int i = start; i < end; i++){ + float x = xyz[order[i] * 3 + 0]; + float y = xyz[order[i] * 3 + 1]; + float z = xyz[order[i] * 3 + 2]; + float d2 = (new_x - x) * (new_x - x) + (new_y - y) * (new_y - y) + (new_z - z) * (new_z - z); + + if (d2 <= 1e-5 || (d2 >= min_radius2 && d2 < max_radius2)){ + dist2[cnt] = d2; + idx[cnt] = order[i]; + cnt += 1; + if (cnt >= nsample) break; + } + } + + if (cnt < nsample) { + for (int i = cnt; i < nsample; i++){ + idx[i] = -1; + dist2[i] = 1e10; + } + } +} + +void random_ball_query_cuda_launcher(int m, int nsample, + float min_radius, float max_radius, const int *order, + const float *xyz, const float *new_xyz, + const int *offset, const int *new_offset, + int *idx, float *dist2) { + // input: new_xyz: (m, 3), xyz: (n, 3), idx: (m, nsample) + dim3 blocks(DIVUP(m, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + random_ball_query_cuda_kernel<<>>(m, nsample, + min_radius, max_radius, order, + xyz, new_xyz, + offset, new_offset, + idx, dist2); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/random_ball_query/random_ball_query_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/random_ball_query/random_ball_query_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..d3e35be21933d95b50e9c42150067071502bbc1e --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/random_ball_query/random_ball_query_cuda_kernel.h @@ -0,0 +1,26 @@ +#ifndef _RANDOM_BALL_QUERY_CUDA_KERNEL +#define _RANDOM_BALL_QUERY_CUDA_KERNEL +#include +#include +#include + +void random_ball_query_cuda(int m, int nsample, + float min_radius, float max_radius, at::Tensor order_tensor, + at::Tensor xyz_tensor, at::Tensor new_xyz_tensor, + at::Tensor offset_tensor, at::Tensor new_offset_tensor, + at::Tensor idx_tensor, at::Tensor dist2_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void random_ball_query_cuda_launcher(int m, int nsample, + float min_radius, float max_radius, const int *order, + const float *xyz, const float *new_xyz, + const int *offset, const int *new_offset, + int *idx, float *dist2); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/sampling/sampling_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/sampling/sampling_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7dc8094c3343f874457fd23d1506b25fd006fd0b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/sampling/sampling_cuda.cpp @@ -0,0 +1,15 @@ +#include +#include +#include +#include "sampling_cuda_kernel.h" + + +void farthest_point_sampling_cuda(int b, int n, at::Tensor xyz_tensor, at::Tensor offset_tensor, at::Tensor new_offset_tensor, at::Tensor tmp_tensor, at::Tensor idx_tensor) +{ + const float *xyz = xyz_tensor.data_ptr(); + const int *offset = offset_tensor.data_ptr(); + const int *new_offset = new_offset_tensor.data_ptr(); + float *tmp = tmp_tensor.data_ptr(); + int *idx = idx_tensor.data_ptr(); + farthest_point_sampling_cuda_launcher(b, n, xyz, offset, new_offset, tmp, idx); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/sampling/sampling_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/sampling/sampling_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..9a8676876672f68cd94913a0500d64813133b387 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/sampling/sampling_cuda_kernel.cu @@ -0,0 +1,171 @@ +#include "../cuda_utils.h" +#include "sampling_cuda_kernel.h" + + +__device__ void __update(float *dists, int *dists_i, int idx1, int idx2) { + const float v1 = dists[idx1], v2 = dists[idx2]; + const int i1 = dists_i[idx1], i2 = dists_i[idx2]; + dists[idx1] = max(v1, v2); + dists_i[idx1] = v2 > v1 ? i2 : i1; +} + +// input xyz: (n, 3), tmp: (b, n_max) +// ouput idx (m) +template +__global__ void farthest_point_sampling_cuda_kernel(const float *xyz, const int *offset, const int *new_offset, float *tmp, int *idx) +{ + __shared__ float dists[block_size]; + __shared__ int dists_i[block_size]; + + int bid = blockIdx.x; + int start_n, end_n, start_m, end_m, old; + if (bid == 0) { + start_n = 0; + end_n = offset[0]; + start_m = 0; + end_m = new_offset[0]; + old = 0; + } + else { + start_n = offset[bid - 1]; + end_n = offset[bid]; + start_m = new_offset[bid - 1]; + end_m = new_offset[bid]; + old = offset[bid - 1]; + } + + const int stride = block_size; + int tid = threadIdx.x; + if (tid == 0) idx[start_m] = start_n; + + __syncthreads(); + for (int j = start_m + 1; j < end_m; j++) + { + int besti = start_n; + float best = -1; + float x1 = xyz[old * 3 + 0]; + float y1 = xyz[old * 3 + 1]; + float z1 = xyz[old * 3 + 2]; + for (int k = start_n + tid; k < end_n; k += stride) + { + float x2 = xyz[k * 3 + 0]; + float y2 = xyz[k * 3 + 1]; + float z2 = xyz[k * 3 + 2]; + float d = (x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1) + (z2 - z1) * (z2 - z1); + float d2 = min(d, tmp[k]); + tmp[k] = d2; + besti = d2 > best ? k : besti; + best = d2 > best ? d2 : best; + } + dists[tid] = best; + dists_i[tid] = besti; + __syncthreads(); + + if (block_size >= 1024) { + if (tid < 512) { + __update(dists, dists_i, tid, tid + 512); + } + __syncthreads(); + } + if (block_size >= 512) { + if (tid < 256) { + __update(dists, dists_i, tid, tid + 256); + } + __syncthreads(); + } + if (block_size >= 256) { + if (tid < 128) { + __update(dists, dists_i, tid, tid + 128); + } + __syncthreads(); + } + if (block_size >= 128) { + if (tid < 64) { + __update(dists, dists_i, tid, tid + 64); + } + __syncthreads(); + } + if (block_size >= 64) { + if (tid < 32) { + __update(dists, dists_i, tid, tid + 32); + } + __syncthreads(); + } + if (block_size >= 32) { + if (tid < 16) { + __update(dists, dists_i, tid, tid + 16); + } + __syncthreads(); + } + if (block_size >= 16) { + if (tid < 8) { + __update(dists, dists_i, tid, tid + 8); + } + __syncthreads(); + } + if (block_size >= 8) { + if (tid < 4) { + __update(dists, dists_i, tid, tid + 4); + } + __syncthreads(); + } + if (block_size >= 4) { + if (tid < 2) { + __update(dists, dists_i, tid, tid + 2); + } + __syncthreads(); + } + if (block_size >= 2) { + if (tid < 1) { + __update(dists, dists_i, tid, tid + 1); + } + __syncthreads(); + } + + old = dists_i[0]; + if (tid == 0) + idx[j] = old; + } +} + +void farthest_point_sampling_cuda_launcher(int b, int n, const float *xyz, const int *offset, const int *new_offset, float *tmp, int *idx) +{ + unsigned int n_threads = opt_n_threads(n); + switch (n_threads) { + case 1024: + farthest_point_sampling_cuda_kernel<1024><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 512: + farthest_point_sampling_cuda_kernel<512><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 256: + farthest_point_sampling_cuda_kernel<256><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 128: + farthest_point_sampling_cuda_kernel<128><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 64: + farthest_point_sampling_cuda_kernel<64><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 32: + farthest_point_sampling_cuda_kernel<32><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 16: + farthest_point_sampling_cuda_kernel<16><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 8: + farthest_point_sampling_cuda_kernel<8><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 4: + farthest_point_sampling_cuda_kernel<4><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 2: + farthest_point_sampling_cuda_kernel<2><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 1: + farthest_point_sampling_cuda_kernel<1><<>>(xyz, offset, new_offset, tmp, idx); + break; + default: + farthest_point_sampling_cuda_kernel<512><<>>(xyz, offset, new_offset, tmp, idx); + } +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/sampling/sampling_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/sampling/sampling_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..f0e07607394a10b2b70c29f7497589d5edb8aab3 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/sampling/sampling_cuda_kernel.h @@ -0,0 +1,18 @@ +#ifndef _SAMPLING_CUDA_KERNEL +#define _SAMPLING_CUDA_KERNEL +#include +#include +#include + +void farthest_point_sampling_cuda(int b, int n, at::Tensor xyz_tensor, at::Tensor offset_tensor, at::Tensor new_offset_tensor, at::Tensor tmp_tensor, at::Tensor idx_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void farthest_point_sampling_cuda_launcher(int b, int n, const float *xyz, const int *offset, const int *new_offset, float *tmp, int *idx); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/subtraction/subtraction_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/subtraction/subtraction_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b343857a1671eafe5199089973e863e2ac5b618c --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/subtraction/subtraction_cuda.cpp @@ -0,0 +1,23 @@ +#include +#include +#include +#include "subtraction_cuda_kernel.h" + + +void subtraction_forward_cuda(int n, int nsample, int c, at::Tensor input1_tensor, at::Tensor input2_tensor, at::Tensor idx_tensor, at::Tensor output_tensor) +{ + const float *input1 = input1_tensor.data_ptr(); + const float *input2 = input2_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + subtraction_forward_cuda_launcher(n, nsample, c, input1, input2, idx, output); +} + +void subtraction_backward_cuda(int n, int nsample, int c, at::Tensor idx_tensor, at::Tensor grad_output_tensor, at::Tensor grad_input1_tensor, at::Tensor grad_input2_tensor) +{ + const int *idx = idx_tensor.data_ptr(); + const float *grad_output = grad_output_tensor.data_ptr(); + float *grad_input1 = grad_input1_tensor.data_ptr(); + float *grad_input2 = grad_input2_tensor.data_ptr(); + subtraction_backward_cuda_launcher(n, nsample, c, idx, grad_output, grad_input1, grad_input2); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/subtraction/subtraction_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/subtraction/subtraction_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..9b8d4f752940d580ee2b49f1b2946a8d6386d11a --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/subtraction/subtraction_cuda_kernel.cu @@ -0,0 +1,44 @@ +#include "../cuda_utils.h" +#include "subtraction_cuda_kernel.h" + + +__global__ void subtraction_forward_cuda_kernel(int n, int nsample, int c, const float *input1, const float *input2, const int *idx, float *output) { + // input: input1: (n, c), input2: (n, c), idx: (n, nsample), output: (n, nsample, c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= n * nsample * c) return; + const int c_idx = index % c; + const int nsample_idx = (index / c) % nsample; + const int n_idx = index / nsample / c; + const int idx_idx = n_idx * nsample + nsample_idx; + const int input1_idx = n_idx * c + c_idx; + const int input2_idx = idx[idx_idx] * c + c_idx; + output[index] = input1[input1_idx] - input2[input2_idx]; +} + +__global__ void subtraction_backward_cuda_kernel(int n, int nsample, int c, const int *idx, const float *grad_output, float *grad_input1, float *grad_input2) { + // input: grad_output: (n, nsample, c), output: grad_input1: (n, c), grad_input2: (n, c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= n * nsample * c) return; + const int c_idx = index % c; + const int nsample_idx = (index / c) % nsample; + const int n_idx = index / nsample / c; + const int idx_idx = n_idx * nsample + nsample_idx; + const int input1_idx = n_idx * c + c_idx; + const int input2_idx = idx[idx_idx] * c + c_idx; + atomicAdd(grad_input1 + input1_idx, grad_output[index]); + atomicAdd(grad_input2 + input2_idx, -grad_output[index]); +} + +void subtraction_forward_cuda_launcher(int n, int nsample, int c, const float *input1, const float *input2, const int *idx, float *output) { + // input: input1: (n, c), input2: (n, c), idx: (n, nsample), output: (n, nsample, c) + dim3 blocks(DIVUP(n * nsample * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + subtraction_forward_cuda_kernel<<>>(n, nsample, c, input1, input2, idx, output); +} + +void subtraction_backward_cuda_launcher(int n, int nsample, int c, const int *idx, const float *grad_output, float *grad_input1, float *grad_input2) { + // input: grad_output: (n, nsample, c), output: grad_input1: (n, c), grad_input2: (n, c) + dim3 blocks(DIVUP(n * nsample * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + subtraction_backward_cuda_kernel<<>>(n, nsample, c, idx, grad_output, grad_input1, grad_input2); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops/src/subtraction/subtraction_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/subtraction/subtraction_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..856133d97bdd3dc58f29c746ff240fc9d489c22e --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops/src/subtraction/subtraction_cuda_kernel.h @@ -0,0 +1,20 @@ +#ifndef _SUBTRACTION_CUDA_KERNEL +#define _SUBTRACTION_CUDA_KERNEL +#include +#include +#include + +void subtraction_forward_cuda(int n, int nsample, int c, at::Tensor input1_tensor, at::Tensor input2_tensor, at::Tensor idx_tensor, at::Tensor output_tensor); +void subtraction_backward_cuda(int n, int nsample, int c, at::Tensor idx_tensor, at::Tensor grad_output_tensor, at::Tensor grad_input1_tensor, at::Tensor grad_input2_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void subtraction_forward_cuda_launcher(int n, int nsample, int c, const float *input1, const float *input2, const int *idx, float *output); +void subtraction_backward_cuda_launcher(int n, int nsample, int c, const int *idx, const float *grad_output, float *grad_input1, float *grad_input2); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/__init__.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/__init__.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..25a2367da463d2f32b923166b48396d7292ad1f2 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/__init__.py @@ -0,0 +1 @@ +from pointops2 import * diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/pointops.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/pointops.py new file mode 100644 index 0000000000000000000000000000000000000000..efda900b3e702c4e5f8576baad5f4168cb756ee9 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/pointops.py @@ -0,0 +1,1193 @@ +""" +The part of attention operations is written by Xin Lai. +Email: xinlai@cse.cuhk.edu.hk +""" + +from typing import Tuple + +import torch +from torch.autograd import Function +import torch.nn as nn + +import pointops2_cuda as pointops_cuda +import time + + +class FurthestSampling(Function): + @staticmethod + def forward(ctx, xyz, offset, new_offset): + """ + input: xyz: (n, 3), offset: (b), new_offset: (b) + output: idx: (m) + """ + assert xyz.is_contiguous() + n, b, n_max = xyz.shape[0], offset.shape[0], offset[0] + for i in range(1, b): + n_max = max(offset[i] - offset[i - 1], n_max) + idx = torch.cuda.IntTensor(new_offset[b - 1].item()).zero_() + tmp = torch.cuda.FloatTensor(n).fill_(1e10) + pointops_cuda.furthestsampling_cuda(b, n_max, xyz, offset, new_offset, tmp, idx) + del tmp + return idx + + +furthestsampling = FurthestSampling.apply + + +class KNNQuery(Function): + @staticmethod + def forward(ctx, nsample, xyz, new_xyz, offset, new_offset): + """ + input: xyz: (n, 3), new_xyz: (m, 3), offset: (b), new_offset: (b) + output: idx: (m, nsample), dist2: (m, nsample) + """ + if new_xyz is None: + new_xyz = xyz + assert xyz.is_contiguous() and new_xyz.is_contiguous() + m = new_xyz.shape[0] + idx = torch.cuda.IntTensor(m, nsample).zero_() + dist2 = torch.cuda.FloatTensor(m, nsample).zero_() + pointops_cuda.knnquery_cuda( + m, nsample, xyz, new_xyz, offset, new_offset, idx, dist2 + ) + return idx, torch.sqrt(dist2) + + +knnquery = KNNQuery.apply + + +class Grouping(Function): + @staticmethod + def forward(ctx, input, idx): + """ + input: input: (n, c), idx : (m, nsample) + output: (m, nsample, c) + """ + assert input.is_contiguous() and idx.is_contiguous() + m, nsample, n, c = idx.shape[0], idx.shape[1], input.shape[0], input.shape[1] + output = torch.cuda.FloatTensor(m, nsample, c) + pointops_cuda.grouping_forward_cuda(m, nsample, c, input, idx, output) + ctx.n = n + ctx.save_for_backward(idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_out: (m, c, nsample) + output: (n, c), None + """ + n = ctx.n + (idx,) = ctx.saved_tensors + m, nsample, c = grad_output.shape + grad_input = torch.cuda.FloatTensor(n, c).zero_() + pointops_cuda.grouping_backward_cuda( + m, nsample, c, grad_output, idx, grad_input + ) + return grad_input, None + + +grouping = Grouping.apply + + +class AttentionStep1(Function): + @staticmethod + def forward(ctx, q, k, index0, index1): + """ + input: q: (N, h, C//h), k: (N, h, C//h), index0: (M), index1: (M) + output: output: [N, h, C//h] + """ + assert ( + q.is_contiguous() + and k.is_contiguous() + and index0.is_contiguous() + and index1.is_contiguous() + ) + + N_q, h, C_div_h = q.shape + N_k = k.shape[0] + M = index0.shape[0] + C = int(C_div_h * h) + + output = torch.cuda.FloatTensor(M, h).zero_() + pointops_cuda.attention_step1_forward_cuda( + N_k, M, h, C, q, k, index0, index1, output + ) + ctx.N_q = N_q + ctx.N_k = N_k + ctx.C = C + ctx.save_for_backward(q, k, index0, index1) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_output: (N, h, C//h) + output: (M, h), (N, h, C//h), None, None + """ + + N_q = ctx.N_q + N_k = ctx.N_k + C = ctx.C + q, k, index0, index1 = ctx.saved_tensors + M, h = grad_output.shape + + grad_output = grad_output.contiguous() + # print("grad_output.is_contiguous(): ", grad_output.is_contiguous()) + assert ( + q.is_contiguous() + and k.is_contiguous() + and index0.is_contiguous() + and index1.is_contiguous() + and grad_output.is_contiguous() + ) + + # print("back: attn[:5,:5]: ", attn[:5, :5]) + + # print("attn.shape: {} v.shape: {}, index0.shape: {}, index1.shape: {}".format(attn.shape, v.shape, index0.shape, index1.shape)) + + grad_q = torch.cuda.FloatTensor(N_q, h, C // h).zero_() + grad_k = torch.cuda.FloatTensor(N_k, h, C // h).zero_() + + # torch.cuda.synchronize() + # start = time.time() + + pointops_cuda.attention_step1_backward_cuda( + N_q, M, h, C, grad_output, index0, index1, q, k, grad_q, grad_k + ) + + # torch.cuda.synchronize() + # end = time.time() + # print("time v7: {}".format(end - start)) + # # input() + + return grad_q, grad_k, None, None + + +attention_step1 = AttentionStep1.apply + + +class AttentionStep1_v2(Function): + @staticmethod + def forward(ctx, q, k, index1, index0_offsets, n_max): + """ + input: q: (N, h, C//h), k: (N, h, C//h), index0: (M), index1: (M) + output: output: [N, h, C//h] + """ + assert ( + q.is_contiguous() + and k.is_contiguous() + and index0_offsets.is_contiguous() + and index1.is_contiguous() + ) + assert n_max <= 1024 + + N_q, h, C_div_h = q.shape + N_k = k.shape[0] + M = index1.shape[0] + C = int(C_div_h * h) + + output = torch.cuda.FloatTensor(M, h).zero_() + pointops_cuda.attention_step1_forward_cuda_v2( + N_k, M, h, C, n_max, q, k, index0_offsets, index1, output + ) + ctx.N_q = N_q + ctx.N_k = N_k + ctx.C = C + ctx.n_max = n_max + ctx.save_for_backward(q, k, index0_offsets, index1) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_output: (N, h, C//h) + output: (M, h), (N, h, C//h), None, None + """ + + N_q = ctx.N_q + N_k = ctx.N_k + C = ctx.C + n_max = ctx.n_max + q, k, index0_offsets, index1 = ctx.saved_tensors + M, h = grad_output.shape + + grad_output = grad_output.contiguous() + # print("grad_output.is_contiguous(): ", grad_output.is_contiguous()) + assert ( + q.is_contiguous() + and k.is_contiguous() + and index0_offsets.is_contiguous() + and index1.is_contiguous() + and grad_output.is_contiguous() + ) + + # print("back: attn[:5,:5]: ", attn[:5, :5]) + + # print("attn.shape: {} v.shape: {}, index0.shape: {}, index1.shape: {}".format(attn.shape, v.shape, index0.shape, index1.shape)) + + grad_q = torch.cuda.FloatTensor(N_q, h, C // h).zero_() + grad_k = torch.cuda.FloatTensor(N_k, h, C // h).zero_() + + # torch.cuda.synchronize() + # start = time.time() + + pointops_cuda.attention_step1_backward_cuda_v2( + N_q, + M, + h, + C, + n_max, + grad_output, + index0_offsets, + index1, + q, + k, + grad_q, + grad_k, + ) + + # torch.cuda.synchronize() + # end = time.time() + # print("time v7: {}".format(end - start)) + # # input() + + return grad_q, grad_k, None, None, None + + +attention_step1_v2 = AttentionStep1_v2.apply + + +class AttentionStep2(Function): + @staticmethod + def forward(ctx, attn, v, index0, index1): + """ + input: attn: (M, h), v: (N, h, C//h), index0: (M), index1: (M) + output: output: [N, h, C//h] + """ + assert ( + attn.is_contiguous() + and v.is_contiguous() + and index0.is_contiguous() + and index1.is_contiguous() + ) + + M, h = attn.shape + N_q = index0.max().item() + 1 + N_v, h, C_div_h = v.shape + C = int(C_div_h * h) + + output = torch.cuda.FloatTensor(N_q, h, C // h).zero_() + pointops_cuda.attention_step2_forward_cuda( + N_q, M, h, C, attn, v, index0, index1, output + ) + ctx.M = M + + # print("attn[:5,:5]: ", attn[:5, :5]) + + ctx.save_for_backward(attn, v, index0, index1) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_output: (N, h, C//h) + output: (M, h), (N, h, C//h), None, None + """ + M = ctx.M + attn, v, index0, index1 = ctx.saved_tensors + N_v = v.shape[0] + N_q, h, C_div_h = grad_output.shape + C = h * C_div_h + + grad_output = grad_output.contiguous() + # print("grad_output.is_contiguous(): ", grad_output.is_contiguous()) + assert ( + attn.is_contiguous() + and v.is_contiguous() + and index0.is_contiguous() + and index1.is_contiguous() + and grad_output.is_contiguous() + ) + + # print("back: attn[:5,:5]: ", attn[:5, :5]) + + # print("attn.shape: {} v.shape: {}, index0.shape: {}, index1.shape: {}".format(attn.shape, v.shape, index0.shape, index1.shape)) + + grad_attn = torch.cuda.FloatTensor(M, h).zero_() + grad_v = torch.cuda.FloatTensor(N_v, h, C // h).zero_() + + # torch.cuda.synchronize() + # start = time.time() + + pointops_cuda.attention_step2_backward_cuda( + N_q, M, h, C, grad_output, index0, index1, attn, v, grad_attn, grad_v + ) + + # torch.cuda.synchronize() + # end = time.time() + # print("time v8: {}".format(end - start)) + # # input() + + return grad_attn, grad_v, None, None + + +attention_step2 = AttentionStep2.apply + + +class AttentionStep2_v2(Function): + @staticmethod + def forward(ctx, attn, v, index0, index1): + """ + input: attn: (M, h), v: (N, h, C//h), index0: (M), index1: (M) + output: output: [L, h, C//h] + """ + assert ( + attn.is_contiguous() + and v.is_contiguous() + and index0.is_contiguous() + and index1.is_contiguous() + ) + + L = int(index0.max().item()) + 1 + + M, h = attn.shape + N, h, C_div_h = v.shape + C = int(C_div_h * h) + + output = torch.cuda.FloatTensor(L, h, C // h).zero_() + pointops_cuda.attention_step2_forward_cuda( + N, M, h, C, attn, v, index0, index1, output + ) + ctx.M = M + + # print("attn[:5,:5]: ", attn[:5, :5]) + + ctx.save_for_backward(attn, v, index0, index1) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_output: (L, h, C//h) + output: (M, h), (N, h, C//h), None, None + """ + M = ctx.M + attn, v, index0, index1 = ctx.saved_tensors + L, h, C_div_h = grad_output.shape + N = v.shape[0] + C = h * C_div_h + + grad_output = grad_output.contiguous() + # print("grad_output.is_contiguous(): ", grad_output.is_contiguous()) + assert ( + attn.is_contiguous() + and v.is_contiguous() + and index0.is_contiguous() + and index1.is_contiguous() + and grad_output.is_contiguous() + ) + + # print("back: attn[:5,:5]: ", attn[:5, :5]) + + # print("attn.shape: {} v.shape: {}, index0.shape: {}, index1.shape: {}".format(attn.shape, v.shape, index0.shape, index1.shape)) + + grad_attn = torch.cuda.FloatTensor(M, h).zero_() + grad_v = torch.cuda.FloatTensor(N, h, C // h).zero_() + + pointops_cuda.attention_step2_backward_cuda( + N, M, h, C, grad_output, index0, index1, attn, v, grad_attn, grad_v + ) + return grad_attn, grad_v, None, None + + +attention_step2_v2 = AttentionStep2_v2.apply + + +class DotProdWithIdx(Function): + @staticmethod + def forward(ctx, q, index, table, rel_idx): + """ + input: q: (N, h, hdim), index: (M), table: (L, h, hdim, 3), rel_idx: (M, 3) + output: output: [M, h] + """ + assert ( + q.is_contiguous() + and index.is_contiguous() + and table.is_contiguous() + and rel_idx.is_contiguous() + ) + + N, h, hdim = q.shape + M = index.shape[0] + + output = torch.cuda.FloatTensor(M, h).zero_() + pointops_cuda.dot_prod_with_idx_forward_cuda( + N, M, h, hdim, q, index, table, rel_idx, output + ) + ctx.save_for_backward(q, index, table, rel_idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_output: [M, h] + output: (N, h, hdim), None, (L, h, hdim, 3), None + """ + q, index, table, rel_idx = ctx.saved_tensors + M, h = grad_output.shape + N, _, hdim = q.shape + L = table.shape[0] + + grad_output = grad_output.contiguous() + assert ( + q.is_contiguous() + and index.is_contiguous() + and table.is_contiguous() + and rel_idx.is_contiguous() + and grad_output.is_contiguous() + ) + + # print("back: attn[:5,:5]: ", attn[:5, :5]) + + # print("attn.shape: {} v.shape: {}, index0.shape: {}, index1.shape: {}".format(attn.shape, v.shape, index0.shape, index1.shape)) + + grad_q = torch.cuda.FloatTensor(N, h, hdim).zero_() + grad_table = torch.cuda.FloatTensor(L, h, hdim, 3).zero_() + + # torch.cuda.synchronize() + # start = time.time() + + pointops_cuda.dot_prod_with_idx_backward_cuda( + N, M, h, hdim, grad_output, q, index, table, rel_idx, grad_q, grad_table + ) + + # torch.cuda.synchronize() + # end = time.time() + # print("time v9: {}".format(end - start)) + # # input() + + return grad_q, None, grad_table, None + + +dot_prod_with_idx = DotProdWithIdx.apply + + +class DotProdWithIdx_v2(Function): + @staticmethod + def forward(ctx, q, index_q, k, index_k, table_q, table_k, rel_idx): + """ + input: q: (N, h, hdim), index_q: (M), k: (N, h, hdim), index_k: (M), table_q: (L, h, hdim, 3), table_k: (L, h, hdim, 3), rel_idx: (M, 3) + output: output: [M, h] + """ + assert ( + q.is_contiguous() + and index_q.is_contiguous() + and k.is_contiguous() + and index_k.is_contiguous() + and table_q.is_contiguous() + and table_k.is_contiguous() + and rel_idx.is_contiguous() + ) + + N, h, hdim = q.shape + M = index_q.shape[0] + L = table_q.shape[0] + assert table_k.shape[0] == L and index_k.shape[0] == M + + # obtain the mapping from block_idx to m_idx + rel_idx_merge = ( + rel_idx[:, 0] + rel_idx[:, 1] * L + rel_idx[:, 2] * (L**2) + ) # [M, ] + sorted_values, sort_indices = torch.sort(rel_idx_merge) + _, counts = torch.unique_consecutive(sorted_values, return_counts=True) + rel_idx_offsets = torch.cumsum(counts, dim=-1) # [T,] + rel_idx_offsets = torch.cat( + [torch.zeros(1, dtype=torch.long).cuda(), rel_idx_offsets], 0 + ) # [T+1,] + n_max = counts.max() + T = counts.shape[0] + + # print("M: {}, L: {}, n_max: {}, T: {}".format(M, L, n_max, T)) + # print("rel_idx_merge.shape: {}, sorted_values.shape: {}".format(rel_idx_merge.shape, sorted_values.shape)) + # print("counts.shape: {}".format(counts.shape)) + + output = torch.cuda.FloatTensor(M, h).zero_() + # pointops_cuda.dot_prod_with_idx_forward_cuda(N, M, h, hdim, q, index, table, rel_idx, output) + pointops_cuda.dot_prod_with_idx_forward_cuda_v2( + N, + M, + h, + hdim, + n_max, + T, + q, + index_q, + k, + index_k, + table_q, + table_k, + rel_idx, + rel_idx_offsets.int(), + sort_indices.int(), + output, + ) + + ctx.n_max = n_max + ctx.T = T + ctx.save_for_backward( + q, + index_q, + k, + index_k, + table_q, + table_k, + rel_idx, + rel_idx_offsets, + sort_indices, + ) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_output: [M, h] + output: (N, h, hdim), None, (L, h, hdim, 3), None + """ + ( + q, + index_q, + k, + index_k, + table_q, + table_k, + rel_idx, + rel_idx_offsets, + sort_indices, + ) = ctx.saved_tensors + M, h = grad_output.shape + N, _, hdim = q.shape + L = table_q.shape[0] + T, n_max = ctx.T, ctx.n_max + + grad_output = grad_output.contiguous() + assert ( + q.is_contiguous() + and index_q.is_contiguous() + and k.is_contiguous() + and index_k.is_contiguous() + and table_q.is_contiguous() + and table_k.is_contiguous() + and rel_idx.is_contiguous() + and rel_idx_offsets.is_contiguous() + and sort_indices.is_contiguous() + and grad_output.is_contiguous() + ) + + # print("back: attn[:5,:5]: ", attn[:5, :5]) + + # print("attn.shape: {} v.shape: {}, index0.shape: {}, index1.shape: {}".format(attn.shape, v.shape, index0.shape, index1.shape)) + + grad_q = torch.cuda.FloatTensor(N, h, hdim).zero_() + grad_table_q = torch.cuda.FloatTensor(L, h, hdim, 3).zero_() + grad_k = torch.cuda.FloatTensor(N, h, hdim).zero_() + grad_table_k = torch.cuda.FloatTensor(L, h, hdim, 3).zero_() + + # torch.cuda.synchronize() + # start = time.time() + + pointops_cuda.dot_prod_with_idx_backward_cuda_v2( + N, + M, + h, + hdim, + n_max, + T, + grad_output, + q, + index_q, + k, + index_k, + table_q, + table_k, + rel_idx, + rel_idx_offsets.int(), + sort_indices.int(), + grad_q, + grad_k, + grad_table_q, + grad_table_k, + ) + + # torch.cuda.synchronize() + # end = time.time() + # print("time v9: {}".format(end - start)) + # # input() + return grad_q, None, grad_k, None, grad_table_q, grad_table_k, None + + +dot_prod_with_idx_v2 = DotProdWithIdx_v2.apply + + +class DotProdWithIdx_v3(Function): + @staticmethod + def forward(ctx, q, index_q_offsets, n_max, k, index_k, table_q, table_k, rel_idx): + """ + input: q: (N, h, hdim), index_q: (M), k: (N, h, hdim), index_k: (M), table_q: (L, h, hdim, 3), table_k: (L, h, hdim, 3), rel_idx: (M, 3) + output: output: [M, h] + """ + assert ( + q.is_contiguous() + and index_q_offsets.is_contiguous() + and k.is_contiguous() + and index_k.is_contiguous() + and table_q.is_contiguous() + and table_k.is_contiguous() + and rel_idx.is_contiguous() + ) + + N, h, hdim = q.shape + M = index_k.shape[0] + L = table_q.shape[0] + assert table_k.shape[0] == L + + # # obtain the mapping from block_idx to m_idx + # rel_idx_merge = rel_idx[:, 0] + rel_idx[:, 1] * L + rel_idx[:, 2] * (L ** 2) #[M, ] + # sorted_values, sort_indices = torch.sort(rel_idx_merge) + # _, counts = torch.unique_consecutive(sorted_values, return_counts=True) + # rel_idx_offsets = torch.cumsum(counts, dim=-1) #[T,] + # rel_idx_offsets = torch.cat([torch.zeros(1, dtype=torch.long).cuda(), rel_idx_offsets], 0) #[T+1,] + # n_max = counts.max() + # T = counts.shape[0] + + # print("M: {}, L: {}, n_max: {}, T: {}".format(M, L, n_max, T)) + # print("rel_idx_merge.shape: {}, sorted_values.shape: {}".format(rel_idx_merge.shape, sorted_values.shape)) + # print("counts.shape: {}".format(counts.shape)) + + # print("M: {}, L: {}, n_max: {}".format(M, L, n_max)) + + output = torch.cuda.FloatTensor(M, h).zero_() + # pointops_cuda.dot_prod_with_idx_forward_cuda(N, M, h, hdim, q, index, table, rel_idx, output) + pointops_cuda.dot_prod_with_idx_forward_cuda_v3( + N, + M, + h, + hdim, + n_max, + q, + index_q_offsets, + k, + index_k, + table_q, + table_k, + rel_idx, + output, + ) + + ctx.n_max = n_max + # ctx.T = T + ctx.save_for_backward(q, index_q_offsets, k, index_k, table_q, table_k, rel_idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_output: [M, h] + output: (N, h, hdim), None, (L, h, hdim, 3), None + """ + q, index_q_offsets, k, index_k, table_q, table_k, rel_idx = ctx.saved_tensors + M, h = grad_output.shape + N, _, hdim = q.shape + L = table_q.shape[0] + n_max = ctx.n_max + + grad_output = grad_output.contiguous() + assert ( + q.is_contiguous() + and index_q_offsets.is_contiguous() + and k.is_contiguous() + and index_k.is_contiguous() + and table_q.is_contiguous() + and table_k.is_contiguous() + and rel_idx.is_contiguous() + and grad_output.is_contiguous() + ) + + # print("back: attn[:5,:5]: ", attn[:5, :5]) + + # print("attn.shape: {} v.shape: {}, index0.shape: {}, index1.shape: {}".format(attn.shape, v.shape, index0.shape, index1.shape)) + + grad_q = torch.cuda.FloatTensor(N, h, hdim).zero_() + grad_table_q = torch.cuda.FloatTensor(L, h, hdim, 3).zero_() + grad_k = torch.cuda.FloatTensor(N, h, hdim).zero_() + grad_table_k = torch.cuda.FloatTensor(L, h, hdim, 3).zero_() + + # torch.cuda.synchronize() + # start = time.time() + + pointops_cuda.dot_prod_with_idx_backward_cuda_v3( + N, + M, + h, + hdim, + n_max, + grad_output, + q, + index_q_offsets, + k, + index_k, + table_q, + table_k, + rel_idx, + grad_q, + grad_k, + grad_table_q, + grad_table_k, + ) + + # torch.cuda.synchronize() + # end = time.time() + # print("time v9: {}".format(end - start)) + # # input() + return grad_q, None, None, grad_k, None, grad_table_q, grad_table_k, None + + +dot_prod_with_idx_v3 = DotProdWithIdx_v3.apply + + +class AttentionStep2WithRelPosValue(Function): + @staticmethod + def forward(ctx, attn, v, index0, index1, table, rel_idx): + """ + input: attn: (M, h), v: (N, h, hdim), index0: (M), index1: (M), table: (L, h, hdim, 3), rel_idx: (M, 3) + output: output: [N, h, hdim] + """ + assert ( + attn.is_contiguous() + and v.is_contiguous() + and index0.is_contiguous() + and index1.is_contiguous() + and table.is_contiguous() + and rel_idx.is_contiguous() + ) + + M, h = attn.shape + N_v, h, hdim = v.shape + N_q = index0.max().item() + 1 + + output = torch.cuda.FloatTensor(N_q, h, hdim).zero_() + pointops_cuda.attention_step2_with_rel_pos_value_forward_cuda( + N_q, M, h, hdim, attn, v, index0, index1, table, rel_idx, output + ) + + # print("attn[:5,:5]: ", attn[:5, :5]) + + ctx.save_for_backward(attn, v, index0, index1, table, rel_idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_output: (N, h, C//h) + output: (M, h), (N, h, C//h), None, None, (L, h, hdim, 3), None + """ + attn, v, index0, index1, table, rel_idx = ctx.saved_tensors + N_q, h, hdim = grad_output.shape + N_v = v.shape[0] + M = attn.shape[0] + L = table.shape[0] + + grad_output = grad_output.contiguous() + # print("grad_output.is_contiguous(): ", grad_output.is_contiguous()) + assert ( + attn.is_contiguous() + and v.is_contiguous() + and index0.is_contiguous() + and index1.is_contiguous() + and grad_output.is_contiguous() + and table.is_contiguous() + and rel_idx.is_contiguous() + ) + + # print("back: attn[:5,:5]: ", attn[:5, :5]) + + # print("attn.shape: {} v.shape: {}, index0.shape: {}, index1.shape: {}".format(attn.shape, v.shape, index0.shape, index1.shape)) + + grad_attn = torch.cuda.FloatTensor(M, h).zero_() + grad_v = torch.cuda.FloatTensor(N_v, h, hdim).zero_() + grad_table = torch.cuda.FloatTensor(L, h, hdim, 3).zero_() + + # print("attn.shape: {}, grad_attn.shape: {}".format(attn.shape, grad_attn.shape)) + # print("v.shape: {}, grad_v.shape: {}".format(v.shape, grad_v.shape)) + # print("table.shape: {}, grad_table.shape: {}".format(table.shape, grad_table.shape)) + + # torch.cuda.synchronize() + # start = time.time() + + pointops_cuda.attention_step2_with_rel_pos_value_backward_cuda( + N_q, + M, + h, + hdim, + grad_output, + index0, + index1, + attn, + v, + table, + rel_idx, + grad_attn, + grad_v, + grad_table, + ) + + # torch.cuda.synchronize() + # end = time.time() + # print("time v10: {}".format(end - start)) + # # input() + return grad_attn, grad_v, None, None, grad_table, None + + +attention_step2_with_rel_pos_value = AttentionStep2WithRelPosValue.apply + + +class AttentionStep2WithRelPosValue_v2(Function): + @staticmethod + def forward(ctx, attn, v, index0_offsets, n_max, index1, table, rel_idx): + """ + input: attn: (M, h), v: (N, h, hdim), index0_offsets: (M), index1: (M), table: (L, h, hdim, 3), rel_idx: (M, 3) + output: output: [N, h, hdim] + """ + assert ( + attn.is_contiguous() + and v.is_contiguous() + and index0_offsets.is_contiguous() + and index1.is_contiguous() + and table.is_contiguous() + and rel_idx.is_contiguous() + ) + + M, h = attn.shape + N, h, hdim = v.shape + # N_q = int(index0_offsets.max().item()) + 1 + + output = torch.cuda.FloatTensor(N, h, hdim).zero_() + pointops_cuda.attention_step2_with_rel_pos_value_forward_cuda_v2( + N, + M, + h, + hdim, + n_max, + attn, + v, + index0_offsets, + index1, + table, + rel_idx, + output, + ) + + # print("attn[:5,:5]: ", attn[:5, :5]) + + ctx.n_max = n_max + ctx.save_for_backward(attn, v, index0_offsets, index1, table, rel_idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_output: (N, h, C//h) + output: (M, h), (N, h, C//h), None, None, (L, h, hdim, 3), None + """ + n_max = ctx.n_max + attn, v, index0_offsets, index1, table, rel_idx = ctx.saved_tensors + N, h, hdim = grad_output.shape + N = v.shape[0] + M = attn.shape[0] + L = table.shape[0] + + # grad_output = grad_output.contiguous() + # print("grad_output.is_contiguous(): ", grad_output.is_contiguous()) + assert ( + attn.is_contiguous() + and v.is_contiguous() + and index0_offsets.is_contiguous() + and index1.is_contiguous() + and grad_output.is_contiguous() + and table.is_contiguous() + and rel_idx.is_contiguous() + ) + + # print("back: attn[:5,:5]: ", attn[:5, :5]) + + # print("attn.shape: {} v.shape: {}, index0_offsets.shape: {}, index1.shape: {}".format(attn.shape, v.shape, index0_offsets.shape, index1.shape)) + + grad_attn = torch.cuda.FloatTensor(M, h).zero_() + grad_v = torch.cuda.FloatTensor(N, h, hdim).zero_() + grad_table = torch.cuda.FloatTensor(L, h, hdim, 3).zero_() + + # print("attn.shape: {}, grad_attn.shape: {}".format(attn.shape, grad_attn.shape)) + # print("v.shape: {}, grad_v.shape: {}".format(v.shape, grad_v.shape)) + # print("table.shape: {}, grad_table.shape: {}".format(table.shape, grad_table.shape)) + + # torch.cuda.synchronize() + # start = time.time() + + pointops_cuda.attention_step2_with_rel_pos_value_backward_cuda_v2( + N, + M, + h, + hdim, + n_max, + grad_output, + index0_offsets, + index1, + attn, + v, + table, + rel_idx, + grad_attn, + grad_v, + grad_table, + ) + + # torch.cuda.synchronize() + # end = time.time() + # print("time v10: {}".format(end - start)) + + return grad_attn, grad_v, None, None, None, grad_table, None + + +attention_step2_with_rel_pos_value_v2 = AttentionStep2WithRelPosValue_v2.apply + + +def queryandgroup( + nsample, + xyz, + new_xyz, + feat, + idx, + offset, + new_offset, + use_xyz=True, + return_indx=False, +): + """ + input: xyz: (n, 3), new_xyz: (m, 3), feat: (n, c), idx: (m, nsample), offset: (b), new_offset: (b) + output: new_feat: (m, c+3, nsample), grouped_idx: (m, nsample) + """ + assert xyz.is_contiguous() and new_xyz.is_contiguous() and feat.is_contiguous() + if new_xyz is None: + new_xyz = xyz + if idx is None: + idx, _ = knnquery(nsample, xyz, new_xyz, offset, new_offset) # (m, nsample) + + n, m, c = xyz.shape[0], new_xyz.shape[0], feat.shape[1] + grouped_xyz = xyz[idx.view(-1).long(), :].view(m, nsample, 3) # (m, nsample, 3) + # grouped_xyz = grouping(xyz, idx) # (m, nsample, 3) + # 相对位置 + grouped_xyz -= new_xyz.unsqueeze(1) # (m, nsample, 3) + grouped_feat = feat[idx.view(-1).long(), :].view(m, nsample, c) # (m, nsample, c) + # grouped_feat = grouping(feat, idx) # (m, nsample, c) + if use_xyz: + if return_indx: + return torch.cat((grouped_xyz, grouped_feat), -1), idx # (m, nsample, 3+c) + else: + return torch.cat((grouped_xyz, grouped_feat), -1) + else: + if return_indx: + return grouped_feat, idx + else: + return grouped_feat + + +def Divide2Patch(nsample, xyz, offset, return_offset=False, anchor_scale=None): + # nsample: 16 xyz: (n, 3) offset: (b) + downsample_scale = anchor_scale or nsample + new_offset, count = [offset[0].item() // downsample_scale], offset[ + 0 + ].item() // downsample_scale + for i in range(1, offset.shape[0]): + count += (offset[i].item() - offset[i - 1].item()) // downsample_scale + new_offset.append(count) + # print("donw sample scale:", downsample_scale,"offset:", offset, "newoffset:", new_offset) + new_offset = torch.cuda.IntTensor(new_offset) + idx = furthestsampling(xyz, offset, new_offset) # (m) + new_xyz = xyz[idx.long()] + p_idx, _ = knnquery(nsample, xyz, new_xyz, offset, new_offset) # (m, nsample) + if return_offset: + return p_idx, new_offset + else: + return p_idx + + +class Subtraction(Function): + @staticmethod + def forward(ctx, input1, input2, idx): + """ + input: input1: (n, c), input2: (n, c), idx: (n, nsample) + output: (n, nsample, c) + """ + assert input1.is_contiguous() and input2.is_contiguous() + n, c = input1.shape + nsample = idx.shape[-1] + output = torch.cuda.FloatTensor(n, nsample, c).zero_() + pointops_cuda.subtraction_forward_cuda( + n, nsample, c, input1, input2, idx, output + ) + ctx.save_for_backward(idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_out: (n, nsample, c) + output: grad_input1: (n, c), grad_input2: (n, c) + """ + (idx,) = ctx.saved_tensors + n, nsample, c = grad_output.shape + grad_input1 = torch.cuda.FloatTensor(n, c).zero_() + grad_input2 = torch.cuda.FloatTensor(n, c).zero_() + pointops_cuda.subtraction_backward_cuda( + n, nsample, c, idx, grad_output, grad_input1, grad_input2 + ) + return grad_input1, grad_input2, None + + +subtraction = Subtraction.apply + + +class Aggregation(Function): + @staticmethod + def forward(ctx, input, position, weight, idx): + """ + input: input: (n, c), position: (n, nsample, c), weight : (n, nsample, c'), idx: (n, nsample) + output: (n, c) + """ + assert ( + input.is_contiguous() + and position.is_contiguous() + and weight.is_contiguous() + ) + n, nsample, c = position.shape + w_c = weight.shape[-1] + output = torch.cuda.FloatTensor(n, c).zero_() + pointops_cuda.aggregation_forward_cuda( + n, nsample, c, w_c, input, position, weight, idx, output + ) + ctx.save_for_backward(input, position, weight, idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_out: (n, c) + output: grad_input: (n, c), grad_position: (n, nsample, c), grad_weight : (n, nsample, c') + """ + input, position, weight, idx = ctx.saved_tensors + n, nsample, c = position.shape + w_c = weight.shape[-1] + grad_input = torch.cuda.FloatTensor(n, c).zero_() + grad_position = torch.cuda.FloatTensor(n, nsample, c).zero_() + grad_weight = torch.cuda.FloatTensor(n, nsample, w_c).zero_() + pointops_cuda.aggregation_backward_cuda( + n, + nsample, + c, + w_c, + input, + position, + weight, + idx, + grad_output, + grad_input, + grad_position, + grad_weight, + ) + return grad_input, grad_position, grad_weight, None + + +aggregation = Aggregation.apply + + +def interpolation(xyz, new_xyz, feat, offset, new_offset, k=3): + """ + input: xyz: (m, 3), new_xyz: (n, 3), feat: (m, c), offset: (b), new_offset: (b) + output: (n, c) + """ + assert xyz.is_contiguous() and new_xyz.is_contiguous() and feat.is_contiguous() + idx, dist = knnquery(k, xyz, new_xyz, offset, new_offset) # (n, 3), (n, 3) + dist_recip = 1.0 / (dist + 1e-8) # (n, 3) + norm = torch.sum(dist_recip, dim=1, keepdim=True) + weight = dist_recip / norm # (n, 3) + + new_feat = torch.cuda.FloatTensor(new_xyz.shape[0], feat.shape[1]).zero_() + for i in range(k): + new_feat += feat[idx[:, i].long(), :] * weight[:, i].unsqueeze(-1) + return new_feat + + +def interpolation_v2(xyz, new_xyz, feat, offset, new_offset, k=3): + """ + input: xyz: (m, 3), new_xyz: (n, 3), feat: (m, c), offset: (b), new_offset: (b) + output: (n, c) + """ + assert xyz.is_contiguous() and new_xyz.is_contiguous() and feat.is_contiguous() + + idx, _ = knnquery(k, xyz, new_xyz, offset, new_offset) # (n, 3), (n, 3) + + # print("e3: idx.shape: {}, idx[:5]: {}".format(idx.shape, idx[:5])) + + dist = torch.sqrt(((new_xyz.unsqueeze(1) - xyz[idx.long()]) ** 2).sum(-1) + 1e-8) + + # print("e4: dist.shape: {}, dist[:5]: {}".format(dist.shape, dist[:5])) + # print("((_-dist)**2).max(): ", ((_-dist)**2).max()) + # input() + + dist_recip = 1.0 / (dist + 1e-8) # (n, 3) + norm = torch.sum(dist_recip, dim=1, keepdim=True) + weight = dist_recip / norm # (n, 3) + + new_feat = torch.cuda.FloatTensor(new_xyz.shape[0], feat.shape[1]).zero_() + for i in range(k): + new_feat += feat[idx[:, i].long(), :] * weight[:, i].unsqueeze(-1) + return new_feat + + +class Interpolation(Function): + @staticmethod + def forward(ctx, xyz, new_xyz, input, offset, new_offset, k=3): + """ + input: xyz: (m, 3), new_xyz: (n, 3), input: (m, c), offset: (b), new_offset: (b) + output: (n, c) + """ + assert xyz.is_contiguous() and new_xyz.is_contiguous() and input.is_contiguous() + idx, dist = knnquery(k, xyz, new_xyz, offset, new_offset) # (n, k), (n, k) + dist_recip = 1.0 / (dist + 1e-8) # (n, k) + norm = torch.sum(dist_recip, dim=1, keepdim=True) + weight = dist_recip / norm # (n, k) + + n, c, m = new_xyz.shape[0], input.shape[1], input.shape[0] + output = torch.cuda.FloatTensor(n, c).zero_() + pointops_cuda.interpolation_forward_cuda(n, c, k, input, idx, weight, output) + ctx.m, ctx.k = m, k + ctx.save_for_backward(idx, weight) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: xyz: (m, 3), new_xyz: (n, 3), input: (m, c), offset: (b), new_offset: (b) + output: (n, c) + """ + m, k = ctx.m, ctx.k + idx, weight = ctx.saved_tensors + n, c = grad_output.shape + grad_input = torch.cuda.FloatTensor(m, c).zero_() + pointops_cuda.interpolation_backward_cuda( + n, c, k, grad_output, idx, weight, grad_input + ) + return None, None, grad_input, None, None, None + + +interpolation2 = Interpolation.apply diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/pointops2.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/pointops2.py new file mode 100644 index 0000000000000000000000000000000000000000..e019eca4235e014421f0df3097c93bcec2d3a3d2 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/pointops2.py @@ -0,0 +1,253 @@ +from typing import Tuple + +import torch +from torch.autograd import Function +import torch.nn as nn + +import pointops2_cuda as pointops_cuda + + +class FurthestSampling(Function): + @staticmethod + def forward(ctx, xyz, offset, new_offset): + """ + input: xyz: (n, 3), offset: (b), new_offset: (b) + output: idx: (m) + """ + assert xyz.is_contiguous() + n, b, n_max = xyz.shape[0], offset.shape[0], offset[0] + for i in range(1, b): + n_max = max(offset[i] - offset[i - 1], n_max) + idx = torch.cuda.IntTensor(new_offset[b - 1].item()).zero_() + tmp = torch.cuda.FloatTensor(n).fill_(1e10) + pointops_cuda.furthestsampling_cuda(b, n_max, xyz, offset, new_offset, tmp, idx) + del tmp + return idx + + +furthestsampling = FurthestSampling.apply + + +class KNNQuery(Function): + @staticmethod + def forward(ctx, nsample, xyz, new_xyz, offset, new_offset): + """ + input: xyz: (n, 3), new_xyz: (m, 3), offset: (b), new_offset: (b) + output: idx: (m, nsample), dist2: (m, nsample) + """ + if new_xyz is None: + new_xyz = xyz + assert xyz.is_contiguous() and new_xyz.is_contiguous() + m = new_xyz.shape[0] + idx = torch.cuda.IntTensor(m, nsample).zero_() + dist2 = torch.cuda.FloatTensor(m, nsample).zero_() + pointops_cuda.knnquery_cuda( + m, nsample, xyz, new_xyz, offset, new_offset, idx, dist2 + ) + return idx, torch.sqrt(dist2) + + +knnquery = KNNQuery.apply + + +class Grouping(Function): + @staticmethod + def forward(ctx, input, idx): + """ + input: input: (n, c), idx : (m, nsample) + output: (m, nsample, c) + """ + assert input.is_contiguous() and idx.is_contiguous() + m, nsample, n, c = idx.shape[0], idx.shape[1], input.shape[0], input.shape[1] + output = torch.cuda.FloatTensor(m, nsample, c) + pointops_cuda.grouping_forward_cuda(m, nsample, c, input, idx, output) + ctx.n = n + ctx.save_for_backward(idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_out: (m, c, nsample) + output: (n, c), None + """ + n = ctx.n + (idx,) = ctx.saved_tensors + m, nsample, c = grad_output.shape + grad_input = torch.cuda.FloatTensor(n, c).zero_() + pointops_cuda.grouping_backward_cuda( + m, nsample, c, grad_output, idx, grad_input + ) + return grad_input, None + + +grouping = Grouping.apply + + +def queryandgroup(nsample, xyz, new_xyz, feat, idx, offset, new_offset, use_xyz=True): + """ + input: xyz: (n, 3), new_xyz: (m, 3), feat: (n, c), idx: (m, nsample), offset: (b), new_offset: (b) + output: new_feat: (m, c+3, nsample), grouped_idx: (m, nsample) + """ + assert xyz.is_contiguous() and new_xyz.is_contiguous() and feat.is_contiguous() + if new_xyz is None: + new_xyz = xyz + if idx is None: + idx, _ = knnquery(nsample, xyz, new_xyz, offset, new_offset) # (m, nsample) + + n, m, c = xyz.shape[0], new_xyz.shape[0], feat.shape[1] + grouped_xyz = xyz[idx.view(-1).long(), :].view(m, nsample, 3) # (m, nsample, 3) + # grouped_xyz = grouping(xyz, idx) # (m, nsample, 3) + grouped_xyz -= new_xyz.unsqueeze(1) # (m, nsample, 3) + grouped_feat = feat[idx.view(-1).long(), :].view(m, nsample, c) # (m, nsample, c) + # grouped_feat = grouping(feat, idx) # (m, nsample, c) + + if use_xyz: + return torch.cat((grouped_xyz, grouped_feat), -1) # (m, nsample, 3+c) + else: + return grouped_feat + + +class Subtraction(Function): + @staticmethod + def forward(ctx, input1, input2, idx): + """ + input: input1: (n, c), input2: (n, c), idx: (n, nsample) + output: (n, nsample, c) + """ + assert input1.is_contiguous() and input2.is_contiguous() + n, c = input1.shape + nsample = idx.shape[-1] + output = torch.cuda.FloatTensor(n, nsample, c).zero_() + pointops_cuda.subtraction_forward_cuda( + n, nsample, c, input1, input2, idx, output + ) + ctx.save_for_backward(idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_out: (n, nsample, c) + output: grad_input1: (n, c), grad_input2: (n, c) + """ + (idx,) = ctx.saved_tensors + n, nsample, c = grad_output.shape + grad_input1 = torch.cuda.FloatTensor(n, c).zero_() + grad_input2 = torch.cuda.FloatTensor(n, c).zero_() + pointops_cuda.subtraction_backward_cuda( + n, nsample, c, idx, grad_output, grad_input1, grad_input2 + ) + return grad_input1, grad_input2, None + + +subtraction = Subtraction.apply + + +class Aggregation(Function): + @staticmethod + def forward(ctx, input, position, weight, idx): + """ + input: input: (n, c), position: (n, nsample, c), weight : (n, nsample, c'), idx: (n, nsample) + output: (n, c) + """ + assert ( + input.is_contiguous() + and position.is_contiguous() + and weight.is_contiguous() + ) + n, nsample, c = position.shape + w_c = weight.shape[-1] + output = torch.cuda.FloatTensor(n, c).zero_() + pointops_cuda.aggregation_forward_cuda( + n, nsample, c, w_c, input, position, weight, idx, output + ) + ctx.save_for_backward(input, position, weight, idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_out: (n, c) + output: grad_input: (n, c), grad_position: (n, nsample, c), grad_weight : (n, nsample, c') + """ + input, position, weight, idx = ctx.saved_tensors + n, nsample, c = position.shape + w_c = weight.shape[-1] + grad_input = torch.cuda.FloatTensor(n, c).zero_() + grad_position = torch.cuda.FloatTensor(n, nsample, c).zero_() + grad_weight = torch.cuda.FloatTensor(n, nsample, w_c).zero_() + pointops_cuda.aggregation_backward_cuda( + n, + nsample, + c, + w_c, + input, + position, + weight, + idx, + grad_output, + grad_input, + grad_position, + grad_weight, + ) + return grad_input, grad_position, grad_weight, None + + +aggregation = Aggregation.apply + + +def interpolation(xyz, new_xyz, feat, offset, new_offset, k=3): + """ + input: xyz: (m, 3), new_xyz: (n, 3), feat: (m, c), offset: (b), new_offset: (b) + output: (n, c) + """ + assert xyz.is_contiguous() and new_xyz.is_contiguous() and feat.is_contiguous() + idx, dist = knnquery(k, xyz, new_xyz, offset, new_offset) # (n, 3), (n, 3) + dist_recip = 1.0 / (dist + 1e-8) # (n, 3) + norm = torch.sum(dist_recip, dim=1, keepdim=True) + weight = dist_recip / norm # (n, 3) + + new_feat = torch.cuda.FloatTensor(new_xyz.shape[0], feat.shape[1]).zero_() + for i in range(k): + new_feat += feat[idx[:, i].long(), :] * weight[:, i].unsqueeze(-1) + return new_feat + + +class Interpolation(Function): + @staticmethod + def forward(ctx, xyz, new_xyz, input, offset, new_offset, k=3): + """ + input: xyz: (m, 3), new_xyz: (n, 3), input: (m, c), offset: (b), new_offset: (b) + output: (n, c) + """ + assert xyz.is_contiguous() and new_xyz.is_contiguous() and input.is_contiguous() + idx, dist = knnquery(k, xyz, new_xyz, offset, new_offset) # (n, k), (n, k) + dist_recip = 1.0 / (dist + 1e-8) # (n, k) + norm = torch.sum(dist_recip, dim=1, keepdim=True) + weight = dist_recip / norm # (n, k) + + n, c, m = new_xyz.shape[0], input.shape[1], input.shape[0] + output = torch.cuda.FloatTensor(n, c).zero_() + pointops_cuda.interpolation_forward_cuda(n, c, k, input, idx, weight, output) + ctx.m, ctx.k = m, k + ctx.save_for_backward(idx, weight) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: xyz: (m, 3), new_xyz: (n, 3), input: (m, c), offset: (b), new_offset: (b) + output: (n, c) + """ + m, k = ctx.m, ctx.k + idx, weight = ctx.saved_tensors + n, c = grad_output.shape + grad_input = torch.cuda.FloatTensor(m, c).zero_() + pointops_cuda.interpolation_backward_cuda( + n, c, k, grad_output, idx, weight, grad_input + ) + return None, None, grad_input, None, None, None + + +interpolation2 = Interpolation.apply diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/pointops_ablation.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/pointops_ablation.py new file mode 100644 index 0000000000000000000000000000000000000000..abfcc8bc1fb99379ff6d0fd97e19b7ca7fb0e723 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/pointops_ablation.py @@ -0,0 +1,256 @@ +from typing import Tuple + +import torch +from torch.autograd import Function +import torch.nn as nn + +import pointops2_cuda as pointops_cuda + + +class FurthestSampling(Function): + @staticmethod + def forward(ctx, xyz, offset, new_offset): + """ + input: xyz: (n, 3), offset: (b), new_offset: (b) + output: idx: (m) + """ + assert xyz.is_contiguous() + n, b, n_max = xyz.shape[0], offset.shape[0], offset[0] + for i in range(1, b): + n_max = max(offset[i] - offset[i - 1], n_max) + idx = torch.cuda.IntTensor(new_offset[b - 1].item()).zero_() + tmp = torch.cuda.FloatTensor(n).fill_(1e10) + pointops_cuda.furthestsampling_cuda(b, n_max, xyz, offset, new_offset, tmp, idx) + del tmp + return idx + + +furthestsampling = FurthestSampling.apply + + +class KNNQuery(Function): + @staticmethod + def forward(ctx, nsample, xyz, new_xyz, offset, new_offset): + """ + input: xyz: (n, 3), new_xyz: (m, 3), offset: (b), new_offset: (b) + output: idx: (m, nsample), dist2: (m, nsample) + """ + if new_xyz is None: + new_xyz = xyz + assert xyz.is_contiguous() and new_xyz.is_contiguous() + m = new_xyz.shape[0] + idx = torch.cuda.IntTensor(m, nsample).zero_() + dist2 = torch.cuda.FloatTensor(m, nsample).zero_() + pointops_cuda.knnquery_cuda( + m, nsample, xyz, new_xyz, offset, new_offset, idx, dist2 + ) + return idx, torch.sqrt(dist2) + + +knnquery = KNNQuery.apply + + +class Grouping(Function): + @staticmethod + def forward(ctx, input, idx): + """ + input: input: (n, c), idx : (m, nsample) + output: (m, nsample, c) + """ + assert input.is_contiguous() and idx.is_contiguous() + m, nsample, n, c = idx.shape[0], idx.shape[1], input.shape[0], input.shape[1] + output = torch.cuda.FloatTensor(m, nsample, c) + pointops_cuda.grouping_forward_cuda(m, nsample, c, input, idx, output) + ctx.n = n + ctx.save_for_backward(idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_out: (m, c, nsample) + output: (n, c), None + """ + n = ctx.n + (idx,) = ctx.saved_tensors + m, nsample, c = grad_output.shape + grad_input = torch.cuda.FloatTensor(n, c).zero_() + pointops_cuda.grouping_backward_cuda( + m, nsample, c, grad_output, idx, grad_input + ) + return grad_input, None + + +grouping = Grouping.apply + + +def queryandgroup( + nsample, xyz, new_xyz, feat, idx, offset, new_offset, use_xyz=True, relative=True +): + """ + input: xyz: (n, 3), new_xyz: (m, 3), feat: (n, c), idx: (m, nsample), offset: (b), new_offset: (b) + output: new_feat: (m, c+3, nsample), grouped_idx: (m, nsample) + """ + assert xyz.is_contiguous() and new_xyz.is_contiguous() and feat.is_contiguous() + if new_xyz is None: + new_xyz = xyz + if idx is None: + idx, _ = knnquery(nsample, xyz, new_xyz, offset, new_offset) # (m, nsample) + + n, m, c = xyz.shape[0], new_xyz.shape[0], feat.shape[1] + grouped_xyz = xyz[idx.view(-1).long(), :].view(m, nsample, 3) # (m, nsample, 3) + # grouped_xyz = grouping(xyz, idx) # (m, nsample, 3) + if relative: + grouped_xyz -= new_xyz.unsqueeze(1) # (m, nsample, 3) + grouped_feat = feat[idx.view(-1).long(), :].view(m, nsample, c) # (m, nsample, c) + # grouped_feat = grouping(feat, idx) # (m, nsample, c) + + if use_xyz: + return torch.cat((grouped_xyz, grouped_feat), -1) # (m, nsample, 3+c) + else: + return grouped_feat + + +class Subtraction(Function): + @staticmethod + def forward(ctx, input1, input2, idx): + """ + input: input1: (n, c), input2: (n, c), idx: (n, nsample) + output: (n, nsample, c) + """ + assert input1.is_contiguous() and input2.is_contiguous() + n, c = input1.shape + nsample = idx.shape[-1] + output = torch.cuda.FloatTensor(n, nsample, c).zero_() + pointops_cuda.subtraction_forward_cuda( + n, nsample, c, input1, input2, idx, output + ) + ctx.save_for_backward(idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_out: (n, nsample, c) + output: grad_input1: (n, c), grad_input2: (n, c) + """ + (idx,) = ctx.saved_tensors + n, nsample, c = grad_output.shape + grad_input1 = torch.cuda.FloatTensor(n, c).zero_() + grad_input2 = torch.cuda.FloatTensor(n, c).zero_() + pointops_cuda.subtraction_backward_cuda( + n, nsample, c, idx, grad_output, grad_input1, grad_input2 + ) + return grad_input1, grad_input2, None + + +subtraction = Subtraction.apply + + +class Aggregation(Function): + @staticmethod + def forward(ctx, input, position, weight, idx): + """ + input: input: (n, c), position: (n, nsample, c), weight : (n, nsample, c'), idx: (n, nsample) + output: (n, c) + """ + assert ( + input.is_contiguous() + and position.is_contiguous() + and weight.is_contiguous() + ) + n, nsample, c = position.shape + w_c = weight.shape[-1] + output = torch.cuda.FloatTensor(n, c).zero_() + pointops_cuda.aggregation_forward_cuda( + n, nsample, c, w_c, input, position, weight, idx, output + ) + ctx.save_for_backward(input, position, weight, idx) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: grad_out: (n, c) + output: grad_input: (n, c), grad_position: (n, nsample, c), grad_weight : (n, nsample, c') + """ + input, position, weight, idx = ctx.saved_tensors + n, nsample, c = position.shape + w_c = weight.shape[-1] + grad_input = torch.cuda.FloatTensor(n, c).zero_() + grad_position = torch.cuda.FloatTensor(n, nsample, c).zero_() + grad_weight = torch.cuda.FloatTensor(n, nsample, w_c).zero_() + pointops_cuda.aggregation_backward_cuda( + n, + nsample, + c, + w_c, + input, + position, + weight, + idx, + grad_output, + grad_input, + grad_position, + grad_weight, + ) + return grad_input, grad_position, grad_weight, None + + +aggregation = Aggregation.apply + + +def interpolation(xyz, new_xyz, feat, offset, new_offset, k=3): + """ + input: xyz: (m, 3), new_xyz: (n, 3), feat: (m, c), offset: (b), new_offset: (b) + output: (n, c) + """ + assert xyz.is_contiguous() and new_xyz.is_contiguous() and feat.is_contiguous() + idx, dist = knnquery(k, xyz, new_xyz, offset, new_offset) # (n, 3), (n, 3) + dist_recip = 1.0 / (dist + 1e-8) # (n, 3) + norm = torch.sum(dist_recip, dim=1, keepdim=True) + weight = dist_recip / norm # (n, 3) + + new_feat = torch.cuda.FloatTensor(new_xyz.shape[0], feat.shape[1]).zero_() + for i in range(k): + new_feat += feat[idx[:, i].long(), :] * weight[:, i].unsqueeze(-1) + return new_feat + + +class Interpolation(Function): + @staticmethod + def forward(ctx, xyz, new_xyz, input, offset, new_offset, k=3): + """ + input: xyz: (m, 3), new_xyz: (n, 3), input: (m, c), offset: (b), new_offset: (b) + output: (n, c) + """ + assert xyz.is_contiguous() and new_xyz.is_contiguous() and input.is_contiguous() + idx, dist = knnquery(k, xyz, new_xyz, offset, new_offset) # (n, k), (n, k) + dist_recip = 1.0 / (dist + 1e-8) # (n, k) + norm = torch.sum(dist_recip, dim=1, keepdim=True) + weight = dist_recip / norm # (n, k) + + n, c, m = new_xyz.shape[0], input.shape[1], input.shape[0] + output = torch.cuda.FloatTensor(n, c).zero_() + pointops_cuda.interpolation_forward_cuda(n, c, k, input, idx, weight, output) + ctx.m, ctx.k = m, k + ctx.save_for_backward(idx, weight) + return output + + @staticmethod + def backward(ctx, grad_output): + """ + input: xyz: (m, 3), new_xyz: (n, 3), input: (m, c), offset: (b), new_offset: (b) + output: (n, c) + """ + m, k = ctx.m, ctx.k + idx, weight = ctx.saved_tensors + n, c = grad_output.shape + grad_input = torch.cuda.FloatTensor(m, c).zero_() + pointops_cuda.interpolation_backward_cuda( + n, c, k, grad_output, idx, weight, grad_input + ) + return None, None, grad_input, None, None, None + + +interpolation2 = Interpolation.apply diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_attention_op_step1.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_attention_op_step1.py new file mode 100644 index 0000000000000000000000000000000000000000..b2d8428c8e283811db156acc0e6ba563f92e72ce --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_attention_op_step1.py @@ -0,0 +1,106 @@ +import torch +import pointops +from torch_scatter import ( + scatter_max, + scatter_mean, + scatter_add, + scatter_min, + scatter_sum, +) + +torch.manual_seed(1) + +M = 800000 +N = 35000 +C = 96 +h = 6 +query = torch.rand(N, h, C // h).cuda() +key = torch.rand(N, h, C // h).cuda() + +index_0 = torch.rand(M) +index_0[index_0 < 0] = 0 +index_0 = (index_0 * N).long().cuda() + +index_1 = torch.rand(M) +index_1[index_1 < 0] = 0 +index_1 = (index_1 * N).long().cuda() + +query.requires_grad = True +key.requires_grad = True + +# rearrange index for acceleration +index_0, indices = torch.sort(index_0) # [M,] +index_1 = index_1[indices] # [M,] +index_0_counts = index_0.bincount() + +print("index_0_counts.shape: ", index_0_counts.shape) + +n_max = index_0_counts.max() +index_0_offsets = index_0_counts.cumsum(dim=-1) # [N] + +print("v1 index_0_offsets.shape: ", index_0_offsets.shape) + +index_0_offsets = torch.cat( + [torch.zeros(1, dtype=torch.long).cuda(), index_0_offsets], 0 +) # [N+1] + +# print("index_0[:100]: ", index_0[:100]) +print("n_max: ", n_max) +print("index_0_offsets.shape: ", index_0_offsets.shape) +# input() + +print("index_0_offsets[:100]: ", index_0_offsets[:100]) +print("index_1[300:320]: ", index_1[300:320]) + + +attn_flat = pointops.attention_step1( + query.float(), key.float(), index_0.int(), index_1.int() +) +# loss = attn_flat.sum() +# loss.backward() +print( + "attn_flat.shape: {}, attn_flat[300:320,:10]: {}".format( + attn_flat.shape, attn_flat[300:320, :10] + ) +) +# print("query.grad[:5, :3, :5]: ", query.grad[:5, :3, :5]) +# print("key.grad[:5, :3, :5]: ", key.grad[:5, :3, :5]) +# input() + +print("query.is_contiguous(): ", query.is_contiguous()) +print("key.is_contiguous(): ", key.is_contiguous()) +print("index_0.is_contiguous(): ", index_0.is_contiguous()) +print("index_1.is_contiguous(): ", index_1.is_contiguous()) + +attn_flat_v2 = pointops.attention_step1_v2( + query.float(), key.float(), index_1.int(), index_0_offsets.int(), n_max +) +# loss = attn_flat_v2.sum() +# loss.backward() +print( + "attn_flat_v2.shape: {}, attn_flat_v2[300:320,:10]: {}".format( + attn_flat_v2.shape, attn_flat_v2[300:320, :10] + ) +) +# print("query.grad[:5, :3, :5]: ", query.grad[:5, :3, :5]) +# print("key.grad[:5, :3, :5]: ", key.grad[:5, :3, :5]) +# input() + +mask = attn_flat_v2.sum(-1) != 0 +print("mask.sum(): ", mask.sum()) +print( + "attn_flat_v2[mask] - attn_flat[mask]: ", + ((attn_flat_v2[mask] - attn_flat[mask]) ** 2).max(), +) + + +print( + "((attn_flat-attn_flat_v2)**2 < 1e-8).all(): ", + ((attn_flat - attn_flat_v2) ** 2 < 1e-8).all(), +) + +selected = 10000 +print( + "torch.max((attn_flat[:selected]-attn_flat_v2[:selected])**2, 0): ", + torch.max((attn_flat[:selected] - attn_flat_v2[:selected]) ** 2, 0), +) diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_attention_op_step1_v2.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_attention_op_step1_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..941ea13da10fd4567aeb16b30740899535d6a0a6 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_attention_op_step1_v2.py @@ -0,0 +1,123 @@ +import torch +import pointops +from torch_scatter import ( + scatter_max, + scatter_mean, + scatter_add, + scatter_min, + scatter_sum, +) + +torch.manual_seed(1) + +M = 800000 +N = 35000 +C = 96 +h = 6 +query = torch.rand(N, h, C // h).cuda() +key = torch.rand(N, h, C // h).cuda() + +index_0 = torch.rand(M) +index_0[index_0 < 0] = 0 +index_0 = (index_0 * N).long().cuda() + +index_1 = torch.rand(M) +index_1[index_1 < 0] = 0 +index_1 = (index_1 * N).long().cuda() + +query.requires_grad = True +key.requires_grad = True + + +attn_flat = pointops.attention_step1( + query.float(), key.float(), index_0.int(), index_1.int() +) +loss = attn_flat.sum() +loss.backward() +print( + "attn_flat.shape: {}, attn_flat[:20,:10]: {}".format( + attn_flat.shape, attn_flat[:20, :10] + ) +) +print("query.grad[:5, :3, :5]: ", query.grad[:5, :3, :5]) +print("key.grad[:5, :3, :5]: ", key.grad[:5, :3, :5]) +input() + + +# rearrange index for acceleration +index_0, indices = torch.sort(index_0) # [M,] +index_1 = index_1[indices] # [M,] +index_0_counts = index_0.bincount() + +print("index_0_counts.shape: ", index_0_counts.shape) + +n_max = index_0_counts.max() +index_0_offsets = index_0_counts.cumsum(dim=-1) # [N] + +print("v1 index_0_offsets.shape: ", index_0_offsets.shape) + +index_0_offsets = torch.cat( + [torch.zeros(1, dtype=torch.long).cuda(), index_0_offsets], 0 +) # [N+1] + +# print("index_0[:100]: ", index_0[:100]) +print("n_max: ", n_max) +print("index_0_offsets.shape: ", index_0_offsets.shape) +# input() + +print("index_0_offsets[:100]: ", index_0_offsets[:100]) +print("index_1[:20]: ", index_1[:20]) + + +attn_flat = pointops.attention_step1( + query.float(), key.float(), index_0.int(), index_1.int() +) +# loss = attn_flat.sum() +# loss.backward() +# # attn_flat = pointops.attention_step1(query.float(), key.float(), index_0.int(), index_1.int()) +# # loss = attn_flat.sum() +# # loss.backward() +# print("attn_flat.shape: {}, attn_flat[:20,:10]: {}".format(attn_flat.shape, attn_flat[:20,:10])) +# print("query.grad[:5, :3, :5]: ", query.grad[:5, :3, :5]) +# print("key.grad[:5, :3, :5]: ", key.grad[:5, :3, :5]) +# input() + +print("query.is_contiguous(): ", query.is_contiguous()) +print("key.is_contiguous(): ", key.is_contiguous()) +print("index_0.is_contiguous(): ", index_0.is_contiguous()) +print("index_1.is_contiguous(): ", index_1.is_contiguous()) + +attn_flat_v2 = pointops.attention_step1_v2( + query.float(), key.float(), index_1.int(), index_0_offsets.int(), n_max +) +loss = attn_flat_v2.sum() +loss.backward() + +# attn_flat_v2 = pointops.attention_step1_v2(query.float(), key.float(), index_1.int(), index_0_offsets.int(), n_max) +# loss = attn_flat_v2.sum() +# loss.backward() + +print( + "attn_flat_v2.shape: {}, attn_flat_v2[:20,:10]: {}".format( + attn_flat_v2.shape, attn_flat_v2[:20, :10] + ) +) +print("query.grad[:5, :3, :5]: ", query.grad[:5, :3, :5]) +print("key.grad[:5, :3, :5]: ", key.grad[:5, :3, :5]) +# input() + +# mask = attn_flat_v2.sum(-1) != 0 +# print("mask.sum(): ", mask.sum()) +# print("attn_flat_v2[mask] - attn_flat[mask]: ", ((attn_flat_v2[mask] - attn_flat[mask])**2).max()) + + +print( + "((attn_flat-attn_flat_v2)**2 < 1e-8).all(): ", + ((attn_flat - attn_flat_v2) ** 2 < 1e-8).all(), +) + +selected = 10000 +print( + "torch.max((attn_flat[:selected]-attn_flat_v2[:selected])**2, 0): ", + torch.max((attn_flat[:selected] - attn_flat_v2[:selected]) ** 2, 0), +) diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_attention_op_step2.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_attention_op_step2.py new file mode 100644 index 0000000000000000000000000000000000000000..036340377abedda932c74bb014ec82052ef2c884 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_attention_op_step2.py @@ -0,0 +1,62 @@ +import torch +import pointops +from torch_scatter import ( + scatter_max, + scatter_mean, + scatter_add, + scatter_min, + scatter_sum, +) + +torch.manual_seed(1) + +M = 800000 +N = 35000 +C = 96 +h = 6 +softmax_attn_flat = torch.rand(M, h).cuda() +value = torch.rand(N, h, C // h).cuda() + +index_0 = torch.rand(M) +index_0[index_0 < 0] = 0 +index_0 = (index_0 * N).long().cuda() + +index_1 = torch.rand(M) +index_1[index_1 < 0] = 0 +index_1 = (index_1 * N).long().cuda() + +softmax_attn_flat.requires_grad = True +value.requires_grad = True + +# value_flat = value[index_1] #[M, num_heads, C // num_heads] +# x = (softmax_attn_flat.unsqueeze(-1) * value_flat).reshape(M, C) +# x = scatter_sum(src=x, index=index_0, dim=0, dim_size=N) #[N, C] +# loss = x.sum() +# loss.backward() + +# print("x.shape: {}, x[:5,:10]: {}".format(x.shape, x[:5,:10])) +# print("softmax_attn_flat.grad[:5, :10]: ", softmax_attn_flat.grad[:5, :10]) +# print("value.grad[:5, :3, :5]: ", value.grad[:5, :3, :5]) +# input() + +print("softmax_attn_flat.is_contiguous(): ", softmax_attn_flat.is_contiguous()) +print("value.is_contiguous(): ", value.is_contiguous()) +print("index_0.is_contiguous(): ", index_0.is_contiguous()) +print("index_1.is_contiguous(): ", index_1.is_contiguous()) + +x_v2 = pointops.attention_step2( + softmax_attn_flat.float(), value.float(), index_0.int(), index_1.int() +) +x_v2 = x_v2.view(N, C) +loss = x_v2.sum() +loss.backward() + +print("x_v2.shape: {}, x_v2[:5,:10]: {}".format(x_v2.shape, x_v2[:5, :10])) + +print("softmax_attn_flat.grad[:5, :10]: ", softmax_attn_flat.grad[:5, :10]) +print("value.grad[:5, :3, :5]: ", value.grad[:5, :3, :5]) +input() + +print("((x-x_v2)**2 < 1e-8).all(): ", ((x - x_v2) ** 2 < 1e-8).all()) + +print("torch.max((x-x_v2)**2): ", torch.max((x - x_v2) ** 2)) diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step1.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step1.py new file mode 100644 index 0000000000000000000000000000000000000000..145c0fcbf765e65b52afc0c9fcc49041fdfd7d0d --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step1.py @@ -0,0 +1,65 @@ +import torch +import pointops +from torch_scatter import ( + scatter_max, + scatter_mean, + scatter_add, + scatter_min, + scatter_sum, +) + +torch.manual_seed(1) + +M = 80000 +N = 3500 +hdim = 16 +h = 6 +L = 31 +query = torch.rand(N, h, hdim).cuda() +table = torch.rand(L, h, hdim, 3).cuda() + +index = torch.rand(M) +index[index < 0] = 0 +index = (index * N).long().cuda() + +rel_index = torch.rand(M, 3) +rel_index[rel_index < 0] = 0 +rel_index = (rel_index * L).long().cuda() + +query.requires_grad = True +table.requires_grad = True + +# query_flat = query[index] #[M, h, hdim] +# table_x, table_y, table_z = table[:,:,:,0], table[:,:,:,1], table[:,:,:,2] #[L, h, hdim] +# rel_index_x, rel_index_y, rel_index_z = rel_index[:,0], rel_index[:,1], rel_index[:,2] #[M] +# rel_pos_encoding = table_x[rel_index_x] + table_y[rel_index_y] + table_z[rel_index_z] #[M, h, hdim] +# output = (query_flat * rel_pos_encoding).sum(-1) #[M, h] +# loss = output.mean() +# loss.backward() + +# print("output.shape: {}, output[:5,:10]: {}".format(output.shape, output[:5,:10])) +# print("query.grad[:5, :3, :5]: ", query.grad[:5, :3, :5]) +# print("table.grad[:5, :3, :5, :2]: ", table.grad[:5, :3, :5, :2]) +# input() + +# print("query.is_contiguous(): ", query.is_contiguous()) +# print("key.is_contiguous(): ", key.is_contiguous()) +# print("index_0.is_contiguous(): ", index_0.is_contiguous()) +# print("index_1.is_contiguous(): ", index_1.is_contiguous()) + +output_v2 = pointops.dot_prod_with_idx(query, index.int(), table, rel_index.int()) +loss = output_v2.mean() +loss.backward() + +print( + "output_v2.shape: {}, output_v2[:5,:10]: {}".format( + output_v2.shape, output_v2[:5, :10] + ) +) +print("v2: query.grad[:5, :3, :5]: ", query.grad[:5, :3, :5]) +print("v2: table.grad[:5, :3, :5, :2]: ", table.grad[:5, :3, :5, :2]) +input() + +# print("((output-output_v2)**2).max(): ", ((output-output_v2)**2).max()) + +# print("torch.max((attn_flat-attn_flat_v2)**2): ", torch.max((attn_flat-attn_flat_v2)**2)) diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step1_v2.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step1_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..9bf9975a725bfb30e8c73449668e41e0f53917d5 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step1_v2.py @@ -0,0 +1,75 @@ +import torch +import pointops +from torch_scatter import ( + scatter_max, + scatter_mean, + scatter_add, + scatter_min, + scatter_sum, +) + +torch.manual_seed(1) + +M = 80000 +N = 3500 +hdim = 16 +h = 6 +L = 31 +query = torch.rand(N, h, hdim).cuda() +table_q = torch.rand(L, h, hdim, 3).cuda() +key = torch.rand(N, h, hdim).cuda() +table_k = torch.rand(L, h, hdim, 3).cuda() + +index_q = torch.rand(M) +index_q[index_q < 0] = 0 +index_q = (index_q * N).long().cuda() + +index_k = torch.rand(M) +index_k[index_k < 0] = 0 +index_k = (index_k * N).long().cuda() + +rel_index = torch.rand(M, 3) +rel_index[rel_index < 0] = 0 +rel_index = (rel_index * L).long().cuda() + +query.requires_grad = True +table_q.requires_grad = True +key.requires_grad = True +table_k.requires_grad = True + +output1 = pointops.dot_prod_with_idx(query, index_q.int(), table_q, rel_index.int()) +output2 = pointops.dot_prod_with_idx(key, index_k.int(), table_k, rel_index.int()) +output = output1 + output2 +# loss = output.mean() +# loss.backward() + +# print("output.shape: {}, output[:5,:10]: {}".format(output.shape, output[:5,:10])) +# print("query.grad[:5, :3, :5]: ", query.grad[:5, :3, :5]) +# print("table_q.grad[:5, :3, :5, :2]: ", table_q.grad[:5, :3, :5, :2]) +# print("key.grad[:5, :3, :5]: ", key.grad[:5, :3, :5]) +# print("table_k.grad[:5, :3, :5, :2]: ", table_k.grad[:5, :3, :5, :2]) +# input() + +# print("query.is_contiguous(): ", query.is_contiguous()) +# print("key.is_contiguous(): ", key.is_contiguous()) +# print("index_0.is_contiguous(): ", index_0.is_contiguous()) +# print("index_1.is_contiguous(): ", index_1.is_contiguous()) + +output_v2 = pointops.dot_prod_with_idx_v2( + query, index_q.int(), key, index_k.int(), table_q, table_k, rel_index.int() +) +loss = output_v2.mean() +loss.backward() + +print( + "output_v2.shape: {}, output_v2[:5,:10]: {}".format( + output_v2.shape, output_v2[:5, :10] + ) +) +print("v2 query.grad[:5, :3, :5]: ", query.grad[:5, :3, :5]) +print("v2 table_q.grad[:5, :3, :5, :2]: ", table_q.grad[:5, :3, :5, :2]) +print("v2 key.grad[:5, :3, :5]: ", key.grad[:5, :3, :5]) +print("v2 table_k.grad[:5, :3, :5, :2]: ", table_k.grad[:5, :3, :5, :2]) +# input() + +print("((output-output_v2)**2).max(): ", ((output - output_v2) ** 2).max()) diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step1_v3.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step1_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..3738ba69b0d9be7aa9e890b0357f8de6cc42708b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step1_v3.py @@ -0,0 +1,106 @@ +import torch +import pointops +from torch_scatter import ( + scatter_max, + scatter_mean, + scatter_add, + scatter_min, + scatter_sum, +) + +torch.manual_seed(1) + +M = 80000 +N = 3500 +# M = 80 +# N = 5 +hdim = 16 +h = 6 +L = 31 +query = torch.rand(N, h, hdim).cuda() +table_q = torch.rand(L, h, hdim, 3).cuda() +key = torch.rand(N, h, hdim).cuda() +table_k = torch.rand(L, h, hdim, 3).cuda() + +index_q = torch.rand(M) +index_q[index_q < 0] = 0 +index_q = (index_q * N).long().cuda() + +index_k = torch.rand(M) +index_k[index_k < 0] = 0 +index_k = (index_k * N).long().cuda() + +rel_index = torch.rand(M, 3) +rel_index[rel_index < 0] = 0 +rel_index = (rel_index * L).long().cuda() + + +# rearrange index for acceleration +index_q, indices = torch.sort(index_q) # [M,] +index_k = index_k[indices] # [M,] +rel_index = rel_index[indices] +index_q_counts = index_q.bincount() + +print("index_q_counts.shape: ", index_q_counts.shape) + +n_max = index_q_counts.max() +index_q_offsets = index_q_counts.cumsum(dim=-1) # [N] + +print("v1 index_q_offsets.shape: ", index_q_offsets.shape) + +index_q_offsets = torch.cat( + [torch.zeros(1, dtype=torch.long).cuda(), index_q_offsets], 0 +) # [N+1] + +# print("index_q[:100]: ", index_q[:100]) +print("n_max: ", n_max) +print("index_q_offsets.shape: ", index_q_offsets.shape) +# input() + +print("index_q_offsets[:100]: ", index_q_offsets[:100]) +print("index_k[:20]: ", index_k[:20]) + +query.requires_grad = True +table_q.requires_grad = True +key.requires_grad = True +table_k.requires_grad = True + +output1 = pointops.dot_prod_with_idx(query, index_q.int(), table_q, rel_index.int()) +output2 = pointops.dot_prod_with_idx(key, index_k.int(), table_k, rel_index.int()) +output = output1 + output2 +loss = output.mean() +loss.backward() + +# print("output.shape: {}, output[:5,:10]: {}".format(output.shape, output[:5,:10])) +# print("query.grad[:5, :3, :5]: ", query.grad[:5, :3, :5]) +# print("table_q.grad[:5, :3, :5, :2]: ", table_q.grad[:5, :3, :5, :2]) +# print("key.grad[:5, :3, :5]: ", key.grad[:5, :3, :5]) +# print("table_k.grad[:5, :3, :5, :2]: ", table_k.grad[:5, :3, :5, :2]) +# input() + +# print("query.is_contiguous(): ", query.is_contiguous()) +# print("key.is_contiguous(): ", key.is_contiguous()) +# print("index_q.is_contiguous(): ", index_q.is_contiguous()) +# print("index_k.is_contiguous(): ", index_k.is_contiguous()) + +output_v2 = pointops.dot_prod_with_idx_v3( + query, + index_q_offsets.int(), + n_max, + key, + index_k.int(), + table_q, + table_k, + rel_index.int(), +) +# loss = output_v2.mean() +# loss.backward() + +# print("output_v2.shape: {}, output_v2[:5,:10]: {}".format(output_v2.shape, output_v2[:5,:10])) +# print("v2 query.grad[:5, :3, :5]: ", query.grad[:5, :3, :5]) +# print("v2 table_q.grad[:5, :3, :5, :2]: ", table_q.grad[:5, :3, :5, :2]) +# print("v2 key.grad[:5, :3, :5]: ", key.grad[:5, :3, :5]) +# print("v2 table_k.grad[:5, :3, :5, :2]: ", table_k.grad[:5, :3, :5, :2]) +# input() + +print("((output-output_v2)**2).max(): ", ((output - output_v2) ** 2).max()) diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step2.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step2.py new file mode 100644 index 0000000000000000000000000000000000000000..f1cb9ef37b4350f50f23857b365729b1a85b24f9 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step2.py @@ -0,0 +1,83 @@ +import torch +import pointops +from torch_scatter import ( + scatter_max, + scatter_mean, + scatter_add, + scatter_min, + scatter_sum, +) + +torch.manual_seed(1) + +M = 80000 +N = 3500 +hdim = 16 +h = 6 +L = 31 +attn = torch.rand(M, h).cuda() +v = torch.rand(N, h, hdim).cuda() +table = torch.rand(L, h, hdim, 3).cuda() + +index_0 = torch.rand(M) +index_0[index_0 < 0] = 0 +index_0 = (index_0 * N).long().cuda() + +index_1 = torch.rand(M) +index_1[index_1 < 0] = 0 +index_1 = (index_1 * N).long().cuda() + +rel_index = torch.rand(M, 3) +rel_index[rel_index < 0] = 0 +rel_index = (rel_index * L).long().cuda() + +attn.requires_grad = True +v.requires_grad = True +table.requires_grad = True + +v_flat = v[index_1] # [M, h, hdim] +table_x, table_y, table_z = ( + table[:, :, :, 0], + table[:, :, :, 1], + table[:, :, :, 2], +) # [L, h, hdim] +rel_index_x, rel_index_y, rel_index_z = ( + rel_index[:, 0], + rel_index[:, 1], + rel_index[:, 2], +) # [M] +rel_pos_encoding = ( + table_x[rel_index_x] + table_y[rel_index_y] + table_z[rel_index_z] +) # [M, h, hdim] +v_flat_new = v_flat + rel_pos_encoding # [M, h, hdim] +output = attn.unsqueeze(-1) * v_flat_new # [M, h, hdim] +output = scatter_sum(src=output, index=index_0, dim=0, dim_size=N) # [N, h, hdim] +loss = output.mean() +loss.backward() + +print( + "output.shape: {}, output[:5,:10,:5]: {}".format(output.shape, output[:5, :10, :5]) +) +print("attn.grad[:5, :3]: ", attn.grad[:5, :3]) +print("v.grad[:5, :3, :5]: ", v.grad[:5, :3, :5]) +print("table.grad[:5, :3, :5, :2]: ", table.grad[:5, :3, :5, :2]) +input() + +# print("query.is_contiguous(): ", query.is_contiguous()) +# print("key.is_contiguous(): ", key.is_contiguous()) +# print("index_0.is_contiguous(): ", index_0.is_contiguous()) +# print("index_1.is_contiguous(): ", index_1.is_contiguous()) + +# output_v2 = pointops.attention_step2_with_rel_pos_value(attn, v, index_0.int(), index_1.int(), table, rel_index.int()) +# loss = output_v2.mean() +# loss.backward() + +# print("output_v2.shape: {}, output_v2[:5,:10,:5]: {}".format(output_v2.shape, output_v2[:5,:10,:5])) +# print("v2 attn.grad[:5, :3]: ", attn.grad[:5, :3]) +# print("v2 v.grad[:5, :3, :5]: ", v.grad[:5, :3, :5]) +# print("v2 table.grad[:5, :3, :5, :2]: ", table.grad[:5, :3, :5, :2]) +# input() + +# print("((output-output_v2)**2).max(): ", ((output-output_v2)**2).max()) + +# print("torch.max((attn_flat-attn_flat_v2)**2): ", torch.max((attn_flat-attn_flat_v2)**2)) diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step2_v2.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step2_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..3090b980cf2ddf9803db40d161a21ce09edc7392 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/functions/test_relative_pos_encoding_op_step2_v2.py @@ -0,0 +1,109 @@ +import torch +import pointops +from torch_scatter import ( + scatter_max, + scatter_mean, + scatter_add, + scatter_min, + scatter_sum, +) + +torch.manual_seed(1) + +M = 80000 +N = 3500 +hdim = 16 +h = 6 +L = 31 +attn = torch.rand(M, h).cuda() +v = torch.rand(N, h, hdim).cuda() +table = torch.rand(L, h, hdim, 3).cuda() + +index_0 = torch.rand(M) +index_0[index_0 < 0] = 0 +index_0 = (index_0 * N).long().cuda() + +index_1 = torch.rand(M) +index_1[index_1 < 0] = 0 +index_1 = (index_1 * N).long().cuda() + +rel_index = torch.rand(M, 3) +rel_index[rel_index < 0] = 0 +rel_index = (rel_index * L).long().cuda() + + +# rearrange index for acceleration +index_0, indices = torch.sort(index_0) # [M,] +index_1 = index_1[indices] # [M,] +rel_index = rel_index[indices] +index_0_counts = index_0.bincount() + +print("index_0_counts.shape: ", index_0_counts.shape) + +n_max = index_0_counts.max() +index_0_offsets = index_0_counts.cumsum(dim=-1) # [N] + +print("v1 index_0_offsets.shape: ", index_0_offsets.shape) + +index_0_offsets = torch.cat( + [torch.zeros(1, dtype=torch.long).cuda(), index_0_offsets], 0 +) # [N+1] + + +attn.requires_grad = True +v.requires_grad = True +table.requires_grad = True + + +output = pointops.attention_step2_with_rel_pos_value( + attn, v, index_0.int(), index_1.int(), table, rel_index.int() +) +loss = output.mean() +loss.backward() + +print( + "output.shape: {}, output[:5,:10,:5]: {}".format(output.shape, output[:5, :10, :5]) +) +print("attn.grad[:5, :3]: ", attn.grad[:5, :3]) +print("v.grad[:5, :3, :5]: ", v.grad[:5, :3, :5]) +print("table.grad[:5, :3, :5, :2]: ", table.grad[:5, :3, :5, :2]) +# input() + +attn_grad = attn.grad.clone() +v_grad = v.grad.clone() +table_grad = table.grad.clone() + +attn.grad.zero_() +v.grad.zero_() +table.grad.zero_() + +# print("query.is_contiguous(): ", query.is_contiguous()) +# print("key.is_contiguous(): ", key.is_contiguous()) +# print("index_0.is_contiguous(): ", index_0.is_contiguous()) +# print("index_1.is_contiguous(): ", index_1.is_contiguous()) + +output_v2 = pointops.attention_step2_with_rel_pos_value_v2( + attn, v, index_0_offsets.int(), n_max, index_1.int(), table, rel_index.int() +) +loss = output_v2.mean() +loss.backward() + +print( + "output_v2.shape: {}, output_v2[:5,:10,:5]: {}".format( + output_v2.shape, output_v2[:5, :10, :5] + ) +) +print("v2 attn.grad[:5, :3]: ", attn.grad[:5, :3]) +print("v2 v.grad[:5, :3, :5]: ", v.grad[:5, :3, :5]) +print("v2 table.grad[:5, :3, :5, :2]: ", table.grad[:5, :3, :5, :2]) +# input() + +print("((output-output_v2)**2).max(): ", ((output - output_v2) ** 2).max()) + +print("((attn_grad-attn.grad)**2).max(): ", ((attn_grad - attn.grad) ** 2).max()) + +print("((v_grad-v.grad)**2).max(): ", ((v_grad - v.grad) ** 2).max()) + +print("((table_grad-table.grad)**2).max(): ", ((table_grad - table.grad) ** 2).max()) + +# print("torch.max((attn_flat-attn_flat_v2)**2): ", torch.max((attn_flat-attn_flat_v2)**2)) diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/setup.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..b33cb3b83c39302500efb464667d346d5699f0aa --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/setup.py @@ -0,0 +1,33 @@ +import os +from setuptools import setup +from torch.utils.cpp_extension import BuildExtension, CUDAExtension +from distutils.sysconfig import get_config_vars + +(opt,) = get_config_vars("OPT") +os.environ["OPT"] = " ".join( + flag for flag in opt.split() if flag != "-Wstrict-prototypes" +) + +src = "src" +sources = [ + os.path.join(root, file) + for root, dirs, files in os.walk(src) + for file in files + if file.endswith(".cpp") or file.endswith(".cu") +] + +setup( + name="pointops2", + version="1.0", + install_requires=["torch", "numpy"], + packages=["pointops2"], + package_dir={"pointops2": "functions"}, + ext_modules=[ + CUDAExtension( + name="pointops2_cuda", + sources=sources, + extra_compile_args={"cxx": ["-g"], "nvcc": ["-O2"]}, + ) + ], + cmdclass={"build_ext": BuildExtension}, +) diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/__init__.py b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/aggregation/aggregation_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/aggregation/aggregation_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..491b6f41660edf9b5ea5656cc88edba8ed807d71 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/aggregation/aggregation_cuda.cpp @@ -0,0 +1,28 @@ +#include +#include +#include +#include "aggregation_cuda_kernel.h" + + +void aggregation_forward_cuda(int n, int nsample, int c, int w_c, at::Tensor input_tensor, at::Tensor position_tensor, at::Tensor weight_tensor, at::Tensor idx_tensor, at::Tensor output_tensor) +{ + const float *input = input_tensor.data_ptr(); + const float *position = position_tensor.data_ptr(); + const float *weight = weight_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + aggregation_forward_cuda_launcher(n, nsample, c, w_c, input, position, weight, idx, output); +} + +void aggregation_backward_cuda(int n, int nsample, int c, int w_c, at::Tensor input_tensor, at::Tensor position_tensor, at::Tensor weight_tensor, at::Tensor idx_tensor, at::Tensor grad_output_tensor, at::Tensor grad_input_tensor, at::Tensor grad_position_tensor, at::Tensor grad_weight_tensor) +{ + const float *input = input_tensor.data_ptr(); + const float *position = position_tensor.data_ptr(); + const float *weight = weight_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + const float *grad_output = grad_output_tensor.data_ptr(); + float *grad_input = grad_input_tensor.data_ptr(); + float *grad_position = grad_position_tensor.data_ptr(); + float *grad_weight = grad_weight_tensor.data_ptr(); + aggregation_backward_cuda_launcher(n, nsample, c, w_c, input, position, weight, idx, grad_output, grad_input, grad_position, grad_weight); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/aggregation/aggregation_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/aggregation/aggregation_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..8339bb7e2088abffefba02c26b248edafed6cf47 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/aggregation/aggregation_cuda_kernel.cu @@ -0,0 +1,53 @@ +#include "../cuda_utils.h" +#include "aggregation_cuda_kernel.h" + + +__global__ void aggregation_forward_cuda_kernel(int n, int nsample, int c, int w_c, const float *input, const float *position, const float *weight, const int *idx, float *output) { + // input: input: (n, c), position: (n, nsample, c), weight: (n, nsample, w_c), idx: (n, nsample), output: (n, c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= n * c) return; + const int c_idx = index % c; + const int n_idx = index / c; + const int w_c_idx = c_idx % w_c; + for (int nsample_idx = 0; nsample_idx < nsample; nsample_idx++) + { + int idx_idx = n_idx * nsample + nsample_idx; + int input_idx = idx[idx_idx] * c + c_idx; + int position_idx = n_idx * nsample * c + nsample_idx * c + c_idx; + int weight_idx = n_idx * nsample * w_c + nsample_idx * w_c + w_c_idx; + output[index] += (input[input_idx] + position[position_idx]) * weight[weight_idx]; + } +} + +__global__ void aggregation_backward_cuda_kernel(int n, int nsample, int c, int w_c, const float *input, const float *position, const float *weight, const int *idx, const float *grad_output, float *grad_input, float *grad_position, float *grad_weight) { + // input: grad_output: (n, c), output: grad_input: (n, c), grad_position: (n, nsample, c), grad_weight: (n, nsample, w_c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= n * c) return; + const int c_idx = index % c; + const int n_idx = index / c; + const int w_c_idx = c_idx % w_c; + for (int nsample_idx = 0; nsample_idx < nsample; nsample_idx++) + { + int idx_idx = n_idx * nsample + nsample_idx; + int input_idx = idx[idx_idx] * c + c_idx; + int position_idx = n_idx * nsample * c + nsample_idx * c + c_idx; + int weight_idx = n_idx * nsample * w_c + nsample_idx * w_c + w_c_idx; + atomicAdd(grad_input + input_idx, grad_output[index] * weight[weight_idx]); + grad_position[position_idx] = grad_output[index] * weight[weight_idx]; + atomicAdd(grad_weight + weight_idx, grad_output[index] * (input[input_idx] + position[position_idx])); + } +} + +void aggregation_forward_cuda_launcher(int n, int nsample, int c, int w_c, const float *input, const float *position, const float *weight, const int *idx, float *output) { + // input: input: (n, c), position: (n, nsample, c), weight: (n, nsample, w_c), idx: (n, nsample), output: (n, c) + dim3 blocks(DIVUP(n * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + aggregation_forward_cuda_kernel<<>>(n, nsample, c, w_c, input, position, weight, idx, output); +} + +void aggregation_backward_cuda_launcher(int n, int nsample, int c, int w_c, const float *input, const float *position, const float *weight, const int *idx, const float *grad_output, float *grad_input, float *grad_position, float *grad_weight) { + // input: grad_output: (n, c), output: grad_input: (n, c), grad_position: (n, nsample, c), grad_weight: (n, nsample, w_c) + dim3 blocks(DIVUP(n * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + aggregation_backward_cuda_kernel<<>>(n, nsample, c, w_c, input, position, weight, idx, grad_output, grad_input, grad_position, grad_weight); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/aggregation/aggregation_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/aggregation/aggregation_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..5211a96aa2acbe0d9baf32bddc9ab4be87703072 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/aggregation/aggregation_cuda_kernel.h @@ -0,0 +1,20 @@ +#ifndef _AGGREGATION_CUDA_KERNEL +#define _AGGREGATION_CUDA_KERNEL +#include +#include +#include + +void aggregation_forward_cuda(int n, int nsample, int c, int w_c, at::Tensor input_tensor, at::Tensor position_tensor, at::Tensor weight_tensor, at::Tensor idx_tensor, at::Tensor output_tensor); +void aggregation_backward_cuda(int n, int nsample, int c, int w_c, at::Tensor input_tensor, at::Tensor position_tensor, at::Tensor weight_tensor, at::Tensor idx_tensor, at::Tensor grad_output_tensor, at::Tensor grad_input_tensor, at::Tensor grad_position_tensor, at::Tensor grad_weight_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void aggregation_forward_cuda_launcher(int n, int nsample, int c, int w_c, const float *input, const float *position, const float *weight, const int *idx, float *output); +void aggregation_backward_cuda_launcher(int n, int nsample, int c, int w_c, const float *input, const float *position, const float *weight, const int *idx, const float *grad_output, float *grad_input, float *grad_position, float *grad_weight); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention/attention_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention/attention_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..27493d19ebfd11b083b8f31455ac12c4416208a9 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention/attention_cuda.cpp @@ -0,0 +1,55 @@ +#include +#include +#include +#include "attention_cuda_kernel.h" + +void attention_step1_forward_cuda(int N, int M, int h, int C, at::Tensor q_tensor, at::Tensor k_tensor, + at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor attn_tensor) +{ + const float *q = q_tensor.data_ptr(); + const float *k = k_tensor.data_ptr(); + const int *index0 = index0_tensor.data_ptr(); + const int *index1 = index1_tensor.data_ptr(); + float *attn = attn_tensor.data_ptr(); + attention_step1_forward_cuda_launcher(N, M, h, C, q, k, index0, index1, attn); +} + +void attention_step1_backward_cuda(int N, int M, int h, int C, at::Tensor grad_out_tensor, + at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor q_tensor, at::Tensor k_tensor, + at::Tensor grad_q_tensor, at::Tensor grad_k_tensor) +{ + const float *grad_out = grad_out_tensor.data_ptr(); + const int *index0 = index0_tensor.data_ptr(); + const int *index1 = index1_tensor.data_ptr(); + const float *q = q_tensor.data_ptr(); + const float *k = k_tensor.data_ptr(); + float *grad_q = grad_q_tensor.data_ptr(); + float *grad_k = grad_k_tensor.data_ptr(); + attention_step1_backward_cuda_launcher(N, M, h, C, grad_out, index0, index1, q, k, grad_q, grad_k); +} + +void attention_step2_forward_cuda(int N, int M, int h, int C, at::Tensor attn_tensor, at::Tensor v_tensor, + at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor output_tensor) +{ + const float *attn = attn_tensor.data_ptr(); + const float *v = v_tensor.data_ptr(); + const int *index0 = index0_tensor.data_ptr(); + const int *index1 = index1_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + attention_step2_forward_cuda_launcher(N, M, h, C, attn, v, index0, index1, output); +} + + +void attention_step2_backward_cuda(int N, int M, int h, int C, at::Tensor grad_out_tensor, + at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor attn_tensor, at::Tensor v_tensor, + at::Tensor grad_attn_tensor, at::Tensor grad_v_tensor) +{ + const float *grad_out = grad_out_tensor.data_ptr(); + const int *index0 = index0_tensor.data_ptr(); + const int *index1 = index1_tensor.data_ptr(); + const float *attn = attn_tensor.data_ptr(); + const float *v = v_tensor.data_ptr(); + float *grad_attn = grad_attn_tensor.data_ptr(); + float *grad_v = grad_v_tensor.data_ptr(); + attention_step2_backward_cuda_launcher(N, M, h, C, grad_out, index0, index1, attn, v, grad_attn, grad_v); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention/attention_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention/attention_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..6bf8d718bf4811fd441b4df869d9498b35316976 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention/attention_cuda_kernel.cu @@ -0,0 +1,105 @@ +/* written by Xin Lai. Email: xinlai@cse.cuhk.edu.hk */ + +#include "../cuda_utils.h" +#include "attention_cuda_kernel.h" + + +__global__ void attention_step1_forward_cuda_kernel( // M, h, C//h + int N, int M, int h, int C, const float *q, const float *k, + const int *index0, const int *index1, float *attn) { + + int c_idx = blockIdx.z; + int h_idx = blockIdx.y; + int m_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (m_idx >= M || h_idx >= h || c_idx >= C / h) return; + + int idx0 = index0[m_idx]; + int idx1 = index1[m_idx]; + float val = q[idx0*C+h_idx*C/h+c_idx] * k[idx1*C+h_idx*C/h+c_idx]; + atomicAdd(attn+m_idx*h+h_idx, val); +} + +__global__ void attention_step1_backward_cuda_kernel( // M, h, C//h + int N, int M, int h, int C, const float *grad_out, const int *index0, const int *index1, const float *q, const float *k, + float *grad_q, float *grad_k) { + + int c_idx = blockIdx.z; + int h_idx = blockIdx.y; + int m_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (m_idx >= M || h_idx >= h || c_idx >= C / h) return; + + int idx0 = index0[m_idx]; + int idx1 = index1[m_idx]; + int grad_out_idx = m_idx*h+h_idx; + int q_idx = idx0*C+h_idx*C/h+c_idx; + int k_idx = idx1*C+h_idx*C/h+c_idx; + atomicAdd(grad_q+q_idx, grad_out[grad_out_idx] * k[k_idx]); + atomicAdd(grad_k+k_idx, grad_out[grad_out_idx] * q[q_idx]); +} + +void attention_step1_forward_cuda_launcher(int N, int M, int h, int C, const float *q, const float *k, + const int *index0, const int *index1, float *attn) { + // input: attn: (M, h), v: (N, h, C/h), index0: (M, ), index1: (M, ) + //dim3 blocks(DIVUP(C/h, THREADS_PER_BLOCK), h, M); + dim3 blocks(DIVUP(M, THREADS_PER_BLOCK), h, C/h); + dim3 threads(THREADS_PER_BLOCK); + attention_step1_forward_cuda_kernel<<>>(N, M, h, C, q, k, index0, index1, attn); +} + +void attention_step1_backward_cuda_launcher(int N, int M, int h, int C, const float *grad_out, const int *index0, const int *index1, + const float *q, const float *k, float *grad_q, float *grad_k) { + // input: grad_output: (n, nsample, c), output: grad_input1: (n, c), grad_input2: (n, c) + //dim3 blocks(DIVUP(C/h, THREADS_PER_BLOCK), h, M); + dim3 blocks(DIVUP(M, THREADS_PER_BLOCK), h, C/h); + dim3 threads(THREADS_PER_BLOCK); + attention_step1_backward_cuda_kernel<<>>(N, M, h, C, grad_out, index0, index1, q, k, grad_q, grad_k); +} + +__global__ void attention_step2_forward_cuda_kernel( // M, h, C//h + int N, int M, int h, int C, const float *attn, const float *v, + const int *index0, const int *index1, float *output) { + + int c_idx = blockIdx.z; + int h_idx = blockIdx.y; + int m_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (m_idx >= M || h_idx >= h || c_idx >= C / h) return; + + int idx1 = index1[m_idx]; + float val = attn[m_idx*h+h_idx] * v[idx1*C+h_idx*C/h+c_idx]; + int idx0 = index0[m_idx]; + atomicAdd(output+idx0*C+h_idx*C/h+c_idx, val); +} + +__global__ void attention_step2_backward_cuda_kernel( // M, h, C//h + int N, int M, int h, int C, const float *grad_out, const int *index0, const int *index1, const float *attn, const float *v, + float *grad_attn, float *grad_v) { + + int c_idx = blockIdx.z; + int h_idx = blockIdx.y; + int m_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (m_idx >= M || h_idx >= h || c_idx >= C / h) return; + + int idx0 = index0[m_idx]; + int idx1 = index1[m_idx]; + int grad_out_idx = idx0*C+h_idx*C/h+c_idx; + atomicAdd(grad_attn+m_idx*h+h_idx, grad_out[grad_out_idx] * v[idx1*C+h_idx*C/h+c_idx]); + atomicAdd(grad_v+idx1*C+h_idx*C/h+c_idx, grad_out[grad_out_idx] * attn[m_idx*h+h_idx]); +} + +void attention_step2_forward_cuda_launcher(int N, int M, int h, int C, const float *attn, const float *v, + const int *index0, const int *index1, float *output) { + // input: attn: (M, h), v: (N, h, C/h), index0: (M, ), index1: (M, ) + //dim3 blocks(DIVUP(C/h, THREADS_PER_BLOCK), h, M); + dim3 blocks(DIVUP(M, THREADS_PER_BLOCK), h, C/h); + dim3 threads(THREADS_PER_BLOCK); + attention_step2_forward_cuda_kernel<<>>(N, M, h, C, attn, v, index0, index1, output); +} + +void attention_step2_backward_cuda_launcher(int N, int M, int h, int C, const float *grad_out, const int *index0, const int *index1, + const float *attn, const float *v, float *grad_attn, float *grad_v) { + // input: grad_output: (n, nsample, c), output: grad_input1: (n, c), grad_input2: (n, c) + //dim3 blocks(DIVUP(C/h, THREADS_PER_BLOCK), h, M); + dim3 blocks(DIVUP(M, THREADS_PER_BLOCK), h, C/h); + dim3 threads(THREADS_PER_BLOCK); + attention_step2_backward_cuda_kernel<<>>(N, M, h, C, grad_out, index0, index1, attn, v, grad_attn, grad_v); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention/attention_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention/attention_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..cbd99b9b6a9c65af76aa95d00fff6306446114cd --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention/attention_cuda_kernel.h @@ -0,0 +1,26 @@ +#ifndef _ATTENTION_CUDA_KERNEL +#define _ATTENTION_CUDA_KERNEL +#include +#include +#include + +void attention_step1_forward_cuda(int N, int M, int h, int C, at::Tensor q_tensor, at::Tensor k_tensor, at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor attn_tensor); +void attention_step1_backward_cuda(int N, int M, int h, int C, at::Tensor grad_out_tensor, at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor q_tensor, at::Tensor k_tensor, at::Tensor grad_q_tensor, at::Tensor grad_k_tensor); + +void attention_step2_forward_cuda(int N, int M, int h, int C, at::Tensor attn_tensor, at::Tensor v_tensor, at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor output_tensor); +void attention_step2_backward_cuda(int N, int M, int h, int C, at::Tensor grad_out_tensor, at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor attn_tensor, at::Tensor v_tensor, at::Tensor grad_attn_tensor, at::Tensor grad_v_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void attention_step1_forward_cuda_launcher(int N, int M, int h, int C, const float *q, const float *k, const int *index0, const int *index1, float *attn); +void attention_step1_backward_cuda_launcher(int N, int M, int h, int C, const float *grad_out, const int *index0, const int *index1, const float *q, const float *k, float *grad_q, float *grad_k); + +void attention_step2_forward_cuda_launcher(int N, int M, int h, int C, const float *attn, const float *v, const int *index0, const int *index1, float *output); +void attention_step2_backward_cuda_launcher(int N, int M, int h, int C, const float *grad_out, const int *index0, const int *index1, const float *attn, const float *v, float *grad_attn, float *grad_v); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention_v2/attention_cuda_kernel_v2.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention_v2/attention_cuda_kernel_v2.cu new file mode 100644 index 0000000000000000000000000000000000000000..52e65fc1f8b47d80212fe28fa0b811fd4442a4ce --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention_v2/attention_cuda_kernel_v2.cu @@ -0,0 +1,195 @@ +/* written by Xin Lai. Email: xinlai@cse.cuhk.edu.hk */ + +#include "../cuda_utils.h" +#include "attention_cuda_kernel_v2.h" + + +template +__global__ void attention_step1_forward_cuda_kernel_v2( // M, h, C//h + int N, int M, int h, const float *q, const float *k, + const int *index0_offsets, const int *index1, float *attn) { + + int h_idx = blockIdx.y; + int q_idx = blockIdx.x; + int n_idx = threadIdx.x; + int C = h * d; + // if (m_idx >= M || h_idx >= h || c_idx >= C / h) return; + + __shared__ float query_vec[d]; + __shared__ int start, end; + + // if(n_idx == 0){ + // printf("blockDim.x: %d\n", blockDim.x); + // } + + if (n_idx == 0){ + start = index0_offsets[q_idx]; + end = index0_offsets[q_idx+1]; + // printf("start: %d, end: %d, blockDim.x: %d\n", start, end, blockDim.x); + } + for(int i = n_idx; i < d; i += blockDim.x) + query_vec[i] = q[q_idx*C + h_idx*d + i]; + + __syncthreads(); + + int m_idx = start + n_idx; + if(m_idx >= end) + return; + + float sum = 0; + for(int i = 0; i < d; i++){ + int k_idx = index1[m_idx]; + float key = k[k_idx * C + h_idx * d + i]; + sum += query_vec[i] * key; + } + attn[m_idx*h + h_idx] = sum; + // int idx0 = index0[m_idx]; + // int idx1 = index1[m_idx]; + // float val = q[idx0*C+h_idx*C/h+c_idx] * k[idx1*C+h_idx*C/h+c_idx]; + // atomicAdd(attn+m_idx*h+h_idx, val); +} + +template +__global__ void attention_step1_backward_cuda_kernel_v2( // M, h, C//h + int N, int M, int h, const float *grad_out, const int *index0_offsets, const int *index1, const float *q, const float *k, + float *grad_q, float *grad_k) { + + int h_idx = blockIdx.y; + int q_idx = blockIdx.x; + int n_idx = threadIdx.x; + int C = d * h; + + __shared__ float query_vec[d]; + __shared__ int start, end; + + if (n_idx == 0){ + start = index0_offsets[q_idx]; + end = index0_offsets[q_idx+1]; + } + for(int i = n_idx; i < d; i += blockDim.x) + query_vec[i] = q[q_idx*C + h_idx*d + i]; + + __shared__ float gradient_new[d]; + for(int i = n_idx; i < d; i += blockDim.x) + gradient_new[i] = 0; + + __syncthreads(); + + int m_idx = start + n_idx; + if(m_idx < end){ + float gradient = grad_out[m_idx*h + h_idx]; + for(int i = 0; i < d; i++){ + int k_idx = index1[m_idx]; + atomicAdd(&gradient_new[i], gradient * k[k_idx*C + h_idx*d + i]); + atomicAdd(grad_k + k_idx*C + h_idx*d + i, gradient * query_vec[i]); + } + } + __syncthreads(); + + for(int i = n_idx; i < d; i += blockDim.x) + grad_q[q_idx*C + h_idx*d + i] = gradient_new[i]; +} + +void attention_step1_forward_cuda_launcher_v2(int N, int M, int h, int C, const unsigned int n_max, + const float *q, const float *k, const int *index0_offsets, const int *index1, float *attn) { + // input: attn: (M, h), v: (N, h, C/h), index0: (M, ), index1: (M, ) + //dim3 blocks(DIVUP(C/h, THREADS_PER_BLOCK), h, M); + dim3 blocks(N, h); + unsigned int n_threads = opt_n_threads(n_max); + + n_threads = n_threads == n_max ? n_threads : n_threads * 2; + // n_threads = n_threads > 1024 ? 512 : n_threads; + + // printf("n_max: %d, n_threads: %d\n", n_max, n_threads); + + // dim3 threads(THREADS_PER_BLOCK); + // attention_step1_forward_cuda_kernel_v2<<>>(N, M, h, C, q, k, index0, index1, attn); + + switch (C / h) { + case 16: + attention_step1_forward_cuda_kernel_v2<16><<>>(N, M, h, q, k, index0_offsets, index1, attn); + break; + case 32: + attention_step1_forward_cuda_kernel_v2<32><<>>(N, M, h, q, k, index0_offsets, index1, attn); + break; + default: + throw "d != 16 and d != 32"; + } +} + +void attention_step1_backward_cuda_launcher_v2(int N, int M, int h, int C, const unsigned int n_max, + const float *grad_out, const int *index0_offsets, const int *index1, const float *q, const float *k, float *grad_q, float *grad_k) { + // input: grad_output: (n, nsample, c), output: grad_input1: (n, c), grad_input2: (n, c) + //dim3 blocks(DIVUP(C/h, THREADS_PER_BLOCK), h, M); + // dim3 blocks(DIVUP(M, THREADS_PER_BLOCK), h, C/h); + // dim3 threads(THREADS_PER_BLOCK); + dim3 blocks(N, h); + unsigned int n_threads = opt_n_threads(n_max); + // attention_step1_backward_cuda_kernel_v2<<>>(N, M, h, C/h, grad_out, index0_offsets, index1, q, k, grad_q, grad_k); + + n_threads = n_threads == n_max ? n_threads : n_threads * 2; + // n_threads = n_threads > 1024 ? 512 : n_threads; + + // printf("n_max: %d, n_threads: %d\n", n_max, n_threads); + + switch (C / h) { + case 16: + attention_step1_backward_cuda_kernel_v2<16><<>>(N, M, h, grad_out, index0_offsets, index1, q, k, grad_q, grad_k); + break; + case 32: + attention_step1_backward_cuda_kernel_v2<32><<>>(N, M, h, grad_out, index0_offsets, index1, q, k, grad_q, grad_k); + break; + default: + throw "d != 16 and d != 32"; + } + +} + +__global__ void attention_step2_forward_cuda_kernel_v2( // M, h, C//h + int N, int M, int h, int C, const float *attn, const float *v, + const int *index0, const int *index1, float *output) { + + int c_idx = blockIdx.z; + int h_idx = blockIdx.y; + int m_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (m_idx >= M || h_idx >= h || c_idx >= C / h) return; + + int idx1 = index1[m_idx]; + float val = attn[m_idx*h+h_idx] * v[idx1*C+h_idx*C/h+c_idx]; + int idx0 = index0[m_idx]; + atomicAdd(output+idx0*C+h_idx*C/h+c_idx, val); +} + +__global__ void attention_step2_backward_cuda_kernel_v2( // M, h, C//h + int N, int M, int h, int C, const float *grad_out, const int *index0, const int *index1, const float *attn, const float *v, + float *grad_attn, float *grad_v) { + + int c_idx = blockIdx.z; + int h_idx = blockIdx.y; + int m_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (m_idx >= M || h_idx >= h || c_idx >= C / h) return; + + int idx0 = index0[m_idx]; + int idx1 = index1[m_idx]; + int grad_out_idx = idx0*C+h_idx*C/h+c_idx; + atomicAdd(grad_attn+m_idx*h+h_idx, grad_out[grad_out_idx] * v[idx1*C+h_idx*C/h+c_idx]); + atomicAdd(grad_v+idx1*C+h_idx*C/h+c_idx, grad_out[grad_out_idx] * attn[m_idx*h+h_idx]); +} + +void attention_step2_forward_cuda_launcher_v2(int N, int M, int h, int C, const float *attn, const float *v, + const int *index0, const int *index1, float *output) { + // input: attn: (M, h), v: (N, h, C/h), index0: (M, ), index1: (M, ) + //dim3 blocks(DIVUP(C/h, THREADS_PER_BLOCK), h, M); + dim3 blocks(DIVUP(M, THREADS_PER_BLOCK), h, C/h); + dim3 threads(THREADS_PER_BLOCK); + attention_step2_forward_cuda_kernel_v2<<>>(N, M, h, C, attn, v, index0, index1, output); +} + +void attention_step2_backward_cuda_launcher_v2(int N, int M, int h, int C, const float *grad_out, const int *index0, const int *index1, + const float *attn, const float *v, float *grad_attn, float *grad_v) { + // input: grad_output: (n, nsample, c), output: grad_input1: (n, c), grad_input2: (n, c) + //dim3 blocks(DIVUP(C/h, THREADS_PER_BLOCK), h, M); + dim3 blocks(DIVUP(M, THREADS_PER_BLOCK), h, C/h); + dim3 threads(THREADS_PER_BLOCK); + attention_step2_backward_cuda_kernel_v2<<>>(N, M, h, C, grad_out, index0, index1, attn, v, grad_attn, grad_v); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention_v2/attention_cuda_kernel_v2.h b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention_v2/attention_cuda_kernel_v2.h new file mode 100644 index 0000000000000000000000000000000000000000..d7e7f047bc318928ddb9402acbcdf20204596450 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention_v2/attention_cuda_kernel_v2.h @@ -0,0 +1,26 @@ +#ifndef _ATTENTION_V2_CUDA_KERNEL +#define _ATTENTION_V2_CUDA_KERNEL +#include +#include +#include + +void attention_step1_forward_cuda_v2(int N, int M, int h, int C, const unsigned int n_max, at::Tensor q_tensor, at::Tensor k_tensor, at::Tensor index0_tensor_offsets, at::Tensor index1_tensor, at::Tensor attn_tensor); +void attention_step1_backward_cuda_v2(int N, int M, int h, int C, const unsigned int n_max, at::Tensor grad_out_tensor, at::Tensor index0_tensor_offsets, at::Tensor index1_tensor, at::Tensor q_tensor, at::Tensor k_tensor, at::Tensor grad_q_tensor, at::Tensor grad_k_tensor); + +void attention_step2_forward_cuda_v2(int N, int M, int h, int C, at::Tensor attn_tensor, at::Tensor v_tensor, at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor output_tensor); +void attention_step2_backward_cuda_v2(int N, int M, int h, int C, at::Tensor grad_out_tensor, at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor attn_tensor, at::Tensor v_tensor, at::Tensor grad_attn_tensor, at::Tensor grad_v_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void attention_step1_forward_cuda_launcher_v2(int N, int M, int h, int C, const unsigned int n_max, const float *q, const float *k, const int *index0_offsets, const int *index1, float *attn); +void attention_step1_backward_cuda_launcher_v2(int N, int M, int h, int C, const unsigned int n_max, const float *grad_out, const int *index0_offsets, const int *index1, const float *q, const float *k, float *grad_q, float *grad_k); + +void attention_step2_forward_cuda_launcher_v2(int N, int M, int h, int C, const float *attn, const float *v, const int *index0, const int *index1, float *output); +void attention_step2_backward_cuda_launcher_v2(int N, int M, int h, int C, const float *grad_out, const int *index0, const int *index1, const float *attn, const float *v, float *grad_attn, float *grad_v); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention_v2/attention_cuda_v2.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention_v2/attention_cuda_v2.cpp new file mode 100644 index 0000000000000000000000000000000000000000..03329e5e6c4bb5ffd4320a94fc25a481785668ed --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/attention_v2/attention_cuda_v2.cpp @@ -0,0 +1,55 @@ +#include +#include +#include +#include "attention_cuda_kernel_v2.h" + +void attention_step1_forward_cuda_v2(int N, int M, int h, int C, const unsigned int n_max, at::Tensor q_tensor, at::Tensor k_tensor, + at::Tensor index0_tensor_offsets, at::Tensor index1_tensor, at::Tensor attn_tensor) +{ + const float *q = q_tensor.data_ptr(); + const float *k = k_tensor.data_ptr(); + const int *index0_offsets = index0_tensor_offsets.data_ptr(); + const int *index1 = index1_tensor.data_ptr(); + float *attn = attn_tensor.data_ptr(); + attention_step1_forward_cuda_launcher_v2(N, M, h, C, n_max, q, k, index0_offsets, index1, attn); +} + +void attention_step1_backward_cuda_v2(int N, int M, int h, int C, const unsigned int n_max, at::Tensor grad_out_tensor, + at::Tensor index0_tensor_offsets, at::Tensor index1_tensor, at::Tensor q_tensor, at::Tensor k_tensor, + at::Tensor grad_q_tensor, at::Tensor grad_k_tensor) +{ + const float *grad_out = grad_out_tensor.data_ptr(); + const int *index0_offsets = index0_tensor_offsets.data_ptr(); + const int *index1 = index1_tensor.data_ptr(); + const float *q = q_tensor.data_ptr(); + const float *k = k_tensor.data_ptr(); + float *grad_q = grad_q_tensor.data_ptr(); + float *grad_k = grad_k_tensor.data_ptr(); + attention_step1_backward_cuda_launcher_v2(N, M, h, C, n_max, grad_out, index0_offsets, index1, q, k, grad_q, grad_k); +} + +void attention_step2_forward_cuda_v2(int N, int M, int h, int C, at::Tensor attn_tensor, at::Tensor v_tensor, + at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor output_tensor) +{ + const float *attn = attn_tensor.data_ptr(); + const float *v = v_tensor.data_ptr(); + const int *index0 = index0_tensor.data_ptr(); + const int *index1 = index1_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + attention_step2_forward_cuda_launcher_v2(N, M, h, C, attn, v, index0, index1, output); +} + + +void attention_step2_backward_cuda_v2(int N, int M, int h, int C, at::Tensor grad_out_tensor, + at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor attn_tensor, at::Tensor v_tensor, + at::Tensor grad_attn_tensor, at::Tensor grad_v_tensor) +{ + const float *grad_out = grad_out_tensor.data_ptr(); + const int *index0 = index0_tensor.data_ptr(); + const int *index1 = index1_tensor.data_ptr(); + const float *attn = attn_tensor.data_ptr(); + const float *v = v_tensor.data_ptr(); + float *grad_attn = grad_attn_tensor.data_ptr(); + float *grad_v = grad_v_tensor.data_ptr(); + attention_step2_backward_cuda_launcher_v2(N, M, h, C, grad_out, index0, index1, attn, v, grad_attn, grad_v); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/cuda_utils.h b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/cuda_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..e67749c4f5f8964ffb5916c13f5260cf8df45f52 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/cuda_utils.h @@ -0,0 +1,23 @@ +#ifndef _CUDA_UTILS_H +#define _CUDA_UTILS_H + +#include +#include + +#define TOTAL_THREADS 1024 +#define THREADS_PER_BLOCK 256 +#define DIVUP(m, n) ((m) / (n) + ((m) % (n) > 0)) + +inline int opt_n_threads(int work_size) { + const int pow_2 = std::log(static_cast(work_size)) / std::log(2.0); + return std::max(std::min(1 << pow_2, TOTAL_THREADS), 1); +} + +inline dim3 opt_block_config(int x, int y) { + const int x_threads = opt_n_threads(x); + const int y_threads = std::max(std::min(opt_n_threads(y), TOTAL_THREADS / x_threads), 1); + dim3 block_config(x_threads, y_threads, 1); + return block_config; +} + +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/grouping/grouping_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/grouping/grouping_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6f7990adaf43f0a77050eed0d55adad19f256e10 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/grouping/grouping_cuda.cpp @@ -0,0 +1,21 @@ +#include +#include +#include +#include "grouping_cuda_kernel.h" + + +void grouping_forward_cuda(int m, int nsample, int c, at::Tensor input_tensor, at::Tensor idx_tensor, at::Tensor output_tensor) +{ + const float *input = input_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + grouping_forward_cuda_launcher(m, nsample, c, input, idx, output); +} + +void grouping_backward_cuda(int m, int nsample, int c, at::Tensor grad_output_tensor, at::Tensor idx_tensor, at::Tensor grad_input_tensor) +{ + const float *grad_output = grad_output_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + float *grad_input = grad_input_tensor.data_ptr(); + grouping_backward_cuda_launcher(m, nsample, c, grad_output, idx, grad_input); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/grouping/grouping_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/grouping/grouping_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..58ec0a21a2949f9f82504ccd24597c544c50af40 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/grouping/grouping_cuda_kernel.cu @@ -0,0 +1,40 @@ +#include "../cuda_utils.h" +#include "grouping_cuda_kernel.h" + + +__global__ void grouping_forward_cuda_kernel(int m, int nsample, int c, const float *__restrict__ input, const int *__restrict__ idx, float *__restrict__ output) { + // input: input: (n, c), idx: (m, nsample), output: (m, nsample, c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= m * nsample * c) return; + const int c_idx = index % c; + const int nsample_idx = (index / c) % nsample; + const int m_idx = index / nsample / c; + const int input_idx = idx[m_idx * nsample + nsample_idx] * c + c_idx; + output[index] = input[input_idx]; +} + +__global__ void grouping_backward_cuda_kernel(int m, int nsample, int c, const float *__restrict__ grad_output, const int *__restrict__ idx, float *__restrict__ grad_input) { + // input: grad_output: (m, nsample, c), idx: (m, nsample), output: grad_input: (n, c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= m * nsample * c) return; + const int c_idx = index % c; + const int nsample_idx = (index / c) % nsample; + const int m_idx = index / nsample / c; + const int input_idx = idx[m_idx * nsample + nsample_idx] * c + c_idx; + atomicAdd(grad_input + input_idx, grad_output[index]); +} + +void grouping_forward_cuda_launcher(int m, int nsample, int c, const float *input, const int *idx, float *output) { + // input: input: (n, c), idx: (m, nsample), output: (m, nsample, c) + dim3 blocks(DIVUP(m * nsample * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + grouping_forward_cuda_kernel<<>>(m, nsample, c, input, idx, output); +} + +void grouping_backward_cuda_launcher(int m, int nsample, int c, const float *grad_output, const int *idx, float *grad_input) +{ + // input: grad_output: (m, nsample, c), idx: (m, nsample), output: grad_input: (n, c) + dim3 blocks(DIVUP(m * nsample * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + grouping_backward_cuda_kernel<<>>(m, nsample, c, grad_output, idx, grad_input); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/grouping/grouping_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/grouping/grouping_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..3db4aaa9fad5811d559d47c500e4b00f0165d9b4 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/grouping/grouping_cuda_kernel.h @@ -0,0 +1,20 @@ +#ifndef _GROUPING_CUDA_KERNEL +#define _GROUPING_CUDA_KERNEL +#include +#include +#include + +void grouping_forward_cuda(int m, int nsample, int c, at::Tensor input_tensor, at::Tensor idx_tensor, at::Tensor output_tensor); +void grouping_backward_cuda(int m, int nsample, int c, at::Tensor grad_output_tensor, at::Tensor idx_tensor, at::Tensor grad_input_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void grouping_forward_cuda_launcher(int m, int nsample, int c, const float *input, const int *idx, float *output); +void grouping_backward_cuda_launcher(int m, int nsample, int c, const float *grad_output, const int *idx, float *grad_input); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/interpolation/interpolation_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/interpolation/interpolation_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f2c1b0078f4b70626705d7b3f5d1d65d37ee6de7 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/interpolation/interpolation_cuda.cpp @@ -0,0 +1,23 @@ +#include +#include +#include +#include "interpolation_cuda_kernel.h" + + +void interpolation_forward_cuda(int n, int c, int k, at::Tensor input_tensor, at::Tensor idx_tensor, at::Tensor weight_tensor, at::Tensor output_tensor) +{ + const float *input = input_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + const float *weight = weight_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + interpolation_forward_cuda_launcher(n, c, k, input, idx, weight, output); +} + +void interpolation_backward_cuda(int n, int c, int k, at::Tensor grad_output_tensor, at::Tensor idx_tensor, at::Tensor weight_tensor, at::Tensor grad_input_tensor) +{ + const float *grad_output = grad_output_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + const float *weight = weight_tensor.data_ptr(); + float *grad_input = grad_input_tensor.data_ptr(); + interpolation_backward_cuda_launcher(n, c, k, grad_output, idx, weight, grad_input); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/interpolation/interpolation_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/interpolation/interpolation_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..f560d8c92c6eac865b8c1e1dc27140fe3fcc2250 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/interpolation/interpolation_cuda_kernel.cu @@ -0,0 +1,47 @@ +#include "../cuda_utils.h" +#include "interpolation_cuda_kernel.h" + + +__global__ void interpolation_forward_cuda_kernel(int n, int c, int k, const float *input, const int *idx, const float *weight, float *output) +{ + // input: input: (m, c), idx: (n, k), weight: (n, k), output: output (n, c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= n * c) return; + int c_idx = index % c; + int n_idx = index / c; + for (int i = 0; i < k; i++) + { + int idx_idx = n_idx * k + i; + int input_idx = idx[idx_idx] * c + c_idx; + output[index] += input[input_idx] * weight[idx_idx]; + } +} + +__global__ void interpolation_backward_cuda_kernel(int n, int c, int k, const float *grad_output, const int *idx, const float *weight, float *grad_input) +{ + // input: grad_output: (n, c), idx: (n, k), weight: (n, k), output: grad_input (m, c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= n * c) return; + int c_idx = index % c; + int n_idx = index / c; + for (int i = 0; i < k; i++) + { + int idx_idx = n_idx * k + i; + int input_idx = idx[idx_idx] * c + c_idx; + atomicAdd(grad_input + input_idx, grad_output[index] * weight[idx_idx]); + } +} + +void interpolation_forward_cuda_launcher(int n, int c, int k, const float *input, const int *idx, const float *weight, float *output) { + // input: input: (m, c), idx: (n, k), weight: (n, k), output: output (n, c) + dim3 blocks(DIVUP(n * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + interpolation_forward_cuda_kernel<<>>(n, c, k, input, idx, weight, output); +} + +void interpolation_backward_cuda_launcher(int n, int c, int k, const float *grad_output, const int *idx, const float *weight, float *grad_input) { + // input: grad_output: (n, c), idx: (n, k), weight: (n, k), output: grad_input (m, c) + dim3 blocks(DIVUP(n * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + interpolation_backward_cuda_kernel<<>>(n, c, k, grad_output, idx, weight, grad_input); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/interpolation/interpolation_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/interpolation/interpolation_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..309e5dd0a34ccb58807bbf32389ba65e7ee6961b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/interpolation/interpolation_cuda_kernel.h @@ -0,0 +1,20 @@ +#ifndef _INTERPOLATION_CUDA_KERNEL +#define _INTERPOLATION_CUDA_KERNEL +#include +#include +#include + +void interpolation_forward_cuda(int n, int c, int k, at::Tensor input_tensor, at::Tensor idx_tensor, at::Tensor weight_tensor, at::Tensor output_tensor); +void interpolation_backward_cuda(int n, int c, int k, at::Tensor grad_output_tensor, at::Tensor idx_tensor, at::Tensor weight_tensor, at::Tensor grad_input_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void interpolation_forward_cuda_launcher(int n, int c, int k, const float *input, const int *idx, const float *weight, float *output); +void interpolation_backward_cuda_launcher(int n, int c, int k, const float *grad_output, const int *idx, const float *weight, float *grad_input); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/knnquery/knnquery_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/knnquery/knnquery_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a90fe9fc44f8d4963b8e0c3246ad09a8c2e01222 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/knnquery/knnquery_cuda.cpp @@ -0,0 +1,16 @@ +#include +#include +#include +#include "knnquery_cuda_kernel.h" + + +void knnquery_cuda(int m, int nsample, at::Tensor xyz_tensor, at::Tensor new_xyz_tensor, at::Tensor offset_tensor, at::Tensor new_offset_tensor, at::Tensor idx_tensor, at::Tensor dist2_tensor) +{ + const float *xyz = xyz_tensor.data_ptr(); + const float *new_xyz = new_xyz_tensor.data_ptr(); + const int *offset = offset_tensor.data_ptr(); + const int *new_offset = new_offset_tensor.data_ptr(); + int *idx = idx_tensor.data_ptr(); + float *dist2 = dist2_tensor.data_ptr(); + knnquery_cuda_launcher(m, nsample, xyz, new_xyz, offset, new_offset, idx, dist2); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/knnquery/knnquery_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/knnquery/knnquery_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..83762bc0110e38c7b5fa8adf0ef4ce255bc9d0b9 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/knnquery/knnquery_cuda_kernel.cu @@ -0,0 +1,116 @@ +#include "../cuda_utils.h" +#include "knnquery_cuda_kernel.h" + + +__device__ void swap_float(float *x, float *y) +{ + float tmp = *x; + *x = *y; + *y = tmp; +} + + +__device__ void swap_int(int *x, int *y) +{ + int tmp = *x; + *x = *y; + *y = tmp; +} + + +__device__ void reheap(float *dist, int *idx, int k) +{ + int root = 0; + int child = root * 2 + 1; + while (child < k) + { + if(child + 1 < k && dist[child+1] > dist[child]) + child++; + if(dist[root] > dist[child]) + return; + swap_float(&dist[root], &dist[child]); + swap_int(&idx[root], &idx[child]); + root = child; + child = root * 2 + 1; + } +} + + +__device__ void heap_sort(float *dist, int *idx, int k) +{ + int i; + for (i = k - 1; i > 0; i--) + { + swap_float(&dist[0], &dist[i]); + swap_int(&idx[0], &idx[i]); + reheap(dist, idx, i); + } +} + + +__device__ int get_bt_idx(int idx, const int *offset) +{ + int i = 0; + while (1) + { + if (idx < offset[i]) + break; + else + i++; + } + return i; +} + + +__global__ void knnquery_cuda_kernel(int m, int nsample, const float *__restrict__ xyz, const float *__restrict__ new_xyz, const int *__restrict__ offset, const int *__restrict__ new_offset, int *__restrict__ idx, float *__restrict__ dist2) { + // input: xyz (n, 3) new_xyz (m, 3) + // output: idx (m, nsample) dist2 (m, nsample) + int pt_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (pt_idx >= m) return; + + new_xyz += pt_idx * 3; + idx += pt_idx * nsample; + dist2 += pt_idx * nsample; + int bt_idx = get_bt_idx(pt_idx, new_offset); + int start; + if (bt_idx == 0) + start = 0; + else + start = offset[bt_idx - 1]; + int end = offset[bt_idx]; + + float new_x = new_xyz[0]; + float new_y = new_xyz[1]; + float new_z = new_xyz[2]; + + float best_dist[100]; + int best_idx[100]; + for(int i = 0; i < nsample; i++){ + best_dist[i] = 1e10; + best_idx[i] = start; + } + for(int i = start; i < end; i++){ + float x = xyz[i * 3 + 0]; + float y = xyz[i * 3 + 1]; + float z = xyz[i * 3 + 2]; + float d2 = (new_x - x) * (new_x - x) + (new_y - y) * (new_y - y) + (new_z - z) * (new_z - z); + if (d2 < best_dist[0]){ + best_dist[0] = d2; + best_idx[0] = i; + reheap(best_dist, best_idx, nsample); + } + } + heap_sort(best_dist, best_idx, nsample); + for(int i = 0; i < nsample; i++){ + idx[i] = best_idx[i]; + dist2[i] = best_dist[i]; + } +} + + +void knnquery_cuda_launcher(int m, int nsample, const float *xyz, const float *new_xyz, const int *offset, const int *new_offset, int *idx, float *dist2) { + // input: new_xyz: (m, 3), xyz: (n, 3), idx: (m, nsample) + dim3 blocks(DIVUP(m, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + knnquery_cuda_kernel<<>>(m, nsample, xyz, new_xyz, offset, new_offset, idx, dist2); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/knnquery/knnquery_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/knnquery/knnquery_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..3c0aedfe8fbe6c427ee15bb550c2c1829e9f4b97 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/knnquery/knnquery_cuda_kernel.h @@ -0,0 +1,18 @@ +#ifndef _KNNQUERY_CUDA_KERNEL +#define _KNNQUERY_CUDA_KERNEL +#include +#include +#include + +void knnquery_cuda(int m, int nsample, at::Tensor xyz_tensor, at::Tensor new_xyz_tensor, at::Tensor offset_tensor, at::Tensor new_offset_tensor, at::Tensor idx_tensor, at::Tensor dist2_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void knnquery_cuda_launcher(int m, int nsample, const float *xyz, const float *new_xyz, const int *offset, const int *new_offset, int *idx, float *dist2); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/pointops_api.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/pointops_api.cpp new file mode 100644 index 0000000000000000000000000000000000000000..812789f7d4fdf961b960641ba6c2fd660c16a654 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/pointops_api.cpp @@ -0,0 +1,45 @@ +#include +#include + +#include "knnquery/knnquery_cuda_kernel.h" +#include "sampling/sampling_cuda_kernel.h" +#include "grouping/grouping_cuda_kernel.h" +#include "interpolation/interpolation_cuda_kernel.h" +#include "aggregation/aggregation_cuda_kernel.h" +#include "subtraction/subtraction_cuda_kernel.h" +#include "attention/attention_cuda_kernel.h" +#include "rpe/relative_pos_encoding_cuda_kernel.h" +#include "attention_v2/attention_cuda_kernel_v2.h" +#include "rpe_v2/relative_pos_encoding_cuda_kernel_v2.h" + + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("knnquery_cuda", &knnquery_cuda, "knnquery_cuda"); + m.def("furthestsampling_cuda", &furthestsampling_cuda, "furthestsampling_cuda"); + m.def("grouping_forward_cuda", &grouping_forward_cuda, "grouping_forward_cuda"); + m.def("grouping_backward_cuda", &grouping_backward_cuda, "grouping_backward_cuda"); + m.def("interpolation_forward_cuda", &interpolation_forward_cuda, "interpolation_forward_cuda"); + m.def("interpolation_backward_cuda", &interpolation_backward_cuda, "interpolation_backward_cuda"); + m.def("subtraction_forward_cuda", &subtraction_forward_cuda, "subtraction_forward_cuda"); + m.def("subtraction_backward_cuda", &subtraction_backward_cuda, "subtraction_backward_cuda"); + m.def("aggregation_forward_cuda", &aggregation_forward_cuda, "aggregation_forward_cuda"); + m.def("aggregation_backward_cuda", &aggregation_backward_cuda, "aggregation_backward_cuda"); + m.def("attention_step1_forward_cuda", &attention_step1_forward_cuda, "attention_step1_forward_cuda"); + m.def("attention_step1_backward_cuda", &attention_step1_backward_cuda, "attention_step1_backward_cuda"); + m.def("attention_step2_forward_cuda", &attention_step2_forward_cuda, "attention_step2_forward_cuda"); + m.def("attention_step2_backward_cuda", &attention_step2_backward_cuda, "attention_step2_backward_cuda"); + m.def("dot_prod_with_idx_forward_cuda", &dot_prod_with_idx_forward_cuda, "dot_prod_with_idx_forward_cuda"); + m.def("dot_prod_with_idx_backward_cuda", &dot_prod_with_idx_backward_cuda, "dot_prod_with_idx_backward_cuda"); + m.def("attention_step2_with_rel_pos_value_forward_cuda", &attention_step2_with_rel_pos_value_forward_cuda, "attention_step2_with_rel_pos_value_forward_cuda"); + m.def("attention_step2_with_rel_pos_value_backward_cuda", &attention_step2_with_rel_pos_value_backward_cuda, "attention_step2_with_rel_pos_value_backward_cuda"); + m.def("attention_step1_forward_cuda_v2", &attention_step1_forward_cuda_v2, "attention_step1_forward_cuda_v2"); + m.def("attention_step1_backward_cuda_v2", &attention_step1_backward_cuda_v2, "attention_step1_backward_cuda_v2"); + m.def("attention_step2_forward_cuda_v2", &attention_step2_forward_cuda_v2, "attention_step2_forward_cuda_v2"); + m.def("attention_step2_backward_cuda_v2", &attention_step2_backward_cuda_v2, "attention_step2_backward_cuda_v2"); + m.def("dot_prod_with_idx_forward_cuda_v2", &dot_prod_with_idx_forward_cuda_v2, "dot_prod_with_idx_forward_cuda_v2"); + m.def("dot_prod_with_idx_backward_cuda_v2", &dot_prod_with_idx_backward_cuda_v2, "dot_prod_with_idx_backward_cuda_v2"); + m.def("attention_step2_with_rel_pos_value_forward_cuda_v2", &attention_step2_with_rel_pos_value_forward_cuda_v2, "attention_step2_with_rel_pos_value_forward_cuda_v2"); + m.def("attention_step2_with_rel_pos_value_backward_cuda_v2", &attention_step2_with_rel_pos_value_backward_cuda_v2, "attention_step2_with_rel_pos_value_backward_cuda_v2"); + m.def("dot_prod_with_idx_forward_cuda_v3", &dot_prod_with_idx_forward_cuda_v3, "dot_prod_with_idx_forward_cuda_v3"); + m.def("dot_prod_with_idx_backward_cuda_v3", &dot_prod_with_idx_backward_cuda_v3, "dot_prod_with_idx_backward_cuda_v3"); + } diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe/relative_pos_encoding_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe/relative_pos_encoding_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..333a1b1a083a2b1e9699917c7f74f73eab176f43 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe/relative_pos_encoding_cuda.cpp @@ -0,0 +1,59 @@ +#include +#include +#include +#include "relative_pos_encoding_cuda_kernel.h" + +void dot_prod_with_idx_forward_cuda(int N, int M, int h, int hdim, at::Tensor q_tensor, at::Tensor index_tensor, + at::Tensor table_tensor, at::Tensor rel_idx_tensor, at::Tensor output_tensor) +{ + const float *q = q_tensor.data_ptr(); + const float *table = table_tensor.data_ptr(); + const int *index = index_tensor.data_ptr(); + const int *rel_idx = rel_idx_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + dot_prod_with_idx_forward_cuda_launcher(N, M, h, hdim, q, index, table, rel_idx, output); +} + +void dot_prod_with_idx_backward_cuda(int N, int M, int h, int hdim, at::Tensor grad_out_tensor, + at::Tensor q_tensor, at::Tensor index_tensor, at::Tensor table_tensor, at::Tensor rel_idx_tensor, + at::Tensor grad_q_tensor, at::Tensor grad_table_tensor) +{ + const float *grad_out = grad_out_tensor.data_ptr(); + const float *q = q_tensor.data_ptr(); + const int *index = index_tensor.data_ptr(); + const float *table = table_tensor.data_ptr(); + const int *rel_idx = rel_idx_tensor.data_ptr(); + float *grad_q = grad_q_tensor.data_ptr(); + float *grad_table = grad_table_tensor.data_ptr(); + dot_prod_with_idx_backward_cuda_launcher(N, M, h, hdim, grad_out, q, index, table, rel_idx, grad_q, grad_table); +} + +void attention_step2_with_rel_pos_value_forward_cuda(int N, int M, int h, int hdim, at::Tensor attn_tensor, at::Tensor v_tensor, + at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor table_tensor, at::Tensor rel_idx_tensor, at::Tensor output_tensor) +{ + const float *attn = attn_tensor.data_ptr(); + const float *v = v_tensor.data_ptr(); + const int *index0 = index0_tensor.data_ptr(); + const int *index1 = index1_tensor.data_ptr(); + const float *table = table_tensor.data_ptr(); + const int *rel_idx = rel_idx_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + attention_step2_with_rel_pos_value_forward_cuda_launcher(N, M, h, hdim, attn, v, index0, index1, table, rel_idx, output); +} + +void attention_step2_with_rel_pos_value_backward_cuda(int N, int M, int h, int hdim, at::Tensor grad_out_tensor, + at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor attn_tensor, at::Tensor v_tensor, at::Tensor table_tensor, + at::Tensor rel_idx_tensor, at::Tensor grad_attn_tensor, at::Tensor grad_v_tensor, at::Tensor grad_table_tensor) +{ + const float *grad_out = grad_out_tensor.data_ptr(); + const int *index0 = index0_tensor.data_ptr(); + const int *index1 = index1_tensor.data_ptr(); + const float *attn = attn_tensor.data_ptr(); + const float *v = v_tensor.data_ptr(); + const float *table = table_tensor.data_ptr(); + const int *rel_idx = rel_idx_tensor.data_ptr(); + float *grad_attn = grad_attn_tensor.data_ptr(); + float *grad_v = grad_v_tensor.data_ptr(); + float *grad_table = grad_table_tensor.data_ptr(); + attention_step2_with_rel_pos_value_backward_cuda_launcher(N, M, h, hdim, grad_out, index0, index1, attn, v, table, rel_idx, grad_attn, grad_v, grad_table); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe/relative_pos_encoding_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe/relative_pos_encoding_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..8ccab4db11e11d57a600787e6cef8a4a281a2791 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe/relative_pos_encoding_cuda_kernel.cu @@ -0,0 +1,136 @@ +/* written by Xin Lai. Email: xinlai@cse.cuhk.edu.hk */ + +#include "../cuda_utils.h" +#include "relative_pos_encoding_cuda_kernel.h" + + +__global__ void dot_prod_with_idx_forward_cuda_kernel( // M, h, hdim + int N, int M, int h, int hdim, const float *q, const int *index, + const float *table, const int *rel_idx, float *output) { + // input: q: (N, h, hdim), index: (M), table: (L, h, hdim, 3), rel_idx: (M, 3), output: (M, h) + + int c_idx = blockIdx.z; + int h_idx = blockIdx.y; + int thread_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (thread_idx >= M*3 || h_idx >= h || c_idx >= hdim) return; + + int dim = thread_idx % 3; + int m_idx = thread_idx / 3; + + int q_idx = index[m_idx]; + int rel_idx_dim = rel_idx[thread_idx]; + float rel_table_val = table[rel_idx_dim*h*hdim*3+h_idx*hdim*3+c_idx*3+dim]; + float val = q[q_idx*h*hdim+h_idx*hdim+c_idx] * rel_table_val; + atomicAdd(output+m_idx*h+h_idx, val); +} + +__global__ void dot_prod_with_idx_backward_cuda_kernel( // M, h, hdim + int N, int M, int h, int hdim, const float *grad_out, const float *q, const int *index, + const float *table, const int *rel_idx, float *grad_q, float *grad_table) { + + int c_idx = blockIdx.z; + int h_idx = blockIdx.y; + int thread_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (thread_idx >= M*3 || h_idx >= h || c_idx >= hdim) return; + + int dim = thread_idx % 3; + int m_idx = thread_idx / 3; + + int q_idx = index[m_idx]; + int rel_idx_dim = rel_idx[thread_idx]; + int grad_out_idx = m_idx*h+h_idx; + float grad_out_value = grad_out[grad_out_idx]; + + float rel_table_val = table[rel_idx_dim*h*hdim*3+h_idx*hdim*3+c_idx*3+dim]; + atomicAdd(grad_q+q_idx*h*hdim+h_idx*hdim+c_idx, grad_out_value * rel_table_val); + + float q_value = q[q_idx*h*hdim+h_idx*hdim+c_idx]; + atomicAdd(grad_table+rel_idx_dim*h*hdim*3+h_idx*hdim*3+c_idx*3+dim, grad_out_value * q_value); +} + +void dot_prod_with_idx_forward_cuda_launcher(int N, int M, int h, int hdim, const float *q, const int *index, + const float *table, const int *rel_idx, float *output) { + // input: q: (N, h, hdim), index: (M), table: (L, h, hdim, 3), rel_idx: (M, 3) + //dim3 blocks(DIVUP(hdim, THREADS_PER_BLOCK), h, M); + dim3 blocks(DIVUP(M*3, THREADS_PER_BLOCK), h, hdim); + dim3 threads(THREADS_PER_BLOCK); + dot_prod_with_idx_forward_cuda_kernel<<>>(N, M, h, hdim, q, index, table, rel_idx, output); +} + +void dot_prod_with_idx_backward_cuda_launcher(int N, int M, int h, int hdim, const float *grad_out, + const float *q, const int *index, const float *table, const int *rel_idx, float *grad_q, float *grad_table) { + // input: grad_out: (M, h), output: grad_q: (N, h, hdim), grad_table: (L, h, hdim, 3) + //dim3 blocks(DIVUP(hdim, THREADS_PER_BLOCK), h, M); + dim3 blocks(DIVUP(M*3, THREADS_PER_BLOCK), h, hdim); + dim3 threads(THREADS_PER_BLOCK); + dot_prod_with_idx_backward_cuda_kernel<<>>(N, M, h, hdim, grad_out, q, index, table, rel_idx, grad_q, grad_table); +} + +__global__ void attention_step2_with_rel_pos_value_forward_cuda_kernel( // M, h, hdim + int N, int M, int h, int hdim, const float *attn, const float *v, + const int *index0, const int *index1, const float *table, const int *rel_idx, float *output) { + // input: attn: (M, h), v: (N, h, hdim), index0: (M, ), index1: (M, ), table: (L, h, hdim, 3), rel_idx: (M, 3) + + int c_idx = blockIdx.z; + int h_idx = blockIdx.y; + int thread_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (thread_idx >= M*3 || h_idx >= h || c_idx >= hdim) return; + + int dim = thread_idx % 3; + int m_idx = thread_idx / 3; + + int idx1 = index1[m_idx]; + + int rel_idx_dim = rel_idx[thread_idx]; + float table_val = table[rel_idx_dim*h*hdim*3+h_idx*hdim*3+c_idx*3+dim]; + + float val = attn[m_idx*h+h_idx] * (v[idx1*h*hdim+h_idx*hdim+c_idx] / 3.0 + table_val); + + int idx0 = index0[m_idx]; + atomicAdd(output+idx0*h*hdim+h_idx*hdim+c_idx, val); +} + + +__global__ void attention_step2_with_rel_pos_value_backward_cuda_kernel( // M, h, hdim + int N, int M, int h, int hdim, const float *grad_out, const int *index0, const int *index1, const float *attn, const float *v, const float *table, + const int *rel_idx, float *grad_attn, float *grad_v, float *grad_table) { + // input: attn: (M, h), v: (N, h, hdim), index0: (M, ), index1: (M, ), table: (L, h, hdim, 3), rel_idx: (M, 3) + + int c_idx = blockIdx.z; + int h_idx = blockIdx.y; + int thread_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (thread_idx >= M*3 || h_idx >= h || c_idx >= hdim) return; + + int dim = thread_idx % 3; + int m_idx = thread_idx / 3; + + int idx0 = index0[m_idx]; + int idx1 = index1[m_idx]; + int grad_out_idx = idx0*h*hdim+h_idx*hdim+c_idx; + + int rel_idx_dim = rel_idx[thread_idx]; + float table_val = table[rel_idx_dim*h*hdim*3+h_idx*hdim*3+c_idx*3+dim]; + float grad_out_value = grad_out[grad_out_idx]; + + atomicAdd(grad_attn+m_idx*h+h_idx, grad_out_value * (v[idx1*h*hdim+h_idx*hdim+c_idx]/3 + table_val)); + atomicAdd(grad_v+idx1*h*hdim+h_idx*hdim+c_idx, grad_out_value * attn[m_idx*h+h_idx]/3); + atomicAdd(grad_table+rel_idx_dim*h*hdim*3+h_idx*hdim*3+c_idx*3+dim, grad_out_value * attn[m_idx*h+h_idx]); +} + +void attention_step2_with_rel_pos_value_forward_cuda_launcher(int N, int M, int h, int hdim, const float *attn, const float *v, const int *index0, + const int *index1, const float *table, const int *rel_idx, float *output) { + // input: attn: (M, h), v: (N, h, hdim), index0: (M, ), index1: (M, ), table: (L, h, hdim, 3), rel_idx: (M, 3) + //dim3 blocks(DIVUP(hdim, THREADS_PER_BLOCK), h, M); + dim3 blocks(DIVUP(M*3, THREADS_PER_BLOCK), h, hdim); + dim3 threads(THREADS_PER_BLOCK); + attention_step2_with_rel_pos_value_forward_cuda_kernel<<>>(N, M, h, hdim, attn, v, index0, index1, table, rel_idx, output); +} + +void attention_step2_with_rel_pos_value_backward_cuda_launcher(int N, int M, int h, int hdim, const float *grad_out, const int *index0, + const int *index1, const float *attn, const float *v, const float *table, const int *rel_idx, float *grad_attn, float *grad_v, float *grad_table) { + // input: grad_output: (n, nsample, c), output: grad_input1: (n, c), grad_input2: (n, c) + //dim3 blocks(DIVUP(hdim, THREADS_PER_BLOCK), h, M); + dim3 blocks(DIVUP(M*3, THREADS_PER_BLOCK), h, hdim); + dim3 threads(THREADS_PER_BLOCK); + attention_step2_with_rel_pos_value_backward_cuda_kernel<<>>(N, M, h, hdim, grad_out, index0, index1, attn, v, table, rel_idx, grad_attn, grad_v, grad_table); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe/relative_pos_encoding_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe/relative_pos_encoding_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..cafc7b69152fff9c0c440a093346fb6005923db0 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe/relative_pos_encoding_cuda_kernel.h @@ -0,0 +1,26 @@ +#ifndef _RPE_CUDA_KERNEL +#define _RPE_CUDA_KERNEL +#include +#include +#include + +void dot_prod_with_idx_forward_cuda(int N, int M, int h, int hdim, at::Tensor q_tensor, at::Tensor index_tensor, at::Tensor table_tensor, at::Tensor rel_idx_tensor, at::Tensor output_tensor); +void dot_prod_with_idx_backward_cuda(int N, int M, int h, int hdim, at::Tensor grad_out_tensor, at::Tensor q_tensor, at::Tensor index_tensor, at::Tensor table_tensor, at::Tensor rel_idx_tensor, at::Tensor grad_q_tensor, at::Tensor grad_table_tensor); + +void attention_step2_with_rel_pos_value_forward_cuda(int N, int M, int h, int hdim, at::Tensor attn_tensor, at::Tensor v_tensor, at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor table_tensor, at::Tensor rel_idx_tensor, at::Tensor output_tensor); +void attention_step2_with_rel_pos_value_backward_cuda(int N, int M, int h, int hdim, at::Tensor grad_out_tensor, at::Tensor index0_tensor, at::Tensor index1_tensor, at::Tensor attn_tensor, at::Tensor v_tensor, at::Tensor table_tensor, at::Tensor rel_idx_tensor, at::Tensor grad_attn_tensor, at::Tensor grad_v_tensor, at::Tensor grad_table_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void dot_prod_with_idx_forward_cuda_launcher(int N, int M, int h, int hdim, const float *q, const int *index, const float *table, const int *rel_idx, float *output); +void dot_prod_with_idx_backward_cuda_launcher(int N, int M, int h, int hdim, const float *grad_out, const float *q, const int *index, const float *table, const int *rel_idx, float *grad_q, float *grad_table); + +void attention_step2_with_rel_pos_value_forward_cuda_launcher(int N, int M, int h, int hdim, const float *attn, const float *v, const int *index0, const int *index1, const float *table, const int *rel_idx, float *output); +void attention_step2_with_rel_pos_value_backward_cuda_launcher(int N, int M, int h, int hdim, const float *grad_out, const int *index0, const int *index1, const float *attn, const float *v, const float *table, const int *rel_idx, float *grad_attn, float *grad_v, float *grad_table); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe_v2/relative_pos_encoding_cuda_kernel_v2.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe_v2/relative_pos_encoding_cuda_kernel_v2.cu new file mode 100644 index 0000000000000000000000000000000000000000..afa536a480c6f20969528e34d7b9d9ea29175b06 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe_v2/relative_pos_encoding_cuda_kernel_v2.cu @@ -0,0 +1,527 @@ +/* written by Xin Lai. Email: xinlai@cse.cuhk.edu.hk */ + +#include "../cuda_utils.h" +#include "relative_pos_encoding_cuda_kernel_v2.h" + + +// N, M, h, q, index_q, k, index_k, table_q, table_k, rel_idx, rel_idx_offsets, output + +template +__global__ void dot_prod_with_idx_forward_cuda_kernel_v2( // M, h, hdim + int N, int M, int h, const float *q, const int *index_q, const float *k, const int *index_k, + const float *table_q, const float *table_k, const int *rel_idx, const int *rel_idx_offsets, + const int *sort_indices, float *output) { + // input: q: (N, h, hdim), index: (M), table: (L, h, hdim, 3), rel_idx: (M, 3), output: (M, h) + + int h_idx = blockIdx.y; + int t_idx = blockIdx.x; + int n_idx = threadIdx.x; + int C = h*d; + + __shared__ int start, end; + if(n_idx == 0){ + start = rel_idx_offsets[t_idx]; + end = rel_idx_offsets[t_idx+1]; + // printf("e2: start: %d, end: %d\n", start, end); + } + + __syncthreads(); + + int m_idx_prev = start + n_idx; + // if(m_idx_prev >= end) + // return; + + __shared__ int m_idx; + if(n_idx == 0) + m_idx = sort_indices[m_idx_prev]; + + __syncthreads(); + + __shared__ int rel_idx_vec[3]; + if(n_idx < 3) + rel_idx_vec[n_idx] = rel_idx[m_idx*3 + n_idx]; + + __syncthreads(); + + __shared__ float table_q_vec[d]; + __shared__ float table_k_vec[d]; + + for(int i = n_idx; i < 2*d; i += blockDim.x){ + if (i < d){ + int ind0 = rel_idx_vec[0] * C * 3 + h_idx * d * 3 + i * 3 + 0; + int ind1 = rel_idx_vec[1] * C * 3 + h_idx * d * 3 + i * 3 + 1; + int ind2 = rel_idx_vec[2] * C * 3 + h_idx * d * 3 + i * 3 + 2; + table_q_vec[i] = table_q[ind0] + table_q[ind1] + table_q[ind2]; + } else{ + int ind0 = rel_idx_vec[0] * C * 3 + h_idx * d * 3 + (i-d) * 3 + 0; + int ind1 = rel_idx_vec[1] * C * 3 + h_idx * d * 3 + (i-d) * 3 + 1; + int ind2 = rel_idx_vec[2] * C * 3 + h_idx * d * 3 + (i-d) * 3 + 2; + table_k_vec[i-d] = table_k[ind0] + table_k[ind1] + table_k[ind2]; + } + } + + __syncthreads(); + + for(int i = m_idx_prev; i < end; i += blockDim.x){ + float sum = 0; + int m_idx_i = sort_indices[i]; + int q_idx = index_q[m_idx_i]; + int k_idx = index_k[m_idx_i]; + for(int j = 0; j < d; j++){ + sum += q[q_idx*C + h_idx*d + j] * table_q_vec[j]; + sum += k[k_idx*C + h_idx*d + j] * table_k_vec[j]; + } + output[m_idx_i*h + h_idx] = sum; + } +} + +// N, M, h, hdim, grad_out, q, index_q, k, index_k, table_q, table_k, rel_idx, rel_idx_offsets, sort_indices, grad_q, grad_k, grad_table_q, grad_table_k + +template +__global__ void dot_prod_with_idx_backward_cuda_kernel_v2( // M, h, hdim + int N, int M, int h, const float *grad_out, const float *q, const int *index_q, + const float *k, const int *index_k, const float *table_q, const float *table_k, + const int *rel_idx, const int *rel_idx_offsets, const int *sort_indices, float *grad_q, + float *grad_k, float *grad_table_q, float *grad_table_k) { + + int h_idx = blockIdx.y; + int t_idx = blockIdx.x; + int n_idx = threadIdx.x; + int C = h*d; + + __shared__ int start, end; + if(n_idx == 0){ + start = rel_idx_offsets[t_idx]; + end = rel_idx_offsets[t_idx+1]; + } + + __syncthreads(); + + int m_idx_prev = start + n_idx; + // if(m_idx_prev >= end) + // return; + + __shared__ int m_idx; + if(n_idx == 0) + m_idx = sort_indices[m_idx_prev]; + + __syncthreads(); + + __shared__ int rel_idx_vec[3]; + if(n_idx < 3) + rel_idx_vec[n_idx] = rel_idx[m_idx*3 + n_idx]; + + __syncthreads(); + + __shared__ float table_q_vec[d]; + __shared__ float table_k_vec[d]; + + for(int i = n_idx; i < 2*d; i += blockDim.x){ + if (i < d){ + int ind0 = rel_idx_vec[0] * C * 3 + h_idx * d * 3 + i * 3 + 0; + int ind1 = rel_idx_vec[1] * C * 3 + h_idx * d * 3 + i * 3 + 1; + int ind2 = rel_idx_vec[2] * C * 3 + h_idx * d * 3 + i * 3 + 2; + table_q_vec[i] = table_q[ind0] + table_q[ind1] + table_q[ind2]; + } else{ + int ind0 = rel_idx_vec[0] * C * 3 + h_idx * d * 3 + (i-d) * 3 + 0; + int ind1 = rel_idx_vec[1] * C * 3 + h_idx * d * 3 + (i-d) * 3 + 1; + int ind2 = rel_idx_vec[2] * C * 3 + h_idx * d * 3 + (i-d) * 3 + 2; + table_k_vec[i-d] = table_k[ind0] + table_k[ind1] + table_k[ind2]; + } + } + + __shared__ float gradient_q[d]; + __shared__ float gradient_k[d]; + for(int i = n_idx; i < d; i += blockDim.x){ + gradient_q[i] = 0; + gradient_k[i] = 0; + } + + __syncthreads(); + + for(int i = m_idx_prev; i < end; i += blockDim.x){ + int m_idx_i = sort_indices[i]; + int q_idx = index_q[m_idx_i]; + int k_idx = index_k[m_idx_i]; + float grad_out_i = grad_out[m_idx_i*h+h_idx]; + for(int j = 0; j < d; j++){ + atomicAdd(&gradient_q[j], q[q_idx*C + h_idx*d + j] * grad_out_i); + atomicAdd(&gradient_k[j], k[k_idx*C + h_idx*d + j] * grad_out_i); + atomicAdd(grad_q + q_idx*C + h_idx*d + j, table_q_vec[j] * grad_out_i); + atomicAdd(grad_k + k_idx*C + h_idx*d + j, table_k_vec[j] * grad_out_i); + } + } + + __syncthreads(); + + for(int i = n_idx; i < d*2; i += blockDim.x){ + if(i < d){ + atomicAdd(grad_table_q + rel_idx_vec[0] * C * 3 + h_idx * d * 3 + i * 3, gradient_q[i]); + atomicAdd(grad_table_q + rel_idx_vec[1] * C * 3 + h_idx * d * 3 + i * 3 + 1, gradient_q[i]); + atomicAdd(grad_table_q + rel_idx_vec[2] * C * 3 + h_idx * d * 3 + i * 3 + 2, gradient_q[i]); + }else{ + atomicAdd(grad_table_k + rel_idx_vec[0] * C * 3 + h_idx * d * 3 + (i-d) * 3, gradient_k[i-d]); + atomicAdd(grad_table_k + rel_idx_vec[1] * C * 3 + h_idx * d * 3 + (i-d) * 3 + 1, gradient_k[i-d]); + atomicAdd(grad_table_k + rel_idx_vec[2] * C * 3 + h_idx * d * 3 + (i-d) * 3 + 2, gradient_k[i-d]); + } + } + + // int c_idx = blockIdx.z; + // int h_idx = blockIdx.y; + // int thread_idx = blockIdx.x * blockDim.x + threadIdx.x; + // if (thread_idx >= M*3 || h_idx >= h || c_idx >= hdim) return; + + // int dim = thread_idx % 3; + // int m_idx = thread_idx / 3; + + // int q_idx = index[m_idx]; + // int rel_idx_dim = rel_idx[thread_idx]; + // int grad_out_idx = m_idx*h+h_idx; + // float grad_out_value = grad_out[grad_out_idx]; + + // float rel_table_val = table[rel_idx_dim*h*hdim*3+h_idx*hdim*3+c_idx*3+dim]; + // atomicAdd(grad_q+q_idx*h*hdim+h_idx*hdim+c_idx, grad_out_value * rel_table_val); + + // float q_value = q[q_idx*h*hdim+h_idx*hdim+c_idx]; + // atomicAdd(grad_table+rel_idx_dim*h*hdim*3+h_idx*hdim*3+c_idx*3+dim, grad_out_value * q_value); +} + +void dot_prod_with_idx_forward_cuda_launcher_v2(int N, int M, int h, int hdim, int n_max, int T, const float *q, + const int *index_q, const float *k, const int *index_k, const float *table_q, const float *table_k, + const int *rel_idx, const int *rel_idx_offsets, const int *sort_indices, float *output) +{ + // input: q: (N, h, hdim), index: (M), table: (L, h, hdim, 3), rel_idx: (M, 3) + //dim3 blocks(DIVUP(hdim, THREADS_PER_BLOCK), h, M); + dim3 blocks(T, h); + // dim3 threads(THREADS_PER_BLOCK); + + unsigned int n_threads = opt_n_threads(n_max); + n_threads = n_threads == n_max ? n_threads : n_threads * 2; + n_threads = n_threads > 1024 ? 512 : n_threads; + + // printf("e1: T: %d, h: %d, n_threads: %d\n", T, h, n_threads); + + switch (hdim) { + case 16: + dot_prod_with_idx_forward_cuda_kernel_v2<16><<>>(N, M, h, q, index_q, k, index_k, table_q, table_k, rel_idx, rel_idx_offsets, sort_indices, output); + break; + case 32: + dot_prod_with_idx_forward_cuda_kernel_v2<32><<>>(N, M, h, q, index_q, k, index_k, table_q, table_k, rel_idx, rel_idx_offsets, sort_indices, output); + break; + default: + throw "d != 16 and d != 32"; + } +} + +void dot_prod_with_idx_backward_cuda_launcher_v2(int N, int M, int h, int hdim, int n_max, int T, + const float *grad_out, const float *q, const int *index_q, const float *k, const int *index_k, + const float *table_q, const float *table_k, const int *rel_idx, const int *rel_idx_offsets, const int *sort_indices, + float *grad_q, float *grad_k, float *grad_table_q, float *grad_table_k) +{ + // input: grad_out: (M, h), output: grad_q: (N, h, hdim), grad_table: (L, h, hdim, 3) + //dim3 blocks(DIVUP(hdim, THREADS_PER_BLOCK), h, M); + // dim3 blocks(DIVUP(M*3, THREADS_PER_BLOCK), h, hdim); + // dim3 threads(THREADS_PER_BLOCK); + + dim3 blocks(T, h); + // dim3 threads(THREADS_PER_BLOCK); + + unsigned int n_threads = opt_n_threads(n_max); + n_threads = n_threads == n_max ? n_threads : n_threads * 2; + n_threads = n_threads > 1024 ? 512 : n_threads; + + switch (hdim) { + case 16: + dot_prod_with_idx_backward_cuda_kernel_v2<16><<>>(N, M, h, grad_out, q, index_q, k, index_k, table_q, table_k, rel_idx, rel_idx_offsets, sort_indices, grad_q, grad_k, grad_table_q, grad_table_k); + break; + case 32: + dot_prod_with_idx_backward_cuda_kernel_v2<32><<>>(N, M, h, grad_out, q, index_q, k, index_k, table_q, table_k, rel_idx, rel_idx_offsets, sort_indices, grad_q, grad_k, grad_table_q, grad_table_k); + break; + default: + throw "d != 16 and d != 32"; + } +} + + + +template +__global__ void dot_prod_with_idx_forward_cuda_kernel_v3( // M, h, hdim + int N, int M, int h, const float *q, const int *index_q_offsets, const float *k, const int *index_k, + const float *table_q, const float *table_k, const int *rel_idx, float *output) { + // input: q: (N, h, hdim), index: (M), table: (L, h, hdim, 3), rel_idx: (M, 3), output: (M, h) + int q_idx = blockIdx.x; + int h_idx = blockIdx.y; + int n_idx = threadIdx.x; + int C = h*d; + + __shared__ float query_vec[d]; + __shared__ int start, end; + if (n_idx == 0){ + start = index_q_offsets[q_idx]; + end = index_q_offsets[q_idx+1]; + } + for(int i = n_idx; i < d; i += blockDim.x) + query_vec[i] = q[q_idx*C + h_idx*d + i]; + + __syncthreads(); + + int m_idx = start + n_idx; + if(m_idx >= end) + return; + + int k_idx = index_k[m_idx]; + int r_idx1 = rel_idx[m_idx*3], r_idx2 = rel_idx[m_idx*3+1], r_idx3 = rel_idx[m_idx*3+2]; + float sum = 0; + for(int i = 0; i < d; i++){ + float table_q_scalar_i = table_q[r_idx1*C*3+h_idx*d*3+i*3] + table_q[r_idx2*C*3+h_idx*d*3+i*3+1] + table_q[r_idx3*C*3+h_idx*d*3+i*3+2]; + sum += query_vec[i] * table_q_scalar_i; + float table_k_scalar_i = table_k[r_idx1*C*3+h_idx*d*3+i*3] + table_k[r_idx2*C*3+h_idx*d*3+i*3+1] + table_k[r_idx3*C*3+h_idx*d*3+i*3+2]; + sum += k[k_idx*C+h_idx*d+i] * table_k_scalar_i; + } + output[m_idx*h + h_idx] = sum; + +} + +// N, M, h, hdim, grad_out, q, index_q, k, index_k, table_q, table_k, rel_idx, rel_idx_offsets, sort_indices, grad_q, grad_k, grad_table_q, grad_table_k + +template +__global__ void dot_prod_with_idx_backward_cuda_kernel_v3( // M, h, hdim + int N, int M, int h, const float *grad_out, const float *q, const int *index_q_offsets, + const float *k, const int *index_k, const float *table_q, const float *table_k, + const int *rel_idx, float *grad_q, float *grad_k, float *grad_table_q, float *grad_table_k) { + + int q_idx = blockIdx.x; + int h_idx = blockIdx.y; + int n_idx = threadIdx.x; + int C = h*d; + + __shared__ float query_vec[d]; + __shared__ int start, end; + if (n_idx == 0){ + start = index_q_offsets[q_idx]; + end = index_q_offsets[q_idx+1]; + } + for(int i = n_idx; i < d; i += blockDim.x) + query_vec[i] = q[q_idx*C + h_idx*d + i]; + + __shared__ float gradients_q[d]; + for(int i = n_idx; i < d; i += blockDim.x){ + gradients_q[i] = 0; + } + + __syncthreads(); + + int m_idx = start + n_idx; + + if(m_idx < end){ + int k_idx = index_k[m_idx]; + int r_idx1 = rel_idx[m_idx*3], r_idx2 = rel_idx[m_idx*3+1], r_idx3 = rel_idx[m_idx*3+2]; + float gradient = grad_out[m_idx*h + h_idx]; + for(int i = 0; i < d; i++){ + float table_q_scalar_i = table_q[r_idx1*C*3+h_idx*d*3+i*3] + table_q[r_idx2*C*3+h_idx*d*3+i*3+1] + table_q[r_idx3*C*3+h_idx*d*3+i*3+2]; + float table_k_scalar_i = table_k[r_idx1*C*3+h_idx*d*3+i*3] + table_k[r_idx2*C*3+h_idx*d*3+i*3+1] + table_k[r_idx3*C*3+h_idx*d*3+i*3+2]; + float q_scalar_i = query_vec[i]; + float k_scalar_i = k[k_idx*C+h_idx*d+i]; + atomicAdd(&gradients_q[i], table_q_scalar_i * gradient); + atomicAdd(grad_k+k_idx*C+h_idx*d+i, table_k_scalar_i * gradient); + atomicAdd(grad_table_q+r_idx1*C*3+h_idx*d*3+i*3, q_scalar_i * gradient); + atomicAdd(grad_table_q+r_idx2*C*3+h_idx*d*3+i*3+1, q_scalar_i * gradient); + atomicAdd(grad_table_q+r_idx3*C*3+h_idx*d*3+i*3+2, q_scalar_i * gradient); + atomicAdd(grad_table_k+r_idx1*C*3+h_idx*d*3+i*3, k_scalar_i * gradient); + atomicAdd(grad_table_k+r_idx2*C*3+h_idx*d*3+i*3+1, k_scalar_i * gradient); + atomicAdd(grad_table_k+r_idx3*C*3+h_idx*d*3+i*3+2, k_scalar_i * gradient); + } + } + __syncthreads(); + + for(int i = n_idx; i < d; i += blockDim.x){ + grad_q[q_idx*C+h_idx*d+i] = gradients_q[i]; + } +} + +void dot_prod_with_idx_forward_cuda_launcher_v3(int N, int M, int h, int hdim, int n_max, const float *q, + const int *index_q_offsets, const float *k, const int *index_k, const float *table_q, const float *table_k, + const int *rel_idx, float *output) +{ + // input: q: (N, h, hdim), index: (M), table: (L, h, hdim, 3), rel_idx: (M, 3) + //dim3 blocks(DIVUP(hdim, THREADS_PER_BLOCK), h, M); + dim3 blocks(N, h); + // dim3 threads(THREADS_PER_BLOCK); + + unsigned int n_threads = opt_n_threads(n_max); + n_threads = n_threads == n_max ? n_threads : n_threads * 2; + + // printf("e1: h: %d, n_max: %d, n_threads: %d\n", h, n_max, n_threads); + + switch (hdim) { + case 16: + dot_prod_with_idx_forward_cuda_kernel_v3<16><<>>(N, M, h, q, index_q_offsets, k, index_k, table_q, table_k, rel_idx, output); + break; + case 32: + dot_prod_with_idx_forward_cuda_kernel_v3<32><<>>(N, M, h, q, index_q_offsets, k, index_k, table_q, table_k, rel_idx, output); + break; + default: + throw "d != 16 and d != 32"; + } +} + +void dot_prod_with_idx_backward_cuda_launcher_v3(int N, int M, int h, int hdim, int n_max, + const float *grad_out, const float *q, const int *index_q_offsets, const float *k, const int *index_k, + const float *table_q, const float *table_k, const int *rel_idx, + float *grad_q, float *grad_k, float *grad_table_q, float *grad_table_k) +{ + // input: grad_out: (M, h), output: grad_q: (N, h, hdim), grad_table: (L, h, hdim, 3) + //dim3 blocks(DIVUP(hdim, THREADS_PER_BLOCK), h, M); + // dim3 blocks(DIVUP(M*3, THREADS_PER_BLOCK), h, hdim); + // dim3 threads(THREADS_PER_BLOCK); + + dim3 blocks(N, h); + // dim3 threads(THREADS_PER_BLOCK); + + unsigned int n_threads = opt_n_threads(n_max); + n_threads = n_threads == n_max ? n_threads : n_threads * 2; + + switch (hdim) { + case 16: + dot_prod_with_idx_backward_cuda_kernel_v3<16><<>>(N, M, h, grad_out, q, index_q_offsets, k, index_k, table_q, table_k, rel_idx, grad_q, grad_k, grad_table_q, grad_table_k); + break; + case 32: + dot_prod_with_idx_backward_cuda_kernel_v3<32><<>>(N, M, h, grad_out, q, index_q_offsets, k, index_k, table_q, table_k, rel_idx, grad_q, grad_k, grad_table_q, grad_table_k); + break; + default: + throw "d != 16 and d != 32"; + } +} + + +template +__global__ void attention_step2_with_rel_pos_value_forward_cuda_kernel_v2( // M, h, hdim + int N, int M, int h, const float *attn, const float *v, + const int *index0_offsets, const int *index1, const float *table, const int *rel_idx, float *output) { + // input: attn: (M, h), v: (N, h, hdim), index0: (M, ), index1: (M, ), table: (L, h, hdim, 3), rel_idx: (M, 3) + + int q_idx = blockIdx.x; + int h_idx = blockIdx.y; + int n_idx = threadIdx.x; + + int C = h*d; + + __shared__ int start, end; + __shared__ float result[d]; + + if (n_idx == 0){ + start = index0_offsets[q_idx]; + end = index0_offsets[q_idx+1]; + } + for (int i = n_idx; i < d; i += blockDim.x){ + result[i] = 0; + } + + __syncthreads(); + + int m_idx = start + n_idx; + if (m_idx < end){ + float attn_scalar = attn[m_idx*h + h_idx]; + int r_idx1 = rel_idx[m_idx*3], r_idx2 = rel_idx[m_idx*3+1], r_idx3 = rel_idx[m_idx*3+2]; + for(int i = 0; i < d; i ++){ + int v_idx = index1[m_idx]; + float table_scaler_i = table[r_idx1*C*3+h_idx*d*3+i*3] + table[r_idx2*C*3+h_idx*d*3+i*3+1] + table[r_idx3*C*3+h_idx*d*3+i*3+2]; + float value_scaler_i = v[v_idx*C + h_idx*d + i]; + atomicAdd(&result[i], (table_scaler_i + value_scaler_i) * attn_scalar); + } + } + + __syncthreads(); + + for (int i = n_idx; i < d; i += blockDim.x) + output[q_idx*C + h_idx*d + i] = result[i]; +} + + +template +__global__ void attention_step2_with_rel_pos_value_backward_cuda_kernel_v2( // M, h, hdim + int N, int M, int h, const float *grad_out, const int *index0_offsets, const int *index1, const float *attn, const float *v, const float *table, + const int *rel_idx, float *grad_attn, float *grad_v, float *grad_table) { + // input: attn: (M, h), v: (N, h, hdim), index0: (M, ), index1: (M, ), table: (L, h, hdim, 3), rel_idx: (M, 3) + + int q_idx = blockIdx.x; + int h_idx = blockIdx.y; + int n_idx = threadIdx.x; + + int C = h*d; + + __shared__ int start, end; + __shared__ float gradients[d]; + + if (n_idx == 0){ + start = index0_offsets[q_idx]; + end = index0_offsets[q_idx+1]; + } + for (int i = n_idx; i < d; i += blockDim.x){ + gradients[i] = grad_out[q_idx*C + h_idx*d + i]; + } + + __syncthreads(); + + int m_idx = start + n_idx; + if (m_idx < end){ + int v_idx = index1[m_idx]; + int r_idx1 = rel_idx[m_idx*3], r_idx2 = rel_idx[m_idx*3+1], r_idx3 = rel_idx[m_idx*3+2]; + float attn_scalar = attn[m_idx*h + h_idx]; + float grad_attn_sum = 0; + for (int i = 0; i < d; i++){ + float grad_out_scaler_i = gradients[i]; + float table_scaler_i = table[r_idx1*C*3+h_idx*d*3+i*3] + table[r_idx2*C*3+h_idx*d*3+i*3+1] + table[r_idx3*C*3+h_idx*d*3+i*3+2]; + float value_scaler_i = v[v_idx*C + h_idx*d + i]; + grad_attn_sum += (table_scaler_i + value_scaler_i) * grad_out_scaler_i; + atomicAdd(grad_v + v_idx*C + h_idx*d + i, attn_scalar * grad_out_scaler_i); + atomicAdd(grad_table + r_idx1*C*3 + h_idx*d*3 + i*3, attn_scalar * grad_out_scaler_i); + atomicAdd(grad_table + r_idx2*C*3 + h_idx*d*3 + i*3 + 1, attn_scalar * grad_out_scaler_i); + atomicAdd(grad_table + r_idx3*C*3 + h_idx*d*3 + i*3 + 2, attn_scalar * grad_out_scaler_i); + } + grad_attn[m_idx*h + h_idx] = grad_attn_sum; + } +} + +void attention_step2_with_rel_pos_value_forward_cuda_launcher_v2(int N, int M, int h, int hdim, int n_max, const float *attn, const float *v, const int *index0_offsets, + const int *index1, const float *table, const int *rel_idx, float *output) { + // input: attn: (M, h), v: (N, h, hdim), index0: (M, ), index1: (M, ), table: (L, h, hdim, 3), rel_idx: (M, 3) + //dim3 blocks(DIVUP(hdim, THREADS_PER_BLOCK), h, M); + // dim3 blocks(DIVUP(M*3, THREADS_PER_BLOCK), h, hdim); + // dim3 threads(THREADS_PER_BLOCK); + dim3 blocks(N, h); + unsigned int n_threads = opt_n_threads(n_max); + n_threads = n_threads == n_max ? n_threads : n_threads * 2; + + switch (hdim) { + case 16: + attention_step2_with_rel_pos_value_forward_cuda_kernel_v2<16><<>>(N, M, h, attn, v, index0_offsets, index1, table, rel_idx, output); + break; + case 32: + attention_step2_with_rel_pos_value_forward_cuda_kernel_v2<32><<>>(N, M, h, attn, v, index0_offsets, index1, table, rel_idx, output); + break; + default: + throw "d != 16 and d != 32"; + } +} + +void attention_step2_with_rel_pos_value_backward_cuda_launcher_v2(int N, int M, int h, int hdim, int n_max, const float *grad_out, const int *index0_offsets, + const int *index1, const float *attn, const float *v, const float *table, const int *rel_idx, float *grad_attn, float *grad_v, float *grad_table) { + // input: grad_output: (n, nsample, c), output: grad_input1: (n, c), grad_input2: (n, c) + //dim3 blocks(DIVUP(hdim, THREADS_PER_BLOCK), h, M); + + dim3 blocks(N, h); + unsigned int n_threads = opt_n_threads(n_max); + n_threads = n_threads == n_max ? n_threads : n_threads * 2; + + switch (hdim) { + case 16: + attention_step2_with_rel_pos_value_backward_cuda_kernel_v2<16><<>>(N, M, h, grad_out, index0_offsets, index1, attn, v, table, rel_idx, grad_attn, grad_v, grad_table); + break; + case 32: + attention_step2_with_rel_pos_value_backward_cuda_kernel_v2<32><<>>(N, M, h, grad_out, index0_offsets, index1, attn, v, table, rel_idx, grad_attn, grad_v, grad_table); + break; + default: + throw "d != 16 and d != 32"; + } +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe_v2/relative_pos_encoding_cuda_kernel_v2.h b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe_v2/relative_pos_encoding_cuda_kernel_v2.h new file mode 100644 index 0000000000000000000000000000000000000000..648b152afe16d3011b62ff141a4e20b2a83579b4 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe_v2/relative_pos_encoding_cuda_kernel_v2.h @@ -0,0 +1,32 @@ +#ifndef _RPE_V2_CUDA_KERNEL +#define _RPE_V2_CUDA_KERNEL +#include +#include +#include + +void dot_prod_with_idx_forward_cuda_v2(int N, int M, int h, int hdim, int n_max, int T, at::Tensor q_tensor, at::Tensor index_q_tensor, at::Tensor k_tensor, at::Tensor index_k_tensor, at::Tensor table_q_tensor, at::Tensor table_k_tensor, at::Tensor rel_idx_tensor, at::Tensor rel_idx_offsets_tensor, at::Tensor sort_indices_tensor, at::Tensor output_tensor); +void dot_prod_with_idx_backward_cuda_v2(int N, int M, int h, int hdim, int n_max, int T, at::Tensor grad_out_tensor, at::Tensor q_tensor, at::Tensor index_q_tensor, at::Tensor k_tensor, at::Tensor index_k_tensor, at::Tensor table_q_tensor, at::Tensor table_k_tensor, at::Tensor rel_idx_tensor, at::Tensor rel_idx_offsets_tensor, at::Tensor sort_indices_tensor, at::Tensor grad_q_tensor, at::Tensor grad_k_tensor, at::Tensor grad_table_q_tensor, at::Tensor grad_table_k_tensor); + +void dot_prod_with_idx_forward_cuda_v3(int N, int M, int h, int hdim, int n_max, at::Tensor q_tensor, at::Tensor index_q_offsets_tensor, at::Tensor k_tensor, at::Tensor index_k_tensor, at::Tensor table_q_tensor, at::Tensor table_k_tensor, at::Tensor rel_idx_tensor, at::Tensor output_tensor); +void dot_prod_with_idx_backward_cuda_v3(int N, int M, int h, int hdim, int n_max, at::Tensor grad_out_tensor, at::Tensor q_tensor, at::Tensor index_q_offsets_tensor, at::Tensor k_tensor, at::Tensor index_k_tensor, at::Tensor table_q_tensor, at::Tensor table_k_tensor, at::Tensor rel_idx_tensor, at::Tensor grad_q_tensor, at::Tensor grad_k_tensor, at::Tensor grad_table_q_tensor, at::Tensor grad_table_k_tensor); + +void attention_step2_with_rel_pos_value_forward_cuda_v2(int N, int M, int h, int hdim, int n_max, at::Tensor attn_tensor, at::Tensor v_tensor, at::Tensor index0_offsets_tensor, at::Tensor index1_tensor, at::Tensor table_tensor, at::Tensor rel_idx_tensor, at::Tensor output_tensor); +void attention_step2_with_rel_pos_value_backward_cuda_v2(int N, int M, int h, int hdim, int n_max, at::Tensor grad_out_tensor, at::Tensor index0_offsets_tensor, at::Tensor index1_tensor, at::Tensor attn_tensor, at::Tensor v_tensor, at::Tensor table_tensor, at::Tensor rel_idx_tensor, at::Tensor grad_attn_tensor, at::Tensor grad_v_tensor, at::Tensor grad_table_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void dot_prod_with_idx_forward_cuda_launcher_v2(int N, int M, int h, int hdim, int n_max, int T, const float *q, const int *index_q, const float *k, const int *index_k, const float *table_q, const float *table_k, const int *rel_idx, const int *rel_idx_offsets, const int *sort_indices, float *output); +void dot_prod_with_idx_backward_cuda_launcher_v2(int N, int M, int h, int hdim, int n_max, int T, const float *grad_out, const float *q, const int *index_q, const float *k, const int *index_k, const float *table_q, const float *table_k, const int *rel_idx, const int *rel_idx_offsets, const int *sort_indices, float *grad_q, float *grad_k, float *grad_table_q, float *grad_table_k); + +void dot_prod_with_idx_forward_cuda_launcher_v3(int N, int M, int h, int hdim, int n_max, const float *q, const int *index_q_offsets, const float *k, const int *index_k, const float *table_q, const float *table_k, const int *rel_idx, float *output); +void dot_prod_with_idx_backward_cuda_launcher_v3(int N, int M, int h, int hdim, int n_max, const float *grad_out, const float *q, const int *index_q_offsets, const float *k, const int *index_k, const float *table_q, const float *table_k, const int *rel_idx, float *grad_q, float *grad_k, float *grad_table_q, float *grad_table_k); + +void attention_step2_with_rel_pos_value_forward_cuda_launcher_v2(int N, int M, int h, int hdim, int n_max, const float *attn, const float *v, const int *index0_offsets, const int *index1, const float *table, const int *rel_idx, float *output); +void attention_step2_with_rel_pos_value_backward_cuda_launcher_v2(int N, int M, int h, int hdim, int n_max, const float *grad_out, const int *index0_offsets, const int *index1, const float *attn, const float *v, const float *table, const int *rel_idx, float *grad_attn, float *grad_v, float *grad_table); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe_v2/relative_pos_encoding_cuda_v2.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe_v2/relative_pos_encoding_cuda_v2.cpp new file mode 100644 index 0000000000000000000000000000000000000000..091380881625d469d1eba681e63321e7f4ea1b2f --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/rpe_v2/relative_pos_encoding_cuda_v2.cpp @@ -0,0 +1,110 @@ +#include +#include +#include +#include "relative_pos_encoding_cuda_kernel_v2.h" + +void dot_prod_with_idx_forward_cuda_v2(int N, int M, int h, int hdim, int n_max, int T, at::Tensor q_tensor, + at::Tensor index_q_tensor, at::Tensor k_tensor, at::Tensor index_k_tensor, at::Tensor table_q_tensor, + at::Tensor table_k_tensor, at::Tensor rel_idx_tensor, at::Tensor rel_idx_offsets_tensor, at::Tensor sort_indices_tensor, at::Tensor output_tensor) +{ + const float *q = q_tensor.data_ptr(); + const int *index_q = index_q_tensor.data_ptr(); + const float *k = k_tensor.data_ptr(); + const int *index_k = index_k_tensor.data_ptr(); + const float *table_q = table_q_tensor.data_ptr(); + const float *table_k = table_k_tensor.data_ptr(); + const int *rel_idx = rel_idx_tensor.data_ptr(); + const int *rel_idx_offsets = rel_idx_offsets_tensor.data_ptr(); + const int *sort_indices = sort_indices_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + dot_prod_with_idx_forward_cuda_launcher_v2(N, M, h, hdim, n_max, T, q, index_q, k, index_k, table_q, table_k, rel_idx, rel_idx_offsets, sort_indices, output); +} + +void dot_prod_with_idx_backward_cuda_v2(int N, int M, int h, int hdim, int n_max, int T, at::Tensor grad_out_tensor, + at::Tensor q_tensor, at::Tensor index_q_tensor, at::Tensor k_tensor, at::Tensor index_k_tensor, + at::Tensor table_q_tensor, at::Tensor table_k_tensor, at::Tensor rel_idx_tensor, at::Tensor rel_idx_offsets_tensor, + at::Tensor sort_indices_tensor, at::Tensor grad_q_tensor, at::Tensor grad_k_tensor, at::Tensor grad_table_q_tensor, at::Tensor grad_table_k_tensor) +{ + const float *grad_out = grad_out_tensor.data_ptr(); + const float *q = q_tensor.data_ptr(); + const int *index_q = index_q_tensor.data_ptr(); + const float *k = k_tensor.data_ptr(); + const int *index_k = index_k_tensor.data_ptr(); + const float *table_q = table_q_tensor.data_ptr(); + const float *table_k = table_k_tensor.data_ptr(); + const int *rel_idx = rel_idx_tensor.data_ptr(); + const int *rel_idx_offsets = rel_idx_offsets_tensor.data_ptr(); + const int *sort_indices = sort_indices_tensor.data_ptr(); + float *grad_q = grad_q_tensor.data_ptr(); + float *grad_k = grad_k_tensor.data_ptr(); + float *grad_table_q = grad_table_q_tensor.data_ptr(); + float *grad_table_k = grad_table_k_tensor.data_ptr(); + dot_prod_with_idx_backward_cuda_launcher_v2(N, M, h, hdim, n_max, T, grad_out, q, index_q, k, index_k, table_q, table_k, rel_idx, rel_idx_offsets, sort_indices, grad_q, grad_k, grad_table_q, grad_table_k); +} + + +void dot_prod_with_idx_forward_cuda_v3(int N, int M, int h, int hdim, int n_max, at::Tensor q_tensor, + at::Tensor index_q_offsets_tensor, at::Tensor k_tensor, at::Tensor index_k_tensor, at::Tensor table_q_tensor, + at::Tensor table_k_tensor, at::Tensor rel_idx_tensor, at::Tensor output_tensor) +{ + const float *q = q_tensor.data_ptr(); + const int *index_q_offsets = index_q_offsets_tensor.data_ptr(); + const float *k = k_tensor.data_ptr(); + const int *index_k = index_k_tensor.data_ptr(); + const float *table_q = table_q_tensor.data_ptr(); + const float *table_k = table_k_tensor.data_ptr(); + const int *rel_idx = rel_idx_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + dot_prod_with_idx_forward_cuda_launcher_v3(N, M, h, hdim, n_max, q, index_q_offsets, k, index_k, table_q, table_k, rel_idx, output); +} + +void dot_prod_with_idx_backward_cuda_v3(int N, int M, int h, int hdim, int n_max, at::Tensor grad_out_tensor, + at::Tensor q_tensor, at::Tensor index_q_offsets_tensor, at::Tensor k_tensor, at::Tensor index_k_tensor, + at::Tensor table_q_tensor, at::Tensor table_k_tensor, at::Tensor rel_idx_tensor, at::Tensor grad_q_tensor, + at::Tensor grad_k_tensor, at::Tensor grad_table_q_tensor, at::Tensor grad_table_k_tensor) +{ + const float *grad_out = grad_out_tensor.data_ptr(); + const float *q = q_tensor.data_ptr(); + const int *index_q_offsets = index_q_offsets_tensor.data_ptr(); + const float *k = k_tensor.data_ptr(); + const int *index_k = index_k_tensor.data_ptr(); + const float *table_q = table_q_tensor.data_ptr(); + const float *table_k = table_k_tensor.data_ptr(); + const int *rel_idx = rel_idx_tensor.data_ptr(); + float *grad_q = grad_q_tensor.data_ptr(); + float *grad_k = grad_k_tensor.data_ptr(); + float *grad_table_q = grad_table_q_tensor.data_ptr(); + float *grad_table_k = grad_table_k_tensor.data_ptr(); + dot_prod_with_idx_backward_cuda_launcher_v3(N, M, h, hdim, n_max, grad_out, q, index_q_offsets, k, index_k, table_q, table_k, rel_idx, grad_q, grad_k, grad_table_q, grad_table_k); +} + + +void attention_step2_with_rel_pos_value_forward_cuda_v2(int N, int M, int h, int hdim, int n_max, at::Tensor attn_tensor, at::Tensor v_tensor, + at::Tensor index0_offsets_tensor, at::Tensor index1_tensor, at::Tensor table_tensor, at::Tensor rel_idx_tensor, at::Tensor output_tensor) +{ + const float *attn = attn_tensor.data_ptr(); + const float *v = v_tensor.data_ptr(); + const int *index0_offsets = index0_offsets_tensor.data_ptr(); + const int *index1 = index1_tensor.data_ptr(); + const float *table = table_tensor.data_ptr(); + const int *rel_idx = rel_idx_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + attention_step2_with_rel_pos_value_forward_cuda_launcher_v2(N, M, h, hdim, n_max, attn, v, index0_offsets, index1, table, rel_idx, output); +} + +void attention_step2_with_rel_pos_value_backward_cuda_v2(int N, int M, int h, int hdim, int n_max, at::Tensor grad_out_tensor, + at::Tensor index0_offsets_tensor, at::Tensor index1_tensor, at::Tensor attn_tensor, at::Tensor v_tensor, at::Tensor table_tensor, + at::Tensor rel_idx_tensor, at::Tensor grad_attn_tensor, at::Tensor grad_v_tensor, at::Tensor grad_table_tensor) +{ + const float *grad_out = grad_out_tensor.data_ptr(); + const int *index0_offsets = index0_offsets_tensor.data_ptr(); + const int *index1 = index1_tensor.data_ptr(); + const float *attn = attn_tensor.data_ptr(); + const float *v = v_tensor.data_ptr(); + const float *table = table_tensor.data_ptr(); + const int *rel_idx = rel_idx_tensor.data_ptr(); + float *grad_attn = grad_attn_tensor.data_ptr(); + float *grad_v = grad_v_tensor.data_ptr(); + float *grad_table = grad_table_tensor.data_ptr(); + attention_step2_with_rel_pos_value_backward_cuda_launcher_v2(N, M, h, hdim, n_max, grad_out, index0_offsets, index1, attn, v, table, rel_idx, grad_attn, grad_v, grad_table); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/sampling/sampling_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/sampling/sampling_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..395b446750e6c9030b3b53f7e65e3ba2b3c01e80 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/sampling/sampling_cuda.cpp @@ -0,0 +1,15 @@ +#include +#include +#include +#include "sampling_cuda_kernel.h" + + +void furthestsampling_cuda(int b, int n, at::Tensor xyz_tensor, at::Tensor offset_tensor, at::Tensor new_offset_tensor, at::Tensor tmp_tensor, at::Tensor idx_tensor) +{ + const float *xyz = xyz_tensor.data_ptr(); + const int *offset = offset_tensor.data_ptr(); + const int *new_offset = new_offset_tensor.data_ptr(); + float *tmp = tmp_tensor.data_ptr(); + int *idx = idx_tensor.data_ptr(); + furthestsampling_cuda_launcher(b, n, xyz, offset, new_offset, tmp, idx); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/sampling/sampling_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/sampling/sampling_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..eeb69da3d213ec17fc349334173de6ecfe07a784 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/sampling/sampling_cuda_kernel.cu @@ -0,0 +1,171 @@ +#include "../cuda_utils.h" +#include "sampling_cuda_kernel.h" + + +__device__ void __update(float *dists, int *dists_i, int idx1, int idx2) { + const float v1 = dists[idx1], v2 = dists[idx2]; + const int i1 = dists_i[idx1], i2 = dists_i[idx2]; + dists[idx1] = max(v1, v2); + dists_i[idx1] = v2 > v1 ? i2 : i1; +} + +// input xyz: (n, 3), tmp: (b, n_max) +// output idx (m) +template +__global__ void furthestsampling_cuda_kernel(const float *xyz, const int *offset, const int *new_offset, float *tmp, int *idx) +{ + __shared__ float dists[block_size]; + __shared__ int dists_i[block_size]; + + int bid = blockIdx.x; + int start_n, end_n, start_m, end_m, old; + if (bid == 0) { + start_n = 0; + end_n = offset[0]; + start_m = 0; + end_m = new_offset[0]; + old = 0; + } + else { + start_n = offset[bid - 1]; + end_n = offset[bid]; + start_m = new_offset[bid - 1]; + end_m = new_offset[bid]; + old = offset[bid - 1]; + } + + const int stride = block_size; + int tid = threadIdx.x; + if (tid == 0) idx[start_m] = start_n; + + __syncthreads(); + for (int j = start_m + 1; j < end_m; j++) + { + int besti = start_n; + float best = -1; + float x1 = xyz[old * 3 + 0]; + float y1 = xyz[old * 3 + 1]; + float z1 = xyz[old * 3 + 2]; + for (int k = start_n + tid; k < end_n; k += stride) + { + float x2 = xyz[k * 3 + 0]; + float y2 = xyz[k * 3 + 1]; + float z2 = xyz[k * 3 + 2]; + float d = (x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1) + (z2 - z1) * (z2 - z1); + float d2 = min(d, tmp[k]); + tmp[k] = d2; + besti = d2 > best ? k : besti; + best = d2 > best ? d2 : best; + } + dists[tid] = best; + dists_i[tid] = besti; + __syncthreads(); + + if (block_size >= 1024) { + if (tid < 512) { + __update(dists, dists_i, tid, tid + 512); + } + __syncthreads(); + } + if (block_size >= 512) { + if (tid < 256) { + __update(dists, dists_i, tid, tid + 256); + } + __syncthreads(); + } + if (block_size >= 256) { + if (tid < 128) { + __update(dists, dists_i, tid, tid + 128); + } + __syncthreads(); + } + if (block_size >= 128) { + if (tid < 64) { + __update(dists, dists_i, tid, tid + 64); + } + __syncthreads(); + } + if (block_size >= 64) { + if (tid < 32) { + __update(dists, dists_i, tid, tid + 32); + } + __syncthreads(); + } + if (block_size >= 32) { + if (tid < 16) { + __update(dists, dists_i, tid, tid + 16); + } + __syncthreads(); + } + if (block_size >= 16) { + if (tid < 8) { + __update(dists, dists_i, tid, tid + 8); + } + __syncthreads(); + } + if (block_size >= 8) { + if (tid < 4) { + __update(dists, dists_i, tid, tid + 4); + } + __syncthreads(); + } + if (block_size >= 4) { + if (tid < 2) { + __update(dists, dists_i, tid, tid + 2); + } + __syncthreads(); + } + if (block_size >= 2) { + if (tid < 1) { + __update(dists, dists_i, tid, tid + 1); + } + __syncthreads(); + } + + old = dists_i[0]; + if (tid == 0) + idx[j] = old; + } +} + +void furthestsampling_cuda_launcher(int b, int n, const float *xyz, const int *offset, const int *new_offset, float *tmp, int *idx) +{ + unsigned int n_threads = opt_n_threads(n); + switch (n_threads) { + case 1024: + furthestsampling_cuda_kernel<1024><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 512: + furthestsampling_cuda_kernel<512><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 256: + furthestsampling_cuda_kernel<256><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 128: + furthestsampling_cuda_kernel<128><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 64: + furthestsampling_cuda_kernel<64><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 32: + furthestsampling_cuda_kernel<32><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 16: + furthestsampling_cuda_kernel<16><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 8: + furthestsampling_cuda_kernel<8><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 4: + furthestsampling_cuda_kernel<4><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 2: + furthestsampling_cuda_kernel<2><<>>(xyz, offset, new_offset, tmp, idx); + break; + case 1: + furthestsampling_cuda_kernel<1><<>>(xyz, offset, new_offset, tmp, idx); + break; + default: + furthestsampling_cuda_kernel<512><<>>(xyz, offset, new_offset, tmp, idx); + } +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/sampling/sampling_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/sampling/sampling_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..c903f638eb30bbf5bf01141ed2740cc0cd37452e --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/sampling/sampling_cuda_kernel.h @@ -0,0 +1,18 @@ +#ifndef _SAMPLING_CUDA_KERNEL +#define _SAMPLING_CUDA_KERNEL +#include +#include +#include + +void furthestsampling_cuda(int b, int n, at::Tensor xyz_tensor, at::Tensor offset_tensor, at::Tensor new_offset_tensor, at::Tensor tmp_tensor, at::Tensor idx_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void furthestsampling_cuda_launcher(int b, int n, const float *xyz, const int *offset, const int *new_offset, float *tmp, int *idx); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/subtraction/subtraction_cuda.cpp b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/subtraction/subtraction_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b343857a1671eafe5199089973e863e2ac5b618c --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/subtraction/subtraction_cuda.cpp @@ -0,0 +1,23 @@ +#include +#include +#include +#include "subtraction_cuda_kernel.h" + + +void subtraction_forward_cuda(int n, int nsample, int c, at::Tensor input1_tensor, at::Tensor input2_tensor, at::Tensor idx_tensor, at::Tensor output_tensor) +{ + const float *input1 = input1_tensor.data_ptr(); + const float *input2 = input2_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + float *output = output_tensor.data_ptr(); + subtraction_forward_cuda_launcher(n, nsample, c, input1, input2, idx, output); +} + +void subtraction_backward_cuda(int n, int nsample, int c, at::Tensor idx_tensor, at::Tensor grad_output_tensor, at::Tensor grad_input1_tensor, at::Tensor grad_input2_tensor) +{ + const int *idx = idx_tensor.data_ptr(); + const float *grad_output = grad_output_tensor.data_ptr(); + float *grad_input1 = grad_input1_tensor.data_ptr(); + float *grad_input2 = grad_input2_tensor.data_ptr(); + subtraction_backward_cuda_launcher(n, nsample, c, idx, grad_output, grad_input1, grad_input2); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/subtraction/subtraction_cuda_kernel.cu b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/subtraction/subtraction_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..9b8d4f752940d580ee2b49f1b2946a8d6386d11a --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/subtraction/subtraction_cuda_kernel.cu @@ -0,0 +1,44 @@ +#include "../cuda_utils.h" +#include "subtraction_cuda_kernel.h" + + +__global__ void subtraction_forward_cuda_kernel(int n, int nsample, int c, const float *input1, const float *input2, const int *idx, float *output) { + // input: input1: (n, c), input2: (n, c), idx: (n, nsample), output: (n, nsample, c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= n * nsample * c) return; + const int c_idx = index % c; + const int nsample_idx = (index / c) % nsample; + const int n_idx = index / nsample / c; + const int idx_idx = n_idx * nsample + nsample_idx; + const int input1_idx = n_idx * c + c_idx; + const int input2_idx = idx[idx_idx] * c + c_idx; + output[index] = input1[input1_idx] - input2[input2_idx]; +} + +__global__ void subtraction_backward_cuda_kernel(int n, int nsample, int c, const int *idx, const float *grad_output, float *grad_input1, float *grad_input2) { + // input: grad_output: (n, nsample, c), output: grad_input1: (n, c), grad_input2: (n, c) + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= n * nsample * c) return; + const int c_idx = index % c; + const int nsample_idx = (index / c) % nsample; + const int n_idx = index / nsample / c; + const int idx_idx = n_idx * nsample + nsample_idx; + const int input1_idx = n_idx * c + c_idx; + const int input2_idx = idx[idx_idx] * c + c_idx; + atomicAdd(grad_input1 + input1_idx, grad_output[index]); + atomicAdd(grad_input2 + input2_idx, -grad_output[index]); +} + +void subtraction_forward_cuda_launcher(int n, int nsample, int c, const float *input1, const float *input2, const int *idx, float *output) { + // input: input1: (n, c), input2: (n, c), idx: (n, nsample), output: (n, nsample, c) + dim3 blocks(DIVUP(n * nsample * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + subtraction_forward_cuda_kernel<<>>(n, nsample, c, input1, input2, idx, output); +} + +void subtraction_backward_cuda_launcher(int n, int nsample, int c, const int *idx, const float *grad_output, float *grad_input1, float *grad_input2) { + // input: grad_output: (n, nsample, c), output: grad_input1: (n, c), grad_input2: (n, c) + dim3 blocks(DIVUP(n * nsample * c, THREADS_PER_BLOCK)); + dim3 threads(THREADS_PER_BLOCK); + subtraction_backward_cuda_kernel<<>>(n, nsample, c, idx, grad_output, grad_input1, grad_input2); +} diff --git a/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/subtraction/subtraction_cuda_kernel.h b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/subtraction/subtraction_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..856133d97bdd3dc58f29c746ff240fc9d489c22e --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/libs/pointops2/src/subtraction/subtraction_cuda_kernel.h @@ -0,0 +1,20 @@ +#ifndef _SUBTRACTION_CUDA_KERNEL +#define _SUBTRACTION_CUDA_KERNEL +#include +#include +#include + +void subtraction_forward_cuda(int n, int nsample, int c, at::Tensor input1_tensor, at::Tensor input2_tensor, at::Tensor idx_tensor, at::Tensor output_tensor); +void subtraction_backward_cuda(int n, int nsample, int c, at::Tensor idx_tensor, at::Tensor grad_output_tensor, at::Tensor grad_input1_tensor, at::Tensor grad_input2_tensor); + +#ifdef __cplusplus +extern "C" { +#endif + +void subtraction_forward_cuda_launcher(int n, int nsample, int c, const float *input1, const float *input2, const int *idx, float *output); +void subtraction_backward_cuda_launcher(int n, int nsample, int c, const int *idx, const float *grad_output, float *grad_input1, float *grad_input2); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a24e44cc46b4067a58f2516de4d0b799e2d7517a --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/__init__.py @@ -0,0 +1,23 @@ +from .defaults import DefaultDataset, ConcatDataset +from .builder import build_dataset +from .utils import point_collate_fn, collate_fn + +# indoor scene +from .s3dis import S3DISDataset +from .scannet import ScanNetDataset, ScanNet200Dataset +from .scannetpp import ScanNetPPDataset +from .scannet_pair import ScanNetPairDataset +from .arkitscenes import ArkitScenesDataset +from .structure3d import Structured3DDataset + +# outdoor scene +from .semantic_kitti import SemanticKITTIDataset +from .nuscenes import NuScenesDataset +from .waymo import WaymoDataset + +# object +from .modelnet import ModelNetDataset +from .shapenet_part import ShapeNetPartDataset + +# dataloader +from .dataloader import MultiDatasetDataloader diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/arkitscenes.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/arkitscenes.py new file mode 100644 index 0000000000000000000000000000000000000000..a5481bf553351b09c5f3081b95bcafc77c37f979 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/arkitscenes.py @@ -0,0 +1,114 @@ +""" +ArkitScenes Dataset + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import glob +import numpy as np +import torch +from copy import deepcopy +from torch.utils.data import Dataset + +from pointcept.utils.logger import get_root_logger +from .builder import DATASETS +from .transform import Compose, TRANSFORMS +from .preprocessing.scannet.meta_data.scannet200_constants import VALID_CLASS_IDS_200 + + +@DATASETS.register_module() +class ArkitScenesDataset(Dataset): + def __init__( + self, + split="Training", + data_root="data/ARKitScenesMesh", + transform=None, + test_mode=False, + test_cfg=None, + loop=1, + ): + super(ArkitScenesDataset, self).__init__() + self.data_root = data_root + self.split = split + self.transform = Compose(transform) + self.loop = ( + loop if not test_mode else 1 + ) # force make loop = 1 while in test mode + self.test_mode = test_mode + self.test_cfg = test_cfg if test_mode else None + self.class2id = np.array(VALID_CLASS_IDS_200) + + if test_mode: + self.test_voxelize = TRANSFORMS.build(self.test_cfg.voxelize) + self.test_crop = TRANSFORMS.build(self.test_cfg.crop) + self.post_transform = Compose(self.test_cfg.post_transform) + self.aug_transform = [Compose(aug) for aug in self.test_cfg.aug_transform] + + self.data_list = self.get_data_list() + logger = get_root_logger() + logger.info( + "Totally {} x {} samples in {} set.".format( + len(self.data_list), self.loop, split + ) + ) + + def get_data_list(self): + if isinstance(self.split, str): + data_list = glob.glob(os.path.join(self.data_root, self.split, "*.pth")) + elif isinstance(self.split, list): + data_list = [] + for split in self.split: + data_list += glob.glob(os.path.join(self.data_root, split, "*.pth")) + else: + raise NotImplementedError + return data_list + + def get_data(self, idx): + data = torch.load(self.data_list[idx % len(self.data_list)]) + coord = data["coord"] + color = data["color"] + normal = data["normal"] + segment = np.zeros(coord.shape[0]) + data_dict = dict(coord=coord, normal=normal, color=color, segment=segment) + return data_dict + + def get_data_name(self, idx): + data_idx = self.data_idx[idx % len(self.data_idx)] + return os.path.basename(self.data_list[data_idx]).split(".")[0] + + def prepare_train_data(self, idx): + # load data + data_dict = self.get_data(idx) + data_dict = self.transform(data_dict) + return data_dict + + def prepare_test_data(self, idx): + # load data + data_dict = self.get_data(idx) + segment = data_dict.pop("segment") + data_dict = self.transform(data_dict) + data_dict_list = [] + for aug in self.aug_transform: + data_dict_list.append(aug(deepcopy(data_dict))) + + input_dict_list = [] + for data in data_dict_list: + data_part_list = self.test_voxelize(data) + for data_part in data_part_list: + data_part_list = self.test_crop(data_part) + input_dict_list += data_part_list + + for i in range(len(input_dict_list)): + input_dict_list[i] = self.post_transform(input_dict_list[i]) + return input_dict_list, segment + + def __getitem__(self, idx): + if self.test_mode: + return self.prepare_test_data(idx) + else: + return self.prepare_train_data(idx) + + def __len__(self): + return len(self.data_list) * self.loop diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/builder.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..1fa5f0ee71bf934d5c1bfe5c71446bfecba49f11 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/builder.py @@ -0,0 +1,15 @@ +""" +Dataset Builder + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from pointcept.utils.registry import Registry + +DATASETS = Registry("datasets") + + +def build_dataset(cfg): + """Build datasets.""" + return DATASETS.build(cfg) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/dataloader.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/dataloader.py new file mode 100644 index 0000000000000000000000000000000000000000..a3c8e1da41179896eb3443e91b2c49d94b62762a --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/dataloader.py @@ -0,0 +1,112 @@ +from functools import partial +import weakref +import torch +import torch.utils.data + +import pointcept.utils.comm as comm +from pointcept.datasets.utils import point_collate_fn +from pointcept.datasets import ConcatDataset +from pointcept.utils.env import set_seed + + +class MultiDatasetDummySampler: + def __init__(self): + self.dataloader = None + + def set_epoch(self, epoch): + if comm.get_world_size() > 1: + for dataloader in self.dataloader.dataloaders: + dataloader.sampler.set_epoch(epoch) + return + + +class MultiDatasetDataloader: + """ + Multiple Datasets Dataloader, batch data from a same dataset and mix up ratio determined by loop of each sub dataset. + The overall length is determined by the main dataset (first) and loop of concat dataset. + """ + + def __init__( + self, + concat_dataset: ConcatDataset, + batch_size_per_gpu: int, + num_worker_per_gpu: int, + mix_prob=0, + seed=None, + ): + self.datasets = concat_dataset.datasets + self.ratios = [dataset.loop for dataset in self.datasets] + # reset data loop, original loop serve as ratios + for dataset in self.datasets: + dataset.loop = 1 + # determine union training epoch by main dataset + self.datasets[0].loop = concat_dataset.loop + # build sub-dataloaders + num_workers = num_worker_per_gpu // len(self.datasets) + self.dataloaders = [] + for dataset_id, dataset in enumerate(self.datasets): + if comm.get_world_size() > 1: + sampler = torch.utils.data.distributed.DistributedSampler(dataset) + else: + sampler = None + + init_fn = ( + partial( + self._worker_init_fn, + dataset_id=dataset_id, + num_workers=num_workers, + num_datasets=len(self.datasets), + rank=comm.get_rank(), + seed=seed, + ) + if seed is not None + else None + ) + self.dataloaders.append( + torch.utils.data.DataLoader( + dataset, + batch_size=batch_size_per_gpu, + shuffle=(sampler is None), + num_workers=num_worker_per_gpu, + sampler=sampler, + collate_fn=partial(point_collate_fn, mix_prob=mix_prob), + pin_memory=True, + worker_init_fn=init_fn, + drop_last=True, + persistent_workers=True, + ) + ) + self.sampler = MultiDatasetDummySampler() + self.sampler.dataloader = weakref.proxy(self) + + def __iter__(self): + iterator = [iter(dataloader) for dataloader in self.dataloaders] + while True: + for i in range(len(self.ratios)): + for _ in range(self.ratios[i]): + try: + batch = next(iterator[i]) + except StopIteration: + if i == 0: + return + else: + iterator[i] = iter(self.dataloaders[i]) + batch = next(iterator[i]) + yield batch + + def __len__(self): + main_data_loader_length = len(self.dataloaders[0]) + return ( + main_data_loader_length // self.ratios[0] * sum(self.ratios) + + main_data_loader_length % self.ratios[0] + ) + + @staticmethod + def _worker_init_fn(worker_id, num_workers, dataset_id, num_datasets, rank, seed): + worker_seed = ( + num_workers * num_datasets * rank + + num_workers * dataset_id + + worker_id + + seed + ) + set_seed(worker_seed) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/defaults.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/defaults.py new file mode 100644 index 0000000000000000000000000000000000000000..7c7accd75993e3fd198f0331f534c34b6416f314 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/defaults.py @@ -0,0 +1,214 @@ +""" +Default Datasets + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import glob +import numpy as np +import torch +from copy import deepcopy +from torch.utils.data import Dataset +from collections.abc import Sequence + +from pointcept.utils.logger import get_root_logger +from pointcept.utils.cache import shared_dict + +from .builder import DATASETS, build_dataset +from .transform import Compose, TRANSFORMS + + +@DATASETS.register_module() +class DefaultDataset(Dataset): + VALID_ASSETS = [ + "coord", + "color", + "normal", + "strength", + "segment", + "instance", + "pose", + ] + + def __init__( + self, + split="train", + data_root="data/dataset", + transform=None, + test_mode=False, + test_cfg=None, + cache=False, + ignore_index=-1, + loop=1, + ): + super(DefaultDataset, self).__init__() + self.data_root = data_root + self.split = split + self.transform = Compose(transform) + self.cache = cache + self.ignore_index = ignore_index + self.loop = ( + loop if not test_mode else 1 + ) # force make loop = 1 while in test mode + self.test_mode = test_mode + self.test_cfg = test_cfg if test_mode else None + + if test_mode: + self.test_voxelize = TRANSFORMS.build(self.test_cfg.voxelize) + self.test_crop = ( + TRANSFORMS.build(self.test_cfg.crop) if self.test_cfg.crop else None + ) + self.post_transform = Compose(self.test_cfg.post_transform) + self.aug_transform = [Compose(aug) for aug in self.test_cfg.aug_transform] + + self.data_list = self.get_data_list() + logger = get_root_logger() + logger.info( + "Totally {} x {} samples in {} set.".format( + len(self.data_list), self.loop, split + ) + ) + + def get_data_list(self): + if isinstance(self.split, str): + data_list = glob.glob(os.path.join(self.data_root, self.split, "*")) + elif isinstance(self.split, Sequence): + data_list = [] + for split in self.split: + data_list += glob.glob(os.path.join(self.data_root, split, "*")) + else: + raise NotImplementedError + return data_list + + def get_data(self, idx): + data_path = self.data_list[idx % len(self.data_list)] + name = self.get_data_name(idx) + if self.cache: + cache_name = f"pointcept-{name}" + return shared_dict(cache_name) + + data_dict = {} + assets = os.listdir(data_path) + for asset in assets: + if not asset.endswith(".npy"): + continue + if asset[:-4] not in self.VALID_ASSETS: + continue + data_dict[asset[:-4]] = np.load(os.path.join(data_path, asset)) + data_dict["name"] = name + + if "coord" in data_dict.keys(): + data_dict["coord"] = data_dict["coord"].astype(np.float32) + + if "color" in data_dict.keys(): + data_dict["color"] = data_dict["color"].astype(np.float32) + + if "normal" in data_dict.keys(): + data_dict["normal"] = data_dict["normal"].astype(np.float32) + + if "segment" in data_dict.keys(): + data_dict["segment"] = data_dict["segment"].reshape([-1]).astype(np.int32) + else: + data_dict["segment"] = ( + np.ones(data_dict["coord"].shape[0], dtype=np.int32) * -1 + ) + + if "instance" in data_dict.keys(): + data_dict["instance"] = data_dict["instance"].reshape([-1]).astype(np.int32) + else: + data_dict["instance"] = ( + np.ones(data_dict["coord"].shape[0], dtype=np.int32) * -1 + ) + return data_dict + + def get_data_name(self, idx): + return os.path.basename(self.data_list[idx % len(self.data_list)]) + + def prepare_train_data(self, idx): + # load data + data_dict = self.get_data(idx) + data_dict = self.transform(data_dict) + return data_dict + + def prepare_test_data(self, idx): + # load data + data_dict = self.get_data(idx) + data_dict = self.transform(data_dict) + result_dict = dict(segment=data_dict.pop("segment"), name=data_dict.pop("name")) + if "origin_segment" in data_dict: + assert "inverse" in data_dict + result_dict["origin_segment"] = data_dict.pop("origin_segment") + result_dict["inverse"] = data_dict.pop("inverse") + + data_dict_list = [] + for aug in self.aug_transform: + data_dict_list.append(aug(deepcopy(data_dict))) + + fragment_list = [] + for data in data_dict_list: + if self.test_voxelize is not None: + data_part_list = self.test_voxelize(data) + else: + data["index"] = np.arange(data["coord"].shape[0]) + data_part_list = [data] + for data_part in data_part_list: + if self.test_crop is not None: + data_part = self.test_crop(data_part) + else: + data_part = [data_part] + fragment_list += data_part + + for i in range(len(fragment_list)): + fragment_list[i] = self.post_transform(fragment_list[i]) + result_dict["fragment_list"] = fragment_list + return result_dict + + def __getitem__(self, idx): + if self.test_mode: + return self.prepare_test_data(idx) + else: + return self.prepare_train_data(idx) + + def __len__(self): + return len(self.data_list) * self.loop + + +@DATASETS.register_module() +class ConcatDataset(Dataset): + def __init__(self, datasets, loop=1): + super(ConcatDataset, self).__init__() + self.datasets = [build_dataset(dataset) for dataset in datasets] + self.loop = loop + self.data_list = self.get_data_list() + logger = get_root_logger() + logger.info( + "Totally {} x {} samples in the concat set.".format( + len(self.data_list), self.loop + ) + ) + + def get_data_list(self): + data_list = [] + for i in range(len(self.datasets)): + data_list.extend( + zip( + np.ones(len(self.datasets[i])) * i, np.arange(len(self.datasets[i])) + ) + ) + return data_list + + def get_data(self, idx): + dataset_idx, data_idx = self.data_list[idx % len(self.data_list)] + return self.datasets[dataset_idx][data_idx] + + def get_data_name(self, idx): + dataset_idx, data_idx = self.data_list[idx % len(self.data_list)] + return self.datasets[dataset_idx].get_data_name(data_idx) + + def __getitem__(self, idx): + return self.get_data(idx) + + def __len__(self): + return len(self.data_list) * self.loop diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/modelnet.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/modelnet.py new file mode 100644 index 0000000000000000000000000000000000000000..213f3ed2dfe4d788380824b87355e6fcae1ee531 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/modelnet.py @@ -0,0 +1,150 @@ +""" +ModelNet40 Dataset + +get sampled point clouds of ModelNet40 (XYZ and normal from mesh, 10k points per shape) +at "https://shapenet.cs.stanford.edu/media/modelnet40_normal_resampled.zip" + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import numpy as np +import pointops +import torch +from torch.utils.data import Dataset +from copy import deepcopy + + +from pointcept.utils.logger import get_root_logger +from .builder import DATASETS +from .transform import Compose + + +@DATASETS.register_module() +class ModelNetDataset(Dataset): + def __init__( + self, + split="train", + data_root="data/modelnet40", + class_names=None, + transform=None, + num_points=8192, + uniform_sampling=True, + save_record=True, + test_mode=False, + test_cfg=None, + loop=1, + ): + super().__init__() + self.data_root = data_root + self.class_names = dict(zip(class_names, range(len(class_names)))) + self.split = split + self.num_point = num_points + self.uniform_sampling = uniform_sampling + self.transform = Compose(transform) + self.loop = ( + loop if not test_mode else 1 + ) # force make loop = 1 while in test mode + self.test_mode = test_mode + self.test_cfg = test_cfg if test_mode else None + if test_mode: + self.post_transform = Compose(self.test_cfg.post_transform) + self.aug_transform = [Compose(aug) for aug in self.test_cfg.aug_transform] + + self.data_list = self.get_data_list() + logger = get_root_logger() + logger.info( + "Totally {} x {} samples in {} set.".format( + len(self.data_list), self.loop, split + ) + ) + + # check, prepare record + record_name = f"modelnet40_{self.split}" + if num_points is not None: + record_name += f"_{num_points}points" + if uniform_sampling: + record_name += "_uniform" + record_path = os.path.join(self.data_root, f"{record_name}.pth") + if os.path.isfile(record_path): + logger.info(f"Loading record: {record_name} ...") + self.data = torch.load(record_path) + else: + logger.info(f"Preparing record: {record_name} ...") + self.data = {} + for idx in range(len(self.data_list)): + data_name = self.data_list[idx] + logger.info(f"Parsing data [{idx}/{len(self.data_list)}]: {data_name}") + self.data[data_name] = self.get_data(idx) + if save_record: + torch.save(self.data, record_path) + + def get_data(self, idx): + data_idx = idx % len(self.data_list) + data_name = self.data_list[data_idx] + if data_name in self.data.keys(): + return self.data[data_name] + else: + data_shape = "_".join(data_name.split("_")[0:-1]) + data_path = os.path.join( + self.data_root, data_shape, self.data_list[data_idx] + ".txt" + ) + data = np.loadtxt(data_path, delimiter=",").astype(np.float32) + if self.num_point is not None: + if self.uniform_sampling: + with torch.no_grad(): + mask = pointops.farthest_point_sampling( + torch.tensor(data).float().cuda(), + torch.tensor([len(data)]).long().cuda(), + torch.tensor([self.num_point]).long().cuda(), + ) + data = data[mask.cpu()] + else: + data = data[: self.num_point] + coord, normal = data[:, 0:3], data[:, 3:6] + category = np.array([self.class_names[data_shape]]) + return dict(coord=coord, normal=normal, category=category) + + def get_data_list(self): + assert isinstance(self.split, str) + split_path = os.path.join( + self.data_root, "modelnet40_{}.txt".format(self.split) + ) + data_list = np.loadtxt(split_path, dtype="str") + return data_list + + def get_data_name(self, idx): + data_idx = idx % len(self.data_list) + return self.data_list[data_idx] + + def __getitem__(self, idx): + if self.test_mode: + return self.prepare_test_data(idx) + else: + return self.prepare_train_data(idx) + + def __len__(self): + return len(self.data_list) * self.loop + + def prepare_train_data(self, idx): + data_dict = self.get_data(idx) + data_dict = self.transform(data_dict) + return data_dict + + def prepare_test_data(self, idx): + assert idx < len(self.data_list) + data_dict = self.get_data(idx) + category = data_dict.pop("category") + data_dict = self.transform(data_dict) + data_dict_list = [] + for aug in self.aug_transform: + data_dict_list.append(aug(deepcopy(data_dict))) + for i in range(len(data_dict_list)): + data_dict_list[i] = self.post_transform(data_dict_list[i]) + data_dict = dict( + voting_list=data_dict_list, + category=category, + name=self.get_data_name(idx), + ) + return data_dict diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/nuscenes.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/nuscenes.py new file mode 100644 index 0000000000000000000000000000000000000000..d3ab8f092fa8854baa14bad71ab5588bb402dc0c --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/nuscenes.py @@ -0,0 +1,125 @@ +""" +nuScenes Dataset + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com), Zheng Zhang +Please cite our work if the code is helpful to you. +""" + +import os +import numpy as np +from collections.abc import Sequence +import pickle + +from .builder import DATASETS +from .defaults import DefaultDataset + + +@DATASETS.register_module() +class NuScenesDataset(DefaultDataset): + def __init__(self, sweeps=10, ignore_index=-1, **kwargs): + self.sweeps = sweeps + self.ignore_index = ignore_index + self.learning_map = self.get_learning_map(ignore_index) + super().__init__(ignore_index=ignore_index, **kwargs) + + def get_info_path(self, split): + assert split in ["train", "val", "test"] + if split == "train": + return os.path.join( + self.data_root, "info", f"nuscenes_infos_{self.sweeps}sweeps_train.pkl" + ) + elif split == "val": + return os.path.join( + self.data_root, "info", f"nuscenes_infos_{self.sweeps}sweeps_val.pkl" + ) + elif split == "test": + return os.path.join( + self.data_root, "info", f"nuscenes_infos_{self.sweeps}sweeps_test.pkl" + ) + else: + raise NotImplementedError + + def get_data_list(self): + if isinstance(self.split, str): + info_paths = [self.get_info_path(self.split)] + elif isinstance(self.split, Sequence): + info_paths = [self.get_info_path(s) for s in self.split] + else: + raise NotImplementedError + data_list = [] + for info_path in info_paths: + with open(info_path, "rb") as f: + info = pickle.load(f) + data_list.extend(info) + return data_list + + def get_data(self, idx): + data = self.data_list[idx % len(self.data_list)] + lidar_path = os.path.join(self.data_root, "raw", data["lidar_path"]) + points = np.fromfile(str(lidar_path), dtype=np.float32, count=-1).reshape( + [-1, 5] + ) + coord = points[:, :3] + strength = points[:, 3].reshape([-1, 1]) / 255 # scale strength to [0, 1] + + if "gt_segment_path" in data.keys(): + gt_segment_path = os.path.join( + self.data_root, "raw", data["gt_segment_path"] + ) + segment = np.fromfile( + str(gt_segment_path), dtype=np.uint8, count=-1 + ).reshape([-1]) + segment = np.vectorize(self.learning_map.__getitem__)(segment).astype( + np.int64 + ) + else: + segment = np.ones((points.shape[0],), dtype=np.int64) * self.ignore_index + data_dict = dict( + coord=coord, + strength=strength, + segment=segment, + name=self.get_data_name(idx), + ) + return data_dict + + def get_data_name(self, idx): + # return data name for lidar seg, optimize the code when need to support detection + return self.data_list[idx % len(self.data_list)]["lidar_token"] + + @staticmethod + def get_learning_map(ignore_index): + learning_map = { + 0: ignore_index, + 1: ignore_index, + 2: 6, + 3: 6, + 4: 6, + 5: ignore_index, + 6: 6, + 7: ignore_index, + 8: ignore_index, + 9: 0, + 10: ignore_index, + 11: ignore_index, + 12: 7, + 13: ignore_index, + 14: 1, + 15: 2, + 16: 2, + 17: 3, + 18: 4, + 19: ignore_index, + 20: ignore_index, + 21: 5, + 22: 8, + 23: 9, + 24: 10, + 25: 11, + 26: 12, + 27: 13, + 28: 14, + 29: ignore_index, + 30: 15, + 31: ignore_index, + } + return learning_map diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/arkitscenes/preprocess_arkitscenes_mesh.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/arkitscenes/preprocess_arkitscenes_mesh.py new file mode 100644 index 0000000000000000000000000000000000000000..9bc9b3e47a35f5baa00bcf0f526d5d986b28494e --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/arkitscenes/preprocess_arkitscenes_mesh.py @@ -0,0 +1,87 @@ +""" +Preprocessing ArkitScenes +""" + +import os +import argparse +import glob +import plyfile +import numpy as np +import pandas as pd +import multiprocessing as mp +from concurrent.futures import ProcessPoolExecutor +from itertools import repeat + +import torch + + +def read_plymesh(filepath): + """Read ply file and return it as numpy array. Returns None if emtpy.""" + with open(filepath, "rb") as f: + plydata = plyfile.PlyData.read(f) + if plydata.elements: + vertices = pd.DataFrame(plydata["vertex"].data).values + faces = np.stack(plydata["face"].data["vertex_indices"], axis=0) + return vertices, faces + + +def face_normal(vertex, face): + v01 = vertex[face[:, 1]] - vertex[face[:, 0]] + v02 = vertex[face[:, 2]] - vertex[face[:, 0]] + vec = np.cross(v01, v02) + length = np.sqrt(np.sum(vec**2, axis=1, keepdims=True)) + 1.0e-8 + nf = vec / length + area = length * 0.5 + return nf, area + + +def vertex_normal(vertex, face): + nf, area = face_normal(vertex, face) + nf = nf * area + + nv = np.zeros_like(vertex) + for i in range(face.shape[0]): + nv[face[i]] += nf[i] + + length = np.sqrt(np.sum(nv**2, axis=1, keepdims=True)) + 1.0e-8 + nv = nv / length + return nv + + +def parse_scene(scene_path, output_dir): + print(f"Parsing scene {scene_path}") + split = os.path.basename(os.path.dirname(os.path.dirname(scene_path))) + scene_id = os.path.basename(os.path.dirname(scene_path)) + vertices, faces = read_plymesh(scene_path) + coords = vertices[:, :3] + colors = vertices[:, 3:6] + data_dict = dict(coord=coords, color=colors, scene_id=scene_id) + data_dict["normal"] = vertex_normal(coords, faces) + torch.save(data_dict, os.path.join(output_dir, split, f"{scene_id}.pth")) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset_root", + required=True, + help="Path to the ScanNet dataset containing scene folders", + ) + parser.add_argument( + "--output_root", + required=True, + help="Output path where train/val folders will be located", + ) + opt = parser.parse_args() + # Create output directories + train_output_dir = os.path.join(opt.output_root, "Training") + os.makedirs(train_output_dir, exist_ok=True) + val_output_dir = os.path.join(opt.output_root, "Validation") + os.makedirs(val_output_dir, exist_ok=True) + # Load scene paths + scene_paths = sorted(glob.glob(opt.dataset_root + "/3dod/*/*/*_mesh.ply")) + # Preprocess data. + pool = ProcessPoolExecutor(max_workers=mp.cpu_count()) + # pool = ProcessPoolExecutor(max_workers=1) + print("Processing scenes...") + _ = list(pool.map(parse_scene, scene_paths, repeat(opt.output_root))) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/meta_data/category_mapping.tsv b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/meta_data/category_mapping.tsv new file mode 100644 index 0000000000000000000000000000000000000000..177c75ae021287bfc2462ae487b042886313cab1 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/meta_data/category_mapping.tsv @@ -0,0 +1,1660 @@ +index raw_category category count nyuId nyu40id eigen13id nyuClass nyu40class eigen13class ModelNet40 ModelNet10 ShapeNetCore55 synsetoffset wnsynsetid wnsynsetkey mpcat40index mpcat40 +1 wall wall 7667 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +2 door door 2544 28 8 12 door door Wall door n03221720 door.n.01 4 door +3 ceiling ceiling 2363 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +4 floor floor 2252 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +5 picture picture 2125 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +6 window window 2013 59 9 13 window window Window n04587648 window.n.01 9 window +7 chair chair 1947 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +8 doorframe door frame 1935 28 8 12 door door Wall door n03221720 door.n.01 4 door +9 remove remove 1622 0 0 0 void void void 0 void +10 pillow pillow 1335 119 18 7 pillow pillow Objects pillow 03938244 n03938244 pillow.n.01 8 cushion +11 object object 1145 40 7 otherprop Objects n00002684 object.n.01 39 objects +12 light light 967 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +13 cabinet cabinet 785 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +14 curtain curtain 692 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +15 table table 691 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +16 plant plant 632 82 40 7 plant otherprop Objects plant n00017222 plant.n.02 14 plant +17 decoration decoration 532 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +18 window frame window frame 526 59 9 13 window window Window n04589593 window_frame.n.01 9 window +19 lamp lamp 540 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +20 mirror mirror 488 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +21 towel towel 437 135 27 7 towel towel Objects n04459362 towel.n.01 20 towel +22 sink sink 417 24 34 7 sink sink Objects sink n04223580 sink.n.01 15 sink +23 shelf shelf 417 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +24 couch couch 290 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +25 dining chair dining chair 277 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +26 bed bed 270 157 4 1 bed bed Bed bed bed bed 02818832 n02818832 bed.n.01 11 bed +27 nightstand nightstand 275 158 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 13 chest_of_drawers +28 toilet toilet 268 124 33 7 toilet toilet Objects toilet toilet n04446276 toilet.n.01 18 toilet +29 sofa chair sofa chair 262 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +30 column pillar 243 94 38 7 column otherstructure Objects n03073977 column.n.07 24 column +31 handrail handrail 241 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +32 stair stair 241 215 38 7 stairs otherstructure Objects stairs n04314914 step.n.04 16 stairs +33 stool stool 232 150 40 7 stool otherprop Objects stool n04326896 stool.n.01 19 stool +34 armchair armchair 224 5 5 4 chair chair Chair chair chair chair 03001627 n02738535 armchair.n.01 3 chair +35 kitchen cabinet kitchen cabinet 212 3 3 6 cabinet cabinet Furniture n02933112 cabinet.n.01 7 cabinet +36 vase vase 205 78 40 7 vase otherprop Objects vase jar 03593526 n04522168 vase.n.01 39 objects +37 cushion cushion 204 119 18 7 pillow pillow Objects n03151500 cushion.n.03 8 cushion +38 tv tv 204 172 25 11 television television TV tv or monitor 03211117 n03211117 display.n.06 22 tv_monitor +39 door frame door frame 190 28 8 12 door door Wall door n03221720 door.n.01 4 door +40 unknown unknown 186 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +41 pot pot 185 16 40 7 pot otherprop Objects n03991062 pot.n.04 39 objects +42 desk desk 187 36 14 10 desk desk Table desk desk table 04379243 n03179701 desk.n.01 5 table +43 painting picture 194 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +44 roof roof 173 4 22 3 ceiling ceiling Ceiling n04105068 roof.n.01 17 ceiling +45 box box 172 26 29 7 box box Objects n02883344 box.n.01 39 objects +46 shower wall shower wall 168 21 1 12 wall wall Wall n04208936 shower.n.01 23 shower +47 coffee table coffee table 159 356 39 6 coffee table otherfurniture Furniture table table table 04379243 n03063968 coffee_table.n.01 5 table +48 countertop countertop 159 7 12 6 counter counter Furniture n03118245 countertop.n.01 26 counter +49 bench bench 154 204 39 6 bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +50 wall frame picture 142 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +51 trash can trashcan 131 12 39 6 garbage bin otherfurniture Furniture trash_bin 02747177 n02747177 ashcan.n.01 39 objects +52 fireplace fireplace 128 372 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 27 fireplace +53 clothes clothes 125 141 21 7 clothes clothes Objects n02728440 apparel.n.01 38 clothes +54 pillar pillar 117 94 38 7 column otherstructure Objects n03073977 column.n.07 24 column +55 bathtub bathtub 121 136 36 7 bathtub bathtub Objects bathtub bathtub tub 02808440 n02808440 bathtub.n.01 25 bathtub +56 ceiling duct ceiling duct 110 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +57 bath cabinet bath cabinet 111 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +58 book book 104 1 23 2 book books Books n02870526 book.n.11 39 objects +59 beam beam 101 38 7 otherstructure Objects n02815950 beam.n.02 29 beam +60 vent vent 101 25 38 7 air vent otherstructure Objects n04526241 vent.n.01 40 misc +61 shower floor shower floor 97 11 2 5 floor floor Floor n04208936 shower.n.01 23 shower +62 faucet faucet 97 9 40 7 faucet otherprop Objects faucet 03325088 n03325088 faucet.n.01 39 objects +63 photo photo 100 508 40 7 photo otherprop Objects n03925226 photograph.n.01 6 picture +64 delete remove 90 0 0 0 void void void 0 void +65 toilet paper toilet paper 90 139 40 7 toilet paper otherprop Objects n15075141 toilet_tissue.n.01 39 objects +66 counter counter 91 7 12 6 counter counter Furniture table table table 04379243 n03116530 counter.n.01 26 counter +67 fan fan 85 74 40 7 fan otherprop Objects n03320046 fan.n.01 39 objects +68 step step 86 38 7 otherstructure Objects n04314914 step.n.04 16 stairs +69 table lamp table lamp 96 144 35 7 lamp lamp Objects lamp lamp 03636649 n04380533 table_lamp.n.01 28 lighting +70 wall / other room wall /otherroom 84 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +71 washbasin washbasin 91 24 34 7 sink sink Objects sink n04553920 washbasin.n.01 15 sink +72 rail railing 83 497 38 7 railing otherstructure Objects n04047401 railing.n.01 30 railing +73 side table table 85 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +74 decor decoration 79 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +75 shelves shelving 79 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +76 statue statue 78 294 40 7 sculpture otherprop Objects n04306847 statue.n.01 39 objects +77 dresser dresser 79 169 17 6 dresser dresser Furniture dresser dresser n03015254 chest_of_drawers.n.01 13 chest_of_drawers +78 stairs stair 76 215 38 7 stairs otherstructure Objects stairs n04314914 step.n.04 16 stairs +79 rug rug 77 130 40 7 rug floor mat Objects n04118021 rug.n.01 2 floor +80 ottoman ottoman 79 359 39 6 ottoman otherfurniture Furniture stool n03380724 footstool.n.01 19 stool +81 round table table 74 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +82 bottle bottle 101 2 40 7 bottle otherprop Objects bottle bottle 02876657 n02876657 bottle.n.01 39 objects +83 kitchen counter kitchen counter 73 7 12 6 counter counter Furniture table table table 04379243 n03116530 counter.n.01 26 counter +84 windowframe window frame 66 59 9 13 window window Window n04589593 window_frame.n.01 9 window +85 office chair office chair 70 5 5 4 chair chair Chair chair chair chair 03001627 n04373704 swivel_chair.n.01 3 chair +86 frame frame 66 38 7 otherstructure Objects 40 misc +87 refrigerator refrigerator 65 17 24 6 refridgerator refridgerator Furniture n04070727 refrigerator.n.01 37 appliances +88 bookshelf bookshelf 63 88 10 6 bookshelf bookshelf Furniture bookshelf bookshelf 02871439 n02871439 bookshelf.n.01 31 shelving +89 end table end table 67 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +90 wardrobe wardrobe 64 772 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 36 furniture +91 toiletry toiletry 63 40 7 otherprop Objects n04447443 toiletry.n.01 39 objects +92 windowsill window frame 62 59 9 13 window window Window n04589593 window_frame.n.01 9 window +93 pipe pipe 62 41 40 7 pipe otherprop Objects n03944672 pipe.n.02 40 misc +94 monitor monitor 60 49 40 7 monitor otherprop Objects monitor monitor tv or monitor 03211117 n03782190 monitor.n.04 22 tv_monitor +95 stand stand 62 50 39 6 stand otherfurniture Furniture table table table 04379243 n04301000 stand.n.04 5 table +96 drawers drawer 60 174 39 6 drawer otherfurniture Furniture n03233905 drawer.n.01 13 chest_of_drawers +97 dining table dining table 61 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +98 container container 59 140 40 7 container otherprop Objects n03094503 container.n.01 39 objects +99 light switch light switch 59 301 38 7 light switch otherstructure Objects n04372370 switch.n.01 39 objects +100 skylight skylight 57 59 9 13 window window Window n04232800 skylight.n.01 9 window +101 purse purse 57 181 40 7 purse otherprop Objects n02774152 bag.n.04 39 objects +102 wall /otherroom wall /otherroom 55 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +103 sofa couch 56 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +104 books book 53 1 23 2 book books Books n02870526 book.n.11 39 objects +105 doorway doorway 54 609 38 7 door way otherstructure Objects door n03224032 doorway.n.01 4 door +106 railing railing 52 497 38 7 railing otherstructure Objects n04047401 railing.n.01 30 railing +107 wainscotting paneling 52 21 1 12 wall wall Wall n03882611 paneling.n.01 1 wall +108 basket basket 52 39 40 7 basket otherprop Objects basket 02801938 n02801938 basket.n.01 39 objects +109 closet closet 50 40 7 otherprop Objects n03148324 cupboard.n.01 7 cabinet +110 arch arch 48 40 7 otherprop Objects n02733524 arch.n.04 40 misc +111 chandelier chandelier 48 342 38 7 chandelier otherstructure Objects n03005285 chandelier.n.01 28 lighting +112 oven oven 47 238 38 7 oven otherstructure Objects n03862676 oven.n.01 37 appliances +113 clock clock 47 56 40 7 clock otherprop Objects clock 03046257 n03046257 clock.n.01 39 objects +114 footstool footstool 48 359 39 6 ottoman otherfurniture Furniture stool n03380724 footstool.n.01 19 stool +115 stove stove 44 242 38 7 stove otherstructure Objects stove 04330267 n04330267 stove.n.02 37 appliances +116 trashcan trashcan 44 12 39 6 garbage bin otherfurniture Furniture trash_bin 02747177 n02747177 ashcan.n.01 39 objects +117 drawer drawer 45 174 39 6 drawer otherfurniture Furniture n03233905 drawer.n.01 13 chest_of_drawers +118 bathroom countertop object object 43 40 7 otherprop Objects n00002684 object.n.01 39 objects +119 washing machine washing machine 43 278 39 6 washing machine otherfurniture Furniture washing_machine 04554684 n04554684 washer.n.03 37 appliances +120 shower curtain shower curtain 43 123 28 7 shower curtain shower curtain Objects curtain n04209239 shower_curtain.n.01 12 curtain +121 rack rack 41 50 39 6 stand otherfurniture Furniture n04038440 rack.n.05 31 shelving +122 art picture 40 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +123 firealarm fire alarm 37 338 40 7 fire alarm otherprop Objects n03343737 fire_alarm.n.02 39 objects +124 bin bin 37 307 40 7 bin otherprop Objects n02839910 bin.n.01 39 objects +125 chest chest 36 344 39 6 chest otherfurniture Furniture dresser dresser 40 misc +126 microwave microwave 36 13 40 7 microwave otherprop Objects microwave 03761084 n03761084 microwave.n.02 37 appliances +127 blinds blinds 36 80 13 13 blinds blinds Window n04589190 window_blind.n.01 32 blinds +128 bowl bowl 35 22 40 7 bowl otherprop Objects bowl bowl 02880940 n02880940 bowl.n.03 39 objects +129 ceiling pipe ceiling pipe 34 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +130 tree tree 34 82 40 7 plant otherprop Objects plant n13104059 tree.n.01 14 plant +131 vanity vanity 34 169 17 6 dresser dresser Furniture dresser dresser table 04379243 n03238586 dressing_table.n.01 5 table +132 ceiling fan ceiling fan 37 74 40 7 fan otherprop Objects n03320046 fan.n.01 39 objects +133 tissue box tissue box 34 138 40 7 tissue box otherprop Objects n02883344 box.n.01 39 objects +134 desk chair desk chair 34 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +135 plate plate 32 233 40 7 plate otherprop Objects n03959485 plate.n.04 39 objects +136 tv stand tv stand 32 291 39 6 tv stand otherfurniture Furniture tv_stand n03290653 entertainment_center.n.01 36 furniture +137 shoes shoe 32 149 40 7 shoe otherprop Objects n04199027 shoe.n.01 39 objects +138 heater heater 32 111 39 6 heater otherfurniture Furniture n03508101 heater.n.01 39 objects +139 bedframe bedframe 33 157 4 1 bed bed Bed n02822579 bedstead.n.01 11 bed +140 headboard headboard 33 161 39 6 headboard otherfurniture Furniture n03502200 headboard.n.01 11 bed +141 post post 32 94 38 7 column otherstructure Objects n03988170 post.n.04 24 column +142 swivel chair swivel chair 31 5 5 4 chair chair Chair chair chair chair 03001627 n04373704 swivel_chair.n.01 3 chair +143 pedestal pedestal 31 50 39 6 stand otherfurniture Furniture 40 misc +144 fence fence 31 38 7 otherstructure Objects n03327234 fence.n.01 40 misc +145 ceiling pipes ceiling pipe 30 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +146 pew pew 28 204 39 6 bench otherfurniture Furniture bench bench 02828884 n03920867 pew.n.01 34 seating +147 decorative object decoration 28 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +148 bucket bucket 28 427 40 7 bucket otherprop Objects n02909870 bucket.n.01 39 objects +149 mask decorative mask 27 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +150 candle candle 27 137 40 7 candle otherprop Objects lamp n02948072 candle.n.01 39 objects +151 flowerpot flowerpot 28 146 40 7 flower pot otherprop Objects flower_pot flower pot 03991062 n03991062 pot.n.04 39 objects +152 speaker speaker 27 54 40 7 speaker otherprop Objects speaker 03691459 n03691459 loudspeaker.n.01 39 objects +153 seat seat 26 524 39 6 furniture otherfurniture Furniture n04161981 seat.n.03 34 seating +154 sign sign 25 208 40 7 sign otherprop Objects n04217882 signboard.n.01 40 misc +155 air conditioner air conditioner 26 79 38 7 air conditioner otherstructure Objects n02686379 air_conditioner.n.01 39 objects +156 shower curtain rod shower curtain rod 25 40 7 otherprop Objects n04100174 rod.n.01 12 curtain +157 unknown / other room unknown /otherroom 24 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +158 flowers plant 25 82 40 7 plant otherprop Objects plant n00017222 plant.n.02 14 plant +159 clutter clutter 24 40 7 otherprop Objects 40 misc +160 pillows pillow 24 119 18 7 pillow pillow Objects pillow 03938244 n03938244 pillow.n.01 8 cushion +161 plants plant 24 82 40 7 plant otherprop Objects plant n00017222 plant.n.02 14 plant +162 wall \other room wall /otherroom 24 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +163 fire extinguisher fire extinguisher 24 10 40 7 fire extinguisher otherprop Objects n03345837 fire_extinguisher.n.01 39 objects +164 towels towel 24 135 27 7 towel towel Objects n04459362 towel.n.01 20 towel +165 curtains curtain 23 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +166 curtain rod curtain rod 23 582 38 7 curtain rod otherstructure Objects n04100174 rod.n.01 12 curtain +167 kitchen countertop object object 23 40 7 otherprop Objects n00002684 object.n.01 39 objects +168 mat mat 23 143 20 5 floor mat floor mat Floor n03727837 mat.n.01 2 floor +169 flower plant 23 82 40 7 plant otherprop Objects plant n00017222 plant.n.02 14 plant +170 sculpture sculpture 23 294 40 7 sculpture otherprop Objects n04157320 sculpture.n.01 39 objects +171 shelving shelving 22 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +172 wall/other room wall /otherroom 22 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +173 knickknack object 21 40 7 otherprop Objects n00002684 object.n.01 39 objects +174 printer printer 21 66 40 7 printer otherprop Objects printer 04004475 n04004475 printer.n.03 39 objects +175 wall behind wall /otherroom 21 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +176 telephone telephone 21 32 40 7 telephone otherprop Objects telephone 04401088 n04401088 telephone.n.01 39 objects +177 bedside table nightstand 21 158 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 13 chest_of_drawers +178 moulding molding 21 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +179 handbag handbag 21 40 7 otherprop Objects n02774152 bag.n.04 39 objects +180 wall /other room wall /otherroom 21 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +181 blanket blanket 21 312 40 7 blanket otherprop Objects n02849154 blanket.n.01 39 objects +182 shower shower 21 38 7 otherstructure Objects n04208936 shower.n.01 23 shower +183 steps step 20 38 7 otherstructure Objects n04314914 step.n.04 16 stairs +184 switch switch 21 40 7 otherprop Objects n04372370 switch.n.01 39 objects +185 toilet paper dispenser toilet paper dispenser 20 40 7 otherprop Objects 40 misc +186 objects object 21 40 7 otherprop Objects n00002684 object.n.01 39 objects +187 handle handle 20 758 40 7 handle otherprop Objects n03485997 handle.n.01 39 objects +188 frame /outside frame /outside 20 38 7 otherstructure Objects 40 misc +189 screen screen 20 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +190 shower head showerhead 19 650 40 7 shower head otherprop Objects n04209383 showerhead.n.01 23 shower +191 baracade barricade 19 40 7 otherprop Objects n04096848 roadblock.n.02 40 misc +192 picture frame picture frame 25 64 11 8 picture picture Picture n03931765 picture_frame.n.01 6 picture +193 soap soap 19 133 40 7 soap otherprop Objects n04253437 soap.n.01 39 objects +194 staircase railing banister 18 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +195 keyboard keyboard 18 47 40 7 keyboard otherprop Objects keyboard computer keyboard 03085013 n03085013 computer_keyboard.n.01 39 objects +196 thermostat thermostat 18 110 40 7 thermostat otherprop Objects n04422875 thermostat.n.01 39 objects +197 radiator radiator 18 236 39 6 radiator otherfurniture Furniture n04041069 radiator.n.02 39 objects +198 kitchen island kitchen island 18 456 38 7 kitchen island otherstructure Objects n03620600 kitchen_island.n.01 26 counter +199 paper towel paper towel 18 113 40 7 paper towel otherprop Objects n03887697 paper_towel.n.01 20 towel +200 wall decoration picture 17 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +201 phone telephone 17 32 40 7 telephone otherprop Objects telephone 04401088 n04401088 telephone.n.01 39 objects +202 mirror frame mirror 17 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +203 clothes dryer clothes dryer 18 39 6 otherfurniture Furniture n03251766 dryer.n.01 37 appliances +204 panel panel 17 559 40 7 sheet otherprop Objects n03882058 panel.n.01 35 board_panel +205 glass glass 16 612 38 7 glass otherstructure Objects n03438257 glass.n.02 39 objects +206 soap dispenser soap dispenser 16 40 7 otherprop Objects n04254120 soap_dispenser.n.01 39 objects +207 dishwasher dishwasher 16 8 38 7 dishwasher otherstructure Objects dishwasher 03207941 n03207941 dishwasher.n.01 37 appliances +208 cup cup 16 35 40 7 cup otherprop Objects cup cup or mug 03797390 n03797390 mug.n.04 39 objects +209 bathroom cabinet bathroom cabinet 17 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +210 ladder ladder 17 48 39 6 ladder otherfurniture Furniture stairs n03632277 ladder.n.01 16 stairs +211 garage door garage door 16 850 38 7 garage door otherstructure Objects door 4 door +212 hat hat 15 193 40 7 hat otherprop Objects n03497657 hat.n.01 38 clothes +213 chest of drawers chest of drawers 15 524 39 6 furniture otherfurniture Furniture dresser dresser n03015254 chest_of_drawers.n.01 13 chest_of_drawers +214 exit sign exit sign 15 86 40 7 exit sign otherprop Objects 40 misc +215 sidetable side table 15 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +216 office table office table 15 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +217 piano piano 15 298 39 6 piano otherfurniture Furniture piano piano 03928116 n03928116 piano.n.01 39 objects +218 painter picture 17 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +219 board board 15 408 38 7 board otherstructure Objects 35 board_panel +220 windows window 14 59 9 13 window window Window n04587648 window.n.01 9 window +221 archway archway 14 21 1 12 wall wall Wall n02734217 arch.n.03 4 door +222 rope rope 14 560 40 7 rope otherprop Objects n04108268 rope.n.01 39 objects +223 ball ball 15 60 40 7 ball otherprop Objects 40 misc +224 gym equipment gym equipment 14 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +225 clothes hangers clothes hanger 13 211 40 7 hanger otherprop Objects n03057920 coat_hanger.n.01 39 objects +226 bathroom object object 13 40 7 otherprop Objects n00002684 object.n.01 39 objects +227 easy chair easy chair 13 5 5 4 chair chair Chair chair chair chair 03001627 n03262932 easy_chair.n.01 3 chair +228 lounge chair lounge chair 15 5 5 4 chair chair Chair chair chair chair 03001627 n03262932 easy_chair.n.01 3 chair +229 furniture furniture 13 524 39 6 furniture otherfurniture Furniture n03405725 furniture.n.01 36 furniture +230 cabinets cabinet 16 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +231 carpet carpet 14 130 40 7 rug floor mat Objects n04118021 rug.n.01 2 floor +232 food food 13 40 7 otherprop Objects n00021265 food.n.01 40 misc +233 plant pot pot 13 16 40 7 pot otherprop Objects n03991062 pot.n.04 39 objects +234 duct duct 13 40 7 otherprop Objects n03253398 duct.n.03 40 misc +235 ridge ridge 13 40 7 otherprop Objects 40 misc +236 candlestick candlestick 12 148 40 7 candlestick otherprop Objects n02948557 candlestick.n.01 39 objects +237 computer desk computer desk 12 36 14 10 desk desk Table desk desk table 04379243 n03179701 desk.n.01 5 table +238 shower door shower door 12 28 8 12 door door Wall door n04208936 shower.n.01 23 shower +239 trash trashcan 12 12 39 6 garbage bin otherfurniture Furniture trash_bin 02747177 n02747177 ashcan.n.01 39 objects +240 crown molding molding 12 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +241 wall sconce sconce 12 62 38 7 light otherstructure Objects n04148703 sconce.n.03 28 lighting +242 door handle door handle 12 758 40 7 handle otherprop Objects 40 misc +243 scale scale 12 639 40 7 scale otherprop Objects n04141975 scale.n.07 39 objects +244 trash bin trashcan 12 12 39 6 garbage bin otherfurniture Furniture trash_bin 02747177 n02747177 ashcan.n.01 39 objects +245 baseboard baseboard 13 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +246 window /otherroom window /otherroom 12 59 9 13 window window Window n04587648 window.n.01 9 window +247 bag bag 11 55 37 7 bag bag Objects suitcase 02773838 n02773838 bag.n.06 39 objects +248 laptop laptop 11 37 40 7 laptop otherprop Objects laptop laptop 03642806 n03642806 laptop.n.01 39 objects +249 treadmill treadmill 12 458 39 6 treadmill otherfurniture Furniture n04477387 treadmill.n.01 33 gym_equipment +250 staircase staircase 11 215 38 7 stairs otherstructure Objects n04298308 stairway.n.01 16 stairs +251 guitar guitar 11 300 40 7 guitar otherprop Objects guitar guitar 03467517 n03467517 guitar.n.01 39 objects +252 light fixture light fixture 11 62 38 7 light otherstructure Objects n03665366 light.n.02 28 lighting +253 pipes pipe 11 41 40 7 pipe otherprop Objects n03944672 pipe.n.02 40 misc +254 display case display case 11 540 39 6 display case otherfurniture Furniture n02975212 case.n.20 39 objects +255 weight machine exercise equipment 10 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +256 toilet paper holder toilet paper holder 10 647 40 7 toilet paper holder otherprop Objects 40 misc +257 basin basin 10 24 34 7 sink sink Objects sink n04223580 sink.n.01 15 sink +258 towel bar towel bar 10 51 38 7 bar otherstructure Objects n04459909 towel_rail.n.01 39 objects +259 floor behind floor /otherroom 10 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +260 wooden chair chair 10 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +261 potted plant plant 10 82 40 7 plant otherprop Objects plant n00017222 plant.n.02 14 plant +262 ceiling / other room ceiling /otherroom 10 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +263 window sill window frame 10 59 9 13 window window Window n04589593 window_frame.n.01 9 window +264 floor / other room floor /otherroom 10 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +265 cupboard cabinet 10 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +266 tray tray 10 179 40 7 tray otherprop Objects n04476259 tray.n.01 39 objects +267 urn urn 10 151 40 7 urn otherprop Objects vase jar 03593526 n04516116 urn.n.01 39 objects +268 church seating pew 10 204 39 6 bench otherfurniture Furniture bench bench 02828884 n03920867 pew.n.01 34 seating +269 decorative plate decorative plate 9 383 40 7 decorative plate otherprop Objects 40 misc +270 doors door 9 28 8 12 door door Wall door n03221720 door.n.01 4 door +271 bar bar 9 51 38 7 bar otherstructure Objects n02788689 bar.n.03 39 objects +272 stair rail banister 9 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +273 window shade window shade 9 40 7 otherprop Objects n04590129 window_shade.n.01 32 blinds +274 grass grass 11 82 40 7 plant otherprop Objects plant n12102133 grass.n.01 14 plant +275 pool table pool table 9 515 39 6 pool table otherfurniture Furniture table table table 04379243 n03982430 pool_table.n.01 5 table +276 coat coat 9 324 40 7 jacket otherprop Objects n03057021 coat.n.01 38 clothes +277 trees tree 9 82 40 7 plant otherprop Objects plant n13104059 tree.n.01 14 plant +278 cloth cloth 11 40 7 otherprop Objects n03309808 fabric.n.01 39 objects +279 bottle of soap bottle of soap 9 502 40 7 bottle of soap otherprop Objects 40 misc +280 floor lamp floor lamp 9 144 35 7 lamp lamp Objects lamp lamp 03636649 n03367059 floor_lamp.n.01 28 lighting +281 water cooler water cooler 9 509 39 6 water cooler otherfurniture Furniture n04559166 water_cooler.n.01 39 objects +282 pews pew 9 204 39 6 bench otherfurniture Furniture bench bench 02828884 n03920867 pew.n.01 34 seating +283 ledge ledge 10 38 7 otherstructure Objects n09337253 ledge.n.01 39 objects +284 kitchen shelf kitchen shelf 9 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +285 bathroom utencils bathroom utensil 8 267 40 7 utensil otherprop Objects n04516672 utensil.n.01 39 objects +286 hanger hanger 8 211 40 7 hanger otherprop Objects n03490884 hanger.n.02 39 objects +287 shrubbery shrubbery 8 40 7 otherprop Objects n08649067 shrubbery.n.01 39 objects +288 teapot teapot 8 678 40 7 tea pot otherprop Objects n04398044 teapot.n.01 39 objects +289 bottles bottle 8 2 40 7 bottle otherprop Objects bottle bottle 02876657 n02876657 bottle.n.01 39 objects +290 exercise equipment exercise equipment 8 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +291 boxes box 8 26 29 7 box box Objects n02883344 box.n.01 39 objects +292 locker locker 8 3 3 6 cabinet cabinet Furniture n02933462 cabinet.n.03 40 misc +293 wall cabinet wall cabinet 8 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +294 wainscotting \other room paneling /otherroom 8 21 1 12 wall wall Wall n03882611 paneling.n.01 1 wall +295 ceiling light ceiling lamp 9 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +296 ornament ornament 8 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +297 bidet bidet 8 124 33 7 toilet toilet Objects toilet toilet n02836174 bidet.n.01 18 toilet +298 shower soap shelf shower soap shelf 8 40 7 otherprop Objects 40 misc +299 window / door window/door 8 40 7 otherprop Objects 40 misc +300 stuffed animal stuffed animal 8 177 40 7 stuffed animal otherprop Objects n04399382 teddy.n.01 39 objects +301 paper towel dispenser paper towel dispenser 8 14 40 7 paper towel dispenser otherprop Objects 40 misc +302 chair bottom chair 8 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +303 fencing fencing 8 40 7 otherprop Objects n03327234 fence.n.01 40 misc +304 lampshade lampshade 8 859 40 7 lamp shade otherprop Objects n03637318 lampshade.n.01 28 lighting +305 door side door frame 12 28 8 12 door door Wall door n03221720 door.n.01 4 door +306 bust bust 7 294 40 7 sculpture otherprop Objects n02926188 bust.n.03 39 objects +307 car car 7 530 40 7 car otherprop Objects car car 02958343 n02958343 car.n.01 39 objects +308 figure figure 7 40 7 otherprop Objects 40 misc +309 sofa set sofa set 7 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +310 commode toilet 7 124 33 7 toilet toilet Objects toilet toilet n04446276 toilet.n.01 18 toilet +311 toilet brush toilet brush 7 630 40 7 toilet brush otherprop Objects 40 misc +312 doll doll 7 99 40 7 doll otherprop Objects n03219135 doll.n.01 39 objects +313 drums drum 7 145 40 7 drum otherprop Objects n03249569 drum.n.01 39 objects +314 bathroom counter bathroom counter 7 7 12 6 counter counter Furniture table table table 04379243 n03116530 counter.n.01 26 counter +315 dress dress 7 40 7 otherprop Objects n03236735 dress.n.01 38 clothes +316 shower handle shower handle 7 758 40 7 handle otherprop Objects 40 misc +317 closet door closet door 7 28 8 12 door door Wall door n03221720 door.n.01 4 door +318 whiteboard whiteboard 7 45 30 7 whiteboard whiteboard Objects n03211616 display_panel.n.01 22 tv_monitor +319 garage door opener garage door opener 7 40 7 otherprop Objects 40 misc +320 range hood range hood 7 380 38 7 range hood otherstructure Objects range_hood n04053677 range_hood.n.01 39 objects +321 window curtain window curtain 7 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +322 easel easel 7 50 39 6 stand otherfurniture Furniture n03262809 easel.n.01 31 shelving +323 bowl of fruit bowl of fruit 7 22 40 7 bowl otherprop Objects bowl bowl 02880940 n02880940 bowl.n.03 39 objects +324 molding molding 10 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +325 pool pool 7 38 7 otherstructure Objects n03982060 pool.n.01 40 misc +326 kitchen appliance kitchen appliance 6 40 7 otherprop Objects n03620052 kitchen_appliance.n.01 37 appliances +327 candelabra candelabra 6 605 40 7 candelabra otherprop Objects n02947818 candelabrum.n.01 39 objects +328 ceiling lamp ceiling lamp 6 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +329 toy toy 6 389 40 7 toy otherprop Objects n03964744 plaything.n.01 39 objects +330 top top 6 40 7 otherprop Objects 40 misc +331 wall art picture 6 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +332 highchair highchair 6 5 5 4 chair chair Chair chair chair chair 03001627 n03518445 highchair.n.01 3 chair +333 footrest footrest 6 163 39 6 foot rest otherfurniture Furniture stool n03380724 footstool.n.01 19 stool +334 bathroom sink sink 6 24 34 7 sink sink Objects sink n04223580 sink.n.01 15 sink +335 soap dish soap dish 6 638 40 7 soap dish otherprop Objects n04254009 soap_dish.n.01 39 objects +336 miscellaneous object object 6 40 7 otherprop Objects n00002684 object.n.01 39 objects +337 trim molding 7 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +338 tabletop object object 6 40 7 otherprop Objects n00002684 object.n.01 39 objects +339 clothes hanger rod clothes hanger rod 6 40 7 otherprop Objects n04100174 rod.n.01 39 objects +340 altar altar 6 19 7 10 table table Table table table table 04379243 n02699629 altar.n.01 5 table +341 candles candle 6 137 40 7 candle otherprop Objects lamp n02948072 candle.n.01 39 objects +342 placemat place mat 6 154 40 7 placemat otherprop Objects n03952886 place_mat.n.01 39 objects +343 kitchen center island kitchen island 6 456 38 7 kitchen island otherstructure Objects n03620600 kitchen_island.n.01 26 counter +344 plate of food plate of food 6 40 7 otherprop Objects 40 misc +345 sheet sheet 7 559 40 7 sheet otherprop Objects 40 misc +346 wood wood 6 40 7 otherprop Objects 40 misc +347 robe robe 6 40 7 otherprop Objects n04097866 robe.n.01 38 clothes +348 bathroom stall bathroom stall 6 38 7 otherstructure Objects n02873839 booth.n.02 40 misc +349 tabletop decoration decoration 6 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +350 plush toy plush toy 6 389 40 7 toy otherprop Objects n04399382 teddy.n.01 39 objects +351 wash basin washbasin 6 24 34 7 sink sink Objects sink n04553920 washbasin.n.01 15 sink +352 celing ceiling 6 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +353 hangers hanger 6 211 40 7 hanger otherprop Objects n03490884 hanger.n.02 39 objects +354 bushes bush 6 82 40 7 plant otherprop Objects plant n13112664 shrub.n.01 14 plant +355 floor/other room floor /otherroom 6 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +356 dinner placesetting place mat 6 154 40 7 placemat otherprop Objects n03952886 place_mat.n.01 39 objects +357 curtain valence curtain valence 6 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +358 control control 6 40 7 otherprop Objects n03096960 control.n.09 39 objects +359 tap tap 6 9 40 7 faucet otherprop Objects faucet 03325088 n04559451 water_faucet.n.01 39 objects +360 shampoo shampoo 6 548 40 7 cleaner otherprop Objects n04183516 shampoo.n.01 39 objects +361 computer computer 7 46 40 7 computer otherprop Objects n03082979 computer.n.01 39 objects +362 massage bed massage bed 6 157 4 1 bed bed Bed bed bed bed 02818832 n02818832 bed.n.01 11 bed +363 knob knob 6 652 40 7 knob otherprop Objects 40 misc +364 door stopper door stopper 6 40 7 otherprop Objects 40 misc +365 bulletin board bulletin board 6 408 38 7 board otherstructure Objects n03211616 display_panel.n.01 22 tv_monitor +366 brick archway archway 6 21 1 12 wall wall Wall n02734217 arch.n.03 4 door +367 fruit bowl fruit bowl 6 22 40 7 bowl otherprop Objects bowl bowl 02880940 n02880940 bowl.n.03 39 objects +368 electric wire casing electric wire casing 5 40 7 otherprop Objects 40 misc +369 bookcase bookshelf 5 88 10 6 bookshelf bookshelf Furniture bookshelf bookshelf 02871439 n02871439 bookshelf.n.01 31 shelving +370 other room unknown /otherroom 6 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +371 exercise machine exercise equipment 5 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +372 storage shelving storage shelving 5 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +373 coffee maker coffee maker 5 40 7 otherprop Objects n03063338 coffee_maker.n.01 37 appliances +374 shower ceiling shower ceiling 5 4 22 3 ceiling ceiling Ceiling n04208936 shower.n.01 23 shower +375 fire alarm fire alarm 5 338 40 7 fire alarm otherprop Objects n03343737 fire_alarm.n.02 39 objects +376 tissue paper tissue paper 5 15 26 7 paper paper Objects 40 misc +377 projector projector 5 90 40 7 projector otherprop Objects n04009552 projector.n.02 39 objects +378 coat hanger coat hanger 5 400 40 7 coat hanger otherprop Objects n03057920 coat_hanger.n.01 39 objects +379 wall cubby wall cubby 5 40 7 otherprop Objects 40 misc +380 balcony railing balcony railing 5 497 38 7 railing otherstructure Objects 40 misc +381 shelf /w clutter shelf /w clutter 5 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +382 case case 5 851 40 7 case otherprop Objects 40 misc +383 coat rack coat rack 5 40 7 otherprop Objects n03059103 coatrack.n.01 40 misc +384 pan pan 5 589 40 7 pan otherprop Objects n03880531 pan.n.01 39 objects +385 fridge refrigerator 5 17 24 6 refridgerator refridgerator Furniture n04070727 refrigerator.n.01 37 appliances +386 suitcase luggage 5 783 40 7 luggage otherprop Objects n02774630 baggage.n.01 39 objects +387 wardrobe rod closet rod 5 40 7 otherprop Objects n04100174 rod.n.01 39 objects +388 hamper clothes hamper 5 39 40 7 basket otherprop Objects basket 02801938 n03050864 clothes_hamper.n.01 39 objects +389 trinket trinket 5 844 40 7 trinket otherprop Objects n02787435 bangle.n.02 39 objects +390 c;lothes hangers clothes hanger 5 211 40 7 hanger otherprop Objects n03057920 coat_hanger.n.01 39 objects +391 paper paper 5 15 26 7 paper paper Objects n14974264 paper.n.01 39 objects +392 back splash backsplash 5 40 7 otherprop Objects 40 misc +393 chimney chimney 5 702 38 7 chimney otherstructure Objects n03017428 chimney.n.01 40 misc +394 arc arch 5 40 7 otherprop Objects n02733524 arch.n.04 40 misc +395 shower bench shower bench 5 204 39 6 bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +396 person person 5 331 31 7 person person Objects person n05217688 person.n.02 39 objects +397 tablet tablet 5 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +398 exercise mat exercise mat 5 143 20 5 floor mat floor mat Floor n03727946 mat.n.03 33 gym_equipment +399 smoke alarm smoke alarm 5 525 40 7 alarm otherprop Objects n03343737 fire_alarm.n.02 39 objects +400 kitchen utensils kitchen utensil 5 267 40 7 utensil otherprop Objects n03621049 kitchen_utensil.n.01 39 objects +401 weights weight 5 40 7 otherprop Objects n04571292 weight.n.02 33 gym_equipment +402 display cabinet display cabinet 5 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +403 showcase display cabinet 5 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +404 bedpost bedpost 5 40 7 otherprop Objects n02821415 bedpost.n.01 11 bed +405 file cabinet file cabinet 5 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +406 umbrella umbrella 5 203 40 7 umbrella otherprop Objects n04507155 umbrella.n.01 39 objects +407 massage table massage table 5 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +408 laundry basket laundry basket 5 164 40 7 laundry basket otherprop Objects basket 02801938 n03050864 clothes_hamper.n.01 39 objects +409 jar jar 5 70 40 7 jar otherprop Objects jar 03593526 n03593526 jar.n.01 39 objects +410 exercise bike exercise bike 5 457 39 6 excercise equipment otherfurniture Furniture n03302671 exercise_bike.n.01 33 gym_equipment +411 hose hose 5 40 7 otherprop Objects n03539875 hose.n.03 40 misc +412 window dormer window dormer 5 40 7 otherprop Objects 40 misc +413 closet shelf closet shelf 5 40 7 otherprop Objects n04190052 shelf.n.01 31 shelving +414 power breaker box power breaker box 5 26 29 7 box box Objects 40 misc +415 smoke detector smoke detector 5 40 7 otherprop Objects 40 misc +416 door knob door knob 4 27 40 7 door knob otherprop Objects 40 misc +417 shelf side shelf 4 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +418 jacuzzi jacuzzi 4 40 7 otherprop Objects 40 misc +419 backpack backpack 4 206 40 7 backpack otherprop Objects n02769748 backpack.n.01 39 objects +420 wooden desk desk 4 36 14 10 desk desk Table desk desk table 04379243 n03179701 desk.n.01 5 table +421 bath mat bath mat 4 40 7 otherprop Objects n02807401 bath_mat.n.01 2 floor +422 unknown clutter unknown clutter 4 40 7 otherprop Objects 40 misc +423 hook hook 4 40 7 otherprop Objects 40 misc +424 sauna wall wall 4 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +425 elevator elevator 4 40 7 otherprop Objects 40 misc +426 tool tool 4 40 7 otherprop Objects n04451818 tool.n.01 39 objects +427 recliner recliner 4 5 5 4 chair chair Chair chair chair chair 03001627 n04062428 recliner.n.01 3 chair +428 recessed wall recessed wall 4 21 1 12 wall wall Wall 40 misc +429 closet bar closet rod 4 40 7 otherprop Objects n04100174 rod.n.01 39 objects +430 tank tank 4 40 7 otherprop Objects 40 misc +431 toaster toaster 4 251 40 7 toaster otherprop Objects n04442312 toaster.n.02 37 appliances +432 toilet brush holder toilet brush holder 4 40 7 otherprop Objects 40 misc +433 handrail /otherroom handrail /otherroom 4 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +434 landing landing 6 40 7 otherprop Objects n03638511 landing.n.01 2 floor +435 book rack book rack 4 224 39 6 bookrack otherfurniture Furniture 40 misc +436 wall mirror mirror 4 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +437 hunting trophy hunting trophy 4 547 40 7 trophy otherprop Objects 40 misc +438 rod rod 4 40 7 otherprop Objects n04100174 rod.n.01 39 objects +439 floor mat floor mat 4 143 20 5 floor mat floor mat Floor n03727837 mat.n.01 2 floor +440 motion detector motion detector 4 40 7 otherprop Objects 40 misc +441 clothing clothes 4 141 21 7 clothes clothes Objects n02728440 apparel.n.01 38 clothes +442 can of paint can of paint 4 40 7 otherprop Objects 40 misc +443 medicine cabinet medicine cabinet 4 3 3 6 cabinet cabinet Furniture cabinet 02933112 n03742115 medicine_chest.n.01 7 cabinet +444 sensor sensor 4 40 7 otherprop Objects n03180969 detector.n.01 39 objects +445 cart cart 4 305 40 7 cart otherprop Objects n03484083 handcart.n.01 39 objects +446 slab slab 4 38 7 otherstructure Objects n04233405 slab.n.01 39 objects +447 bean bag chair bean bag chair 4 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +448 window valence window valence 4 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +449 window /outside window /outside 4 59 9 13 window window Window n04587648 window.n.01 9 window +450 pole pole 4 40 7 otherprop Objects n03976657 pole.n.01 39 objects +451 picture / other room picture /otherroom 4 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +452 canister canister 4 794 40 7 canister otherprop Objects 40 misc +453 washbasin counter washbasin counter 4 7 12 6 counter counter Furniture table table table 04379243 n03116530 counter.n.01 26 counter +454 kitchen top kitchen top 4 7 12 6 counter counter Furniture n03118245 countertop.n.01 26 counter +455 unknown object object 4 40 7 otherprop Objects n00002684 object.n.01 39 objects +456 pitcher pitcher 4 273 40 7 pitcher otherprop Objects n03950228 pitcher.n.02 39 objects +457 showerhead showerhead 4 650 40 7 shower head otherprop Objects n04209383 showerhead.n.01 23 shower +458 podium podium 4 40 7 otherprop Objects n03159640 dais.n.01 39 objects +459 ceiling vent ceiling vent 4 25 38 7 air vent otherstructure Objects n04526241 vent.n.01 40 misc +460 throw pillow pillow 4 119 18 7 pillow pillow Objects pillow 03938244 n03938244 pillow.n.01 8 cushion +461 grill grill 4 700 38 7 grill otherstructure Objects n03459591 grill.n.02 40 misc +462 sink cabinet sink cabinet 4 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +463 tapestry tapestry 4 40 7 otherprop Objects n04393549 tapestry.n.02 39 objects +464 bed sheet bed sheet 5 559 40 7 sheet otherprop Objects n04188179 sheet.n.03 11 bed +465 shade shade 4 40 7 otherprop Objects n04181718 shade.n.03 39 objects +466 doorknob doorknob 4 27 40 7 door knob otherprop Objects n03222959 doorknob.n.01 4 door +467 stair railing banister 4 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +468 vacuum cleaner vacuum cleaner 4 306 40 7 vacuum cleaner otherprop Objects n04517823 vacuum.n.04 37 appliances +469 bed comforter bed comforter 4 484 40 7 comforter otherprop Objects 40 misc +470 door / other room door /otherroom 4 28 8 12 door door Wall door n03221720 door.n.01 4 door +471 pictures picture 4 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +472 bed table bed table 4 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +473 shirt shirt 4 40 7 otherprop Objects n04197391 shirt.n.01 38 clothes +474 dressing table dressing table 4 19 7 10 table table Table table table table 04379243 n03238586 dressing_table.n.01 5 table +475 shower wall cubby shower wall cubby 4 40 7 otherprop Objects n04208936 shower.n.01 23 shower +476 side wall wall 4 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +477 beside table beside table 4 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +478 curb curb 4 40 7 otherprop Objects n03149135 curb.n.02 39 objects +479 arches arch 4 40 7 otherprop Objects n02733524 arch.n.04 40 misc +480 storage bin storage bin 4 812 40 7 storage bin otherprop Objects 40 misc +481 support beam support beam 4 40 7 otherprop Objects n02815950 beam.n.02 29 beam +482 globe globe 5 347 40 7 globe otherprop Objects 40 misc +483 pantry pantry 4 38 7 otherstructure Objects n03885535 pantry.n.01 40 misc +484 skateboard skateboard 4 408 38 7 board otherstructure Objects skateboard 04225987 n04225987 skateboard.n.01 39 objects +485 stove hood range hood 4 380 38 7 range hood otherstructure Objects range_hood n04053677 range_hood.n.01 39 objects +486 cabin cabin 4 40 7 otherprop Objects 40 misc +487 shower bar shower bar 4 51 38 7 bar otherstructure Objects 40 misc +488 chaise chaise 4 5 5 4 chair chair Chair chair chair chair 03001627 n03002711 chaise_longue.n.01 3 chair +489 flower wash flower vase 4 78 40 7 vase otherprop Objects vase jar 03593526 n04522168 vase.n.01 39 objects +490 desk and chairs desk and chairs 4 40 7 otherprop Objects 40 misc +491 plant vase flower vase 4 78 40 7 vase otherprop Objects vase jar 03593526 n04522168 vase.n.01 39 objects +492 towel rack towel rack 4 40 7 otherprop Objects n04459773 towel_rack.n.01 40 misc +493 cross cross 4 49 40 7 monitor otherprop Objects n03857828 oscilloscope.n.01 39 objects +494 sliding door sliding door 4 28 8 12 door door Wall door n04239074 sliding_door.n.01 4 door +495 cosmetics cosmetics 4 40 7 otherprop Objects n03113152 cosmetic.n.01 39 objects +496 wall other room wall /otherroom 4 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +497 kettle kettle 4 16 40 7 pot otherprop Objects n03612814 kettle.n.01 39 objects +498 junk junk 4 40 7 otherprop Objects n14857897 debris.n.01 39 objects +499 stationery stationery 3 15 26 7 paper paper Objects n06258852 stationery.n.01 39 objects +500 lights light 3 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +501 window / other room window /otherroom 3 59 9 13 window window Window n04587648 window.n.01 9 window +502 dish cabinet dish cabinet 3 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +503 wall panel wall panel 3 21 1 12 wall wall Wall n04548503 wall_panel.n.01 1 wall +504 gate gate 3 223 38 7 gate otherstructure Objects n03427296 gate.n.01 40 misc +505 safe safe 3 26 29 7 box box Objects n04125021 safe.n.01 39 objects +506 ventilation ventilation 3 40 7 otherprop Objects n04526520 ventilation.n.02 39 objects +507 logs firewood 3 40 7 otherprop Objects n15100644 firewood.n.01 40 misc +508 sliding doors sliding door 3 28 8 12 door door Wall door n04239074 sliding_door.n.01 4 door +509 shower rod shower rod 3 40 7 otherprop Objects n04100174 rod.n.01 12 curtain +510 towel shelf towel shelf 3 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +511 row of theater chairs row of theater chairs 3 40 7 otherprop Objects 40 misc +512 closet wall wall 3 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +513 bath utencils bath utensil 3 267 40 7 utensil otherprop Objects 40 misc +514 sconce sconce 3 62 38 7 light otherstructure Objects n04148703 sconce.n.03 28 lighting +515 garage light garage light 3 62 38 7 light otherstructure Objects 40 misc +516 ceiling beam beam 3 38 7 otherstructure Objects n02815950 beam.n.02 29 beam +517 toolbox toolbox 3 344 39 6 chest otherfurniture Furniture n04452615 toolbox.n.01 39 objects +518 sponge stool stool 3 150 40 7 stool otherprop Objects stool n04326896 stool.n.01 19 stool +519 security camera security camera 3 212 40 7 security camera otherprop Objects camera 02942699 n02942699 camera.n.01 39 objects +520 gym machine exercise equipment 3 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +521 wall /outside wall /outside 3 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +522 mantle mantle 3 874 38 7 mantle otherstructure Objects mantel n03719343 mantel.n.01 27 fireplace +523 floor 2 floor 3 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +524 skirting skirting 3 40 7 otherprop Objects 40 misc +525 banister banister 3 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +526 lockers locker 3 3 3 6 cabinet cabinet Furniture n02933462 cabinet.n.03 40 misc +527 trophy trophy 3 547 40 7 trophy otherprop Objects 40 misc +528 tile tile 3 40 7 otherprop Objects n04435180 tile.n.01 39 objects +529 unknown / room below unknown /otherroom 3 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +530 picture car picture car 3 530 40 7 car otherprop Objects car car 02958343 n02958343 car.n.01 39 objects +531 outlet outlet 3 40 7 otherprop Objects n04548771 wall_socket.n.01 39 objects +532 plant ridge plant ridge 3 40 7 otherprop Objects 40 misc +533 backsplash backsplash 3 40 7 otherprop Objects 40 misc +534 computer monitor monitor 3 49 40 7 monitor otherprop Objects monitor monitor tv or monitor 03211117 n03782190 monitor.n.04 22 tv_monitor +535 doorframe / other room doorframe /otherroom 3 615 38 7 door frame otherstructure Objects n03222722 doorframe.n.01 4 door +536 shower wall /otherroom shower wall /otherroom 3 21 1 12 wall wall Wall n04208936 shower.n.01 23 shower +537 shelves with wine shelving 3 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +538 shoe shoe 3 149 40 7 shoe otherprop Objects n04199027 shoe.n.01 39 objects +539 hedge hedge 3 40 7 otherprop Objects n03511175 hedge.n.01 40 misc +540 glass window window 3 59 9 13 window window Window n04587648 window.n.01 9 window +541 sofachair sofa chair 3 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +542 hand soap hand soap 3 133 40 7 soap otherprop Objects n04253437 soap.n.01 39 objects +543 window blinds blinds 3 80 13 13 blinds blinds Window n04589190 window_blind.n.01 32 blinds +544 flower vase flower vase 3 78 40 7 vase otherprop Objects vase jar 03593526 n04522168 vase.n.01 39 objects +545 appliance appliance 3 40 7 otherprop Objects 40 misc +546 christmas tree christmas tree 3 40 7 otherprop Objects 40 misc +547 support column column 3 94 38 7 column otherstructure Objects n03074380 column.n.06 40 misc +548 dish plate 3 233 40 7 plate otherprop Objects n03959485 plate.n.04 39 objects +549 closet floor closet floor 3 11 2 5 floor floor Floor 40 misc +550 celling ceiling 3 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +551 stuffed animals stuffed animal 3 177 40 7 stuffed animal otherprop Objects n04399382 teddy.n.01 39 objects +552 casket casket 3 40 7 otherprop Objects 40 misc +553 rowing machine exercise machine 3 220 40 7 machine otherprop Objects 40 misc +554 dining table centerpiece dining table centerpiece 3 878 40 7 centerpiece otherprop Objects 40 misc +555 bedside lamp bedside lamp 5 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +556 kitchen countertop item kitchen countertop item 3 40 7 otherprop Objects 40 misc +557 fountain fountain 3 40 7 otherprop Objects n03388043 fountain.n.01 40 misc +558 window glass window glass 3 612 38 7 glass otherstructure Objects n03881893 pane.n.01 39 objects +559 wall soffet soffit 3 40 7 otherprop Objects n04256758 soffit.n.01 39 objects +560 urinal urinal 3 40 7 otherprop Objects n04515991 urinal.n.01 39 objects +561 kitchen hood range hood 3 380 38 7 range hood otherstructure Objects range_hood n04053677 range_hood.n.01 39 objects +562 showpiece decoration 3 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +563 wall trim molding 3 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +564 barrel barrel 3 343 39 6 barrel otherfurniture Furniture 40 misc +565 firewood firewood 3 40 7 otherprop Objects n15100644 firewood.n.01 40 misc +566 columns pillar 3 94 38 7 column otherstructure Objects n03073977 column.n.07 24 column +567 beams beam 3 38 7 otherstructure Objects n02815950 beam.n.02 29 beam +568 carpet roll carpet roll 3 40 7 otherprop Objects 40 misc +569 portrait portrait 3 64 11 8 picture picture Picture n03987079 portrait.n.02 6 picture +570 table light table light 3 62 38 7 light otherstructure Objects 40 misc +571 water heater water heater 3 588 40 7 water heater otherprop Objects n04560113 water_heater.n.01 39 objects +572 stairs 2 stair 3 215 38 7 stairs otherstructure Objects stairs n04314914 step.n.04 16 stairs +573 pouffe pouffe 3 359 39 6 ottoman otherfurniture Furniture n03858418 ottoman.n.03 34 seating +574 ceiling behind ceiling /otherroom 3 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +575 pillars pillar 3 94 38 7 column otherstructure Objects n03073977 column.n.07 24 column +576 concrete block concrete block 3 40 7 otherprop Objects 40 misc +577 range stove 3 242 38 7 stove otherstructure Objects stove 04330267 n04330267 stove.n.02 37 appliances +578 shelf with objects shelf 3 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +579 toilet seat liner dispenser toilet seat liner dispenser 3 40 7 otherprop Objects 40 misc +580 patio chair patio chair 3 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +581 folding chair folding chair 3 5 5 4 chair chair Chair chair chair chair 03001627 n03376595 folding_chair.n.01 3 chair +582 island island 4 456 38 7 kitchen island otherstructure Objects n03620600 kitchen_island.n.01 26 counter +583 toaster oven toaster oven 3 275 40 7 toaster oven otherprop Objects n04442441 toaster_oven.n.01 37 appliances +584 overlook railing railing 3 497 38 7 railing otherstructure Objects n04047401 railing.n.01 30 railing +585 bathroom mirror mirror 3 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +586 recycle bin recycle bin 3 307 40 7 bin otherprop Objects 40 misc +587 counter top countertop 3 7 12 6 counter counter Furniture n03118245 countertop.n.01 26 counter +588 rafter rafter 3 40 7 otherprop Objects n04045644 rafter.n.01 29 beam +589 dryer clothes dryer 3 39 6 otherfurniture Furniture n03251766 dryer.n.01 37 appliances +590 bed lamp bedside lamp 3 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +591 wall paneling paneling 3 21 1 12 wall wall Wall n03882611 paneling.n.01 1 wall +592 stage stage 3 40 7 otherprop Objects 40 misc +593 fire sprinkler fire sprinkler 3 40 7 otherprop Objects 40 misc +594 brush brush 3 40 7 otherprop Objects n02908217 brush.n.02 39 objects +595 wall 2 wall 3 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +596 balcony balcony 3 40 7 otherprop Objects 40 misc +597 water tank water tank 2 263 40 7 vessel otherprop Objects n03035715 cistern.n.02 39 objects +598 pile of objects clutter 2 40 7 otherprop Objects 40 misc +599 garage door frame garage door frame 2 40 7 otherprop Objects 40 misc +600 back wall wall 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +601 plant soil plant soil 2 40 7 otherprop Objects 40 misc +602 globe stand globe stand 2 466 40 7 globe stand otherprop Objects 40 misc +603 bicycle bicycle 2 189 40 7 bicycle otherprop Objects bicycle 02834778 n02834778 bicycle.n.01 39 objects +604 stairframe stair frame 2 40 7 otherprop Objects 40 misc +605 separating screen partition 2 21 1 12 wall wall Wall n03894379 partition.n.01 40 misc +606 unknown wall object object 2 40 7 otherprop Objects n00002684 object.n.01 39 objects +607 air duct air duct 2 38 38 7 air duct otherstructure Objects n02690941 air_passage.n.01 40 misc +608 kitchen upper cabinet kitchen cabinet 2 3 3 6 cabinet cabinet Furniture n02933112 cabinet.n.01 7 cabinet +609 chair pillow pillow 2 119 18 7 pillow pillow Objects pillow 03938244 n03938244 pillow.n.01 8 cushion +610 led tv led tv 2 40 7 otherprop Objects 40 misc +611 deco decoration 2 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +612 ceiling under staircase ceiling under staircase 2 40 7 otherprop Objects 40 misc +613 chair /w books chair /w books 2 85 23 2 books books Books 40 misc +614 bed back rest headboard 2 161 39 6 headboard otherfurniture Furniture n03502200 headboard.n.01 11 bed +615 giraffe giraffe 2 40 7 otherprop Objects n02439033 giraffe.n.01 39 objects +616 lightswitch light switch 2 301 38 7 light switch otherstructure Objects n04372370 switch.n.01 39 objects +617 doorframe \other room doorframe /otherroom 2 615 38 7 door frame otherstructure Objects n03222722 doorframe.n.01 4 door +618 object / camera? object 2 40 7 otherprop Objects n00002684 object.n.01 39 objects +619 grandfather clock grandfather clock 2 462 39 6 grandfather clock otherfurniture Furniture clock 03046257 n03452594 grandfather_clock.n.01 39 objects +620 jewelry box jewelry box 2 26 29 7 box box Objects 40 misc +621 bottles of wine bottles of wine 2 766 40 7 wine otherprop Objects 40 misc +622 massage table base massage table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +623 hood range hood 2 380 38 7 range hood otherstructure Objects range_hood n04053677 range_hood.n.01 39 objects +624 mirrorframe mirror frame 2 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +625 wall beam wall beam 2 40 7 otherprop Objects 40 misc +626 wooden trim molding 2 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +627 stalls stall 2 38 7 otherstructure Objects n02873839 booth.n.02 40 misc +628 partition partition 2 21 1 12 wall wall Wall n03894379 partition.n.01 40 misc +629 dog dog 2 701 40 7 dog otherprop Objects 40 misc +630 valance valance 2 40 7 otherprop Objects n03111296 cornice.n.01 40 misc +631 radio radio 2 188 40 7 radio otherprop Objects radio 40 misc +632 bush bush 2 82 40 7 plant otherprop Objects plant n13112664 shrub.n.01 14 plant +633 row of theater seats row of theater seats 2 40 7 otherprop Objects 40 misc +634 bath utencil bath utensil 2 267 40 7 utensil otherprop Objects 40 misc +635 basket of towels basket of towels 2 40 7 otherprop Objects 40 misc +636 laundry machine washing machine 2 278 39 6 washing machine otherfurniture Furniture washing_machine 04554684 n04554684 washer.n.03 37 appliances +637 mirror /otherroom mirror /otherroom 2 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +638 toilet sink toilet sink 2 24 34 7 sink sink Objects sink 40 misc +639 sauna heater sauna heater 2 111 39 6 heater otherfurniture Furniture 40 misc +640 dining bench dining bench 2 204 39 6 bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +641 fume cupboard fume cupboard 2 40 7 otherprop Objects 40 misc +642 mouse mouse 2 103 40 7 mouse otherprop Objects n03793489 mouse.n.04 39 objects +643 boiler boiler 2 40 7 otherprop Objects 40 misc +644 hearth hearth 2 372 38 7 fireplace otherstructure Objects 40 misc +645 curtain darker curtain 2 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +646 round chair round chair 2 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +647 toilet bowl toilet 2 124 33 7 toilet toilet Objects toilet toilet n04446276 toilet.n.01 18 toilet +648 whine rack wine rack 2 299 40 7 wine rack otherprop Objects 40 misc +649 doorstep doorstep 3 28 8 12 door door Wall n03223686 doorsill.n.01 39 objects +650 binder binder 2 399 40 7 binder otherprop Objects 40 misc +651 shower door frame shower door frame 2 40 7 otherprop Objects n04208936 shower.n.01 23 shower +652 bed runner bed runner 2 157 4 1 bed bed Bed n02822579 bedstead.n.01 11 bed +653 cubicle cubicle 2 40 7 otherprop Objects 40 misc +654 fitness device exercise equipment 2 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +655 support support 2 40 7 otherprop Objects 40 misc +656 overhang overhang 2 40 7 otherprop Objects n03864356 overhang.n.01 40 misc +657 electric box electric box 2 550 38 7 electric box otherstructure Objects 40 misc +658 bathrobe bathrobe 3 40 7 otherprop Objects n02807616 bathrobe.n.01 38 clothes +659 door mat doormat 2 143 20 5 floor mat floor mat Floor n03223299 doormat.n.02 2 floor +660 jacket jacket 2 324 40 7 jacket otherprop Objects n03589791 jacket.n.01 38 clothes +661 cabinet table cabinet table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +662 side frame frame 2 38 7 otherstructure Objects 40 misc +663 wainscoting paneling 2 21 1 12 wall wall Wall n03882611 paneling.n.01 1 wall +664 staircase trim staircase trim 2 40 7 otherprop Objects 40 misc +665 box /w books box /w books 2 85 23 2 books books Books 40 misc +666 nighstand nightstand 2 158 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 13 chest_of_drawers +667 window reflection window reflection 2 40 7 otherprop Objects 40 misc +668 pulpit pulpit 2 40 7 otherprop Objects n03159640 dais.n.01 39 objects +669 set of armchairs set of armchairs 2 40 7 otherprop Objects 40 misc +670 fish tank fish tank 2 782 38 7 fish tank otherstructure Objects n02732072 aquarium.n.01 40 misc +671 bathroom countertop objects objects 2 40 7 otherprop Objects n00002684 object.n.01 39 objects +672 concrete shelf shelf 2 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +673 ceiling wall ceiling wall 2 21 1 12 wall wall Wall 40 misc +674 picture \other room picture /otherroom 2 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +675 wall entry wall 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +676 lintel lintel 2 40 7 otherprop Objects n03503233 header.n.02 29 beam +677 wall table wall table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +678 small table table 3 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +679 lighting fixture lighting fixture 2 40 7 otherprop Objects n03667380 lighting_fixture.n.01 39 objects +680 bed frame bedframe 2 157 4 1 bed bed Bed n02822579 bedstead.n.01 11 bed +681 freezer freezer 2 17 24 6 refridgerator refridgerator Furniture n03170635 deep-freeze.n.01 37 appliances +682 glass doors door 2 28 8 12 door door Wall door n03221720 door.n.01 4 door +683 extractor extractor 2 40 7 otherprop Objects 40 misc +684 flower pot flowerpot 2 146 40 7 flower pot otherprop Objects flower_pot flower pot 03991062 n03991062 pot.n.04 39 objects +685 soffet soffit 2 40 7 otherprop Objects n04256758 soffit.n.01 39 objects +686 platform platform 2 38 7 otherstructure Objects 40 misc +687 hot tub hot tub 2 136 36 7 bathtub bathtub Objects bathtub bathtub tub 02808440 n03543603 hot_tub.n.01 25 bathtub +688 paper towels paper towel 2 113 40 7 paper towel otherprop Objects n03887697 paper_towel.n.01 20 towel +689 kitchen utencils kitchen utensil 2 267 40 7 utensil otherprop Objects n03621049 kitchen_utensil.n.01 39 objects +690 shower grab bar shower grab bar 2 51 38 7 bar otherstructure Objects 40 misc +691 wall detail wall detail 2 40 7 otherprop Objects 40 misc +692 whineshelf whine shelf 2 40 7 otherprop Objects 40 misc +693 painting/other room painting /otherroom 2 64 11 8 picture picture Picture n03876519 painting.n.01 39 objects +694 television wall tv 2 40 7 otherprop Objects 40 misc +695 sauna bench bench 2 204 39 6 bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +696 fruits fruit 2 286 40 7 fruit otherprop Objects n13134947 fruit.n.01 39 objects +697 picture frames picture frame 2 64 11 8 picture picture Picture n03931765 picture_frame.n.01 6 picture +698 buffet buffet 2 7 12 6 counter counter Furniture table table table 04379243 n04247736 snack_bar.n.01 26 counter +699 billow billow 2 40 7 otherprop Objects 40 misc +700 computer chair computer chair 2 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +701 sofa seat sofa seat 2 40 7 otherprop Objects 40 misc +702 wall tv wall tv 2 40 7 otherprop Objects 40 misc +703 ground floor 2 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +704 wall2 wall 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +705 ceiling under stairs ceiling under stairs 2 215 38 7 stairs otherstructure Objects stairs 40 misc +706 calendar calendar 2 583 40 7 calendar otherprop Objects 40 misc +707 dome dome 2 40 7 otherprop Objects 40 misc +708 object /outside object /outside 2 40 7 otherprop Objects n00002684 object.n.01 39 objects +709 poll poll 2 40 7 otherprop Objects n01817346 poll.n.04 39 objects +710 wet bar wet bar 2 51 38 7 bar otherstructure Objects table table table 04379243 n04573513 wet_bar.n.01 26 counter +711 folding table folding table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +712 stovetop stovetop 2 40 7 otherprop Objects 40 misc +713 gym equip gym equipment 2 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +714 fruitbowl fruit bowl 2 22 40 7 bowl otherprop Objects bowl bowl 02880940 n02880940 bowl.n.03 39 objects +715 vending machine vending machine 2 220 40 7 machine otherprop Objects n04525305 vending_machine.n.01 39 objects +716 handwash hand wash 2 40 7 otherprop Objects 40 misc +717 wall clock wall clock 2 56 40 7 clock otherprop Objects clock 03046257 n04548280 wall_clock.n.01 39 objects +718 liquid soap liquid soap 2 133 40 7 soap otherprop Objects 40 misc +719 trinkets trinket 2 844 40 7 trinket otherprop Objects n02787435 bangle.n.02 39 objects +720 small table / stand small table/stand 2 40 7 otherprop Objects 40 misc +721 window shuts window shutters 2 40 7 otherprop Objects n04211356 shutter.n.02 39 objects +722 door frame 3 door frame 2 28 8 12 door door Wall door n03221720 door.n.01 4 door +723 stone stone 2 578 40 7 stones otherprop Objects n09416076 rock.n.01 39 objects +724 tripod tripod 2 50 39 6 stand otherfurniture Furniture n04485082 tripod.n.01 31 shelving +725 projector screen projector screen 2 53 38 7 projector screen otherstructure Objects 40 misc +726 window frame / other room window frame /otherroom 2 477 38 7 window frame otherstructure Objects n04589593 window_frame.n.01 9 window +727 wreath wreath 2 881 40 7 wreathe otherprop Objects n04606014 wreath.n.01 39 objects +728 door hinge door hinge 2 40 7 otherprop Objects 40 misc +729 sound speaker 2 54 40 7 speaker otherprop Objects speaker 03691459 n03691459 loudspeaker.n.01 39 objects +730 french door french door 2 28 8 12 door door Wall door n03394649 french_door.n.01 4 door +731 staircase handrail staircase handrail 2 40 7 otherprop Objects 40 misc +732 nighttable night table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +733 photo in frame picure 2 64 11 8 picture picture Picture 40 misc +734 photoframe picture 2 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +735 stair wall stair wall 2 21 1 12 wall wall Wall 40 misc +736 closet ceiling closet ceiling 2 4 22 3 ceiling ceiling Ceiling 40 misc +737 stick stick 2 529 40 7 stick otherprop Objects 40 misc +738 fluorescent light fluorescent light 2 62 38 7 light otherstructure Objects 40 misc +739 wash cabinet wash cabinet 2 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +740 shower seat shower seat 2 40 7 otherprop Objects 40 misc +741 trellis trellis 2 40 7 otherprop Objects n04478512 trellis.n.01 40 misc +742 patio patio 2 40 7 otherprop Objects n03899768 patio.n.01 40 misc +743 dart board dartboard 2 408 38 7 board otherstructure Objects n03162940 dartboard.n.01 39 objects +744 comforter comforter 2 484 40 7 comforter otherprop Objects n04033995 quilt.n.01 39 objects +745 table /w books table /w books 2 85 23 2 books books Books 40 misc +746 picture/other room picture /otherroom 2 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +747 dirt dirt 2 40 7 otherprop Objects 40 misc +748 base base 2 40 7 otherprop Objects 40 misc +749 chemical tank chemical tank 2 40 7 otherprop Objects 40 misc +750 step stool step stool 2 276 40 7 step stool otherprop Objects stool n04315713 step_stool.n.01 19 stool +751 misc misc 2 40 7 otherprop Objects 40 misc +752 sink table sink table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +753 curved wall wall 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +754 roof beam beam 2 38 7 otherstructure Objects n02815950 beam.n.02 29 beam +755 cover cover 2 312 40 7 blanket otherprop Objects 40 misc +756 reading table reading table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +757 side steps wall steps wall 2 21 1 12 wall wall Wall 40 misc +758 sideboard sideboard 2 7 12 6 counter counter Furniture 40 misc +759 wall from another room wall /otherroom 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +760 electric outlet electric outlet 2 98 40 7 electrical outlet otherprop Objects n04548771 wall_socket.n.01 39 objects +761 stall door stall door 2 28 8 12 door door Wall door n03221720 door.n.01 4 door +762 separator separator 2 40 7 otherprop Objects n02995998 centrifuge.n.01 39 objects +763 toilet bowl brush holder toilet bowl brush holder 2 40 7 otherprop Objects 40 misc +764 vessel vessel 2 263 40 7 vessel otherprop Objects watercraft 04530566 n04530566 vessel.n.02 39 objects +765 table clutter table clutter 2 40 7 otherprop Objects 40 misc +766 partition wall wall 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +767 bed curtain bed curtain 2 89 16 13 curtain curtain Window curtain 40 misc +768 stairs skirt stairs skirt 2 40 7 otherprop Objects 40 misc +769 small chair chair 2 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +770 ceiling window ceiling window 2 59 9 13 window window Window 40 misc +771 rocking chair rocking chair 2 5 5 4 chair chair Chair chair chair chair 03001627 n04099969 rocking_chair.n.01 3 chair +772 window \other room window /otherroom 2 59 9 13 window window Window n04587648 window.n.01 9 window +773 under stair under stair 2 40 7 otherprop Objects 40 misc +774 unknown outside building unknown /outside 2 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +775 bath wall bath wall 2 21 1 12 wall wall Wall 40 misc +776 panel screen panel screen 2 40 7 otherprop Objects 40 misc +777 doorfra,e doorframe 2 615 38 7 door frame otherstructure Objects n03222722 doorframe.n.01 4 door +778 shower mat shower mat 2 143 20 5 floor mat floor mat Floor n03727837 mat.n.01 2 floor +779 blackboard blackboard 2 225 38 7 blackboard otherstructure Objects n02846511 blackboard.n.01 39 objects +780 drawer desk drawer desk 2 36 14 10 desk desk Table desk desk 40 misc +781 poster picture 2 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +782 fireplace sconce fireplace sconce 2 40 7 otherprop Objects 40 misc +783 table support table support 2 40 7 otherprop Objects 40 misc +784 wall lamp wall lamp 2 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +785 closet shelving closet shelving 5 40 7 otherprop Objects 40 misc +786 closest area closest area 2 40 7 otherprop Objects 40 misc +787 scroll scroll 2 40 7 otherprop Objects 40 misc +788 foot stand foot stand 2 50 39 6 stand otherfurniture Furniture 40 misc +789 button button 2 774 40 7 button otherprop Objects 40 misc +790 art / clutter art/clutter 2 40 7 otherprop Objects 40 misc +791 door arc arch 2 40 7 otherprop Objects n02733524 arch.n.04 40 misc +792 stairs railing stairs railing 2 497 38 7 railing otherstructure Objects 40 misc +793 floor /otherroom floor /otherroom 2 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +794 shovel shovel 2 607 40 7 shovel otherprop Objects 40 misc +795 alarm controls alarm control 2 40 7 otherprop Objects 40 misc +796 lamb lamp 2 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +797 cluttered objects clutter 2 40 7 otherprop Objects 40 misc +798 door/other room door /otherroom 2 28 8 12 door door Wall door n03221720 door.n.01 4 door +799 closet shelves closet shelf 2 40 7 otherprop Objects n04190052 shelf.n.01 31 shelving +800 scultpure sculpture 2 294 40 7 sculpture otherprop Objects n04157320 sculpture.n.01 39 objects +801 arm chair armchair 2 5 5 4 chair chair Chair chair chair chair 03001627 n02738535 armchair.n.01 3 chair +802 door behind door /otherroom 2 28 8 12 door door Wall door n03221720 door.n.01 4 door +803 exercise ball exercise ball 2 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +804 yard yard 2 40 7 otherprop Objects 40 misc +805 semi chair semi chair 2 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +806 bouquet bouquet 2 40 7 otherprop Objects n02879087 bouquet.n.01 39 objects +807 garage door opener bar garage door opener bar 2 51 38 7 bar otherstructure Objects 40 misc +808 pots pot 2 16 40 7 pot otherprop Objects n03991062 pot.n.04 39 objects +809 decorations decoration 2 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +810 unkown unknown 2 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +811 kitchen decoration kitchen decoration 2 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +812 archway corner archway corner 2 40 7 otherprop Objects 40 misc +813 kitchen wall kitchen wall 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +814 cabinet door cabinet door 2 28 8 12 door door Wall door 40 misc +815 sauna bowl sauna bowl 2 22 40 7 bowl otherprop Objects bowl bowl 02880940 n02880940 bowl.n.03 39 objects +816 bar chair bar chair 2 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +817 shelf cubby shelf cubby 2 40 7 otherprop Objects 40 misc +818 toilet plunger toilet plunger 2 563 40 7 toilet plunger otherprop Objects n03970156 plunger.n.03 39 objects +819 belts belt 2 610 40 7 belt otherprop Objects 40 misc +820 sewing machine sewing machine 2 890 40 7 sewing machine otherprop Objects n04179913 sewing_machine.n.01 37 appliances +821 hot water/cold water knob hot water/cold water knob 2 652 40 7 knob otherprop Objects 40 misc +822 barbeque barbecue 2 40 7 otherprop Objects n02790669 barbecue.n.03 40 misc +823 cutting board cutting board 2 247 40 7 cutting board otherprop Objects n03025513 chopping_board.n.01 39 objects +824 soapbox soapbox 2 671 40 7 soap box otherprop Objects 40 misc +825 washing stuff washing stuff 2 40 7 otherprop Objects 40 misc +826 dining table decoration decoration 2 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +827 copier copier 2 40 7 otherprop Objects n03257586 duplicator.n.01 39 objects +828 unknown (picture or window) unknown picture/window 2 40 7 otherprop Objects 40 misc +829 stair handle stair handle 2 758 40 7 handle otherprop Objects n04047401 railing.n.01 30 railing +830 reflection reflection 2 64 11 8 picture picture Picture n04068976 reflection.n.05 6 picture +831 horizonal bar for exercise? exercise equipment 2 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +832 shelving / other room shelving /otherroom 2 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +833 seats seat 2 524 39 6 furniture otherfurniture Furniture n04161981 seat.n.03 34 seating +834 shower stall shower stall 2 40 7 otherprop Objects n04209613 shower_stall.n.01 40 misc +835 chairs chair 2 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +836 stair /otherroom stair /otherroom 2 215 38 7 stairs otherstructure Objects stairs n04314914 step.n.04 16 stairs +837 kitchen stuff clutter 2 40 7 otherprop Objects 40 misc +838 unkown object object 2 40 7 otherprop Objects n00002684 object.n.01 39 objects +839 throne throne 2 5 5 4 chair chair Chair chair chair chair 03001627 n04429376 throne.n.01 3 chair +840 socket socket 2 40 7 otherprop Objects n04255163 socket.n.02 39 objects +841 bathroom art bathroom art 2 40 7 otherprop Objects n02743547 art.n.01 39 objects +842 night table night table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +843 display display 2 40 7 otherprop Objects n03211117 display.n.06 22 tv_monitor +844 tabletop tabletop 2 19 7 10 table table Table n04381860 tabletop.n.01 39 objects +845 thrash bin trash bin 2 307 40 7 bin otherprop Objects trash_bin 02747177 n02747177 ashcan.n.01 39 objects +846 l shape sofa l-shaped sofa 2 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +847 cardboard box cardboard box 2 26 29 7 box box Objects 40 misc +848 wall borader wall board 1 408 38 7 board otherstructure Objects 40 misc +849 kitchen appliances kitchen appliance 1 40 7 otherprop Objects n03620052 kitchen_appliance.n.01 37 appliances +850 ceiling object object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +851 wall hanging decoration wall hanging decoration 1 40 7 otherprop Objects 40 misc +852 stand / small table stand/small table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +853 kitchen ceiling kitchen ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +854 floor matt floor mat 1 143 20 5 floor mat floor mat Floor n03727837 mat.n.01 2 floor +855 wall indent wall indent 1 40 7 otherprop Objects 40 misc +856 chairir chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +857 window seat window seat 1 777 38 7 window seat otherstructure Objects bench bench 02828884 n04590021 window_seat.n.01 34 seating +858 bike bicycle 1 189 40 7 bicycle otherprop Objects bicycle 02834778 n02834778 bicycle.n.01 39 objects +859 towels (rolled) towel 1 135 27 7 towel towel Objects n04459362 towel.n.01 20 towel +860 wall \other oom wall /otherroom 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +861 shower pipe shower pipe 1 664 40 7 shower pipe otherprop Objects 40 misc +862 towel or curtain bar towel/curtain bar 1 51 38 7 bar otherstructure Objects 40 misc +863 shower glass shower glass 1 612 38 7 glass otherstructure Objects 40 misc +864 stone bench stone bench 1 204 39 6 bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +865 window/other room window /otherroom 1 59 9 13 window window Window n04587648 window.n.01 9 window +866 back splash sink sink 1 24 34 7 sink sink Objects sink n04223580 sink.n.01 15 sink +867 iron board iron board 1 408 38 7 board otherstructure Objects 40 misc +868 computer equipment computer equipment 1 40 7 otherprop Objects 40 misc +869 shelf / cabinet shelf/cabinet 1 40 7 otherprop Objects 40 misc +870 stove door stove door 1 28 8 12 door door Wall door 40 misc +871 door inside door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +872 unknown stove accessory unknown stove accessory 1 40 7 otherprop Objects 40 misc +873 circular sofa circular sofa 1 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +874 dustpan dustpan 1 40 7 otherprop Objects n03259009 dustpan.n.02 39 objects +875 bathroom door door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +876 paining painting 1 64 11 8 picture picture Picture n03876519 painting.n.01 39 objects +877 luggage luggage 1 783 40 7 luggage otherprop Objects n02774630 baggage.n.01 39 objects +878 dooframe doorframe 1 615 38 7 door frame otherstructure Objects n03222722 doorframe.n.01 4 door +879 outside wall wall 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +880 alarm control alarm control 1 40 7 otherprop Objects 40 misc +881 oil lamp oil lamp 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +882 scaffolding scaffolding 1 40 7 otherprop Objects n04141712 scaffolding.n.01 39 objects +883 bed light bed light 1 62 38 7 light otherstructure Objects 40 misc +884 baluster baluster 1 453 38 7 banister otherstructure Objects n02783994 baluster.n.01 39 objects +885 leg rest leg rest 1 40 7 otherprop Objects 40 misc +886 ceiling / upstairs room ceiling /otheroom 1 40 7 otherprop Objects 40 misc +887 tv cabinet tv stand 1 291 39 6 tv stand otherfurniture Furniture tv_stand n03290653 entertainment_center.n.01 36 furniture +888 hole hole 1 40 7 otherprop Objects n09304750 hole.n.05 39 objects +889 ping pong table ping pong table 1 625 39 6 ping pong table otherfurniture Furniture table table table 04379243 n04379243 table.n.02 5 table +890 hutch hutch 1 40 7 otherprop Objects 40 misc +891 low shelf shelf 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +892 foliage foliage 1 40 7 otherprop Objects 40 misc +893 windows sill window frame 1 59 9 13 window window Window n04589593 window_frame.n.01 9 window +894 toilet cistern toilet 1 124 33 7 toilet toilet Objects toilet toilet n04446276 toilet.n.01 18 toilet +895 stone circle stone circle 1 40 7 otherprop Objects 40 misc +896 bathroom sink sink 1 24 34 7 sink sink Objects sink n04223580 sink.n.01 15 sink +897 record player record player 1 220 40 7 machine otherprop Objects n04064401 record_player.n.01 39 objects +898 table cushion table cushion 1 40 7 otherprop Objects 40 misc +899 power outlet outlet 1 40 7 otherprop Objects n04548771 wall_socket.n.01 39 objects +900 machine machine 1 220 40 7 machine otherprop Objects n03699975 machine.n.01 39 objects +901 door post doorpost 1 40 7 otherprop Objects n03222857 doorjamb.n.01 39 objects +902 briefcase briefcase 1 617 40 7 briefcase otherprop Objects n02900705 briefcase.n.01 39 objects +903 wall of doors door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +904 showe wall shower wall 1 21 1 12 wall wall Wall n04208936 shower.n.01 23 shower +905 door stand door stand 1 50 39 6 stand otherfurniture Furniture 40 misc +906 energy box energy box 1 26 29 7 box box Objects 40 misc +907 balcony floor floor 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +908 bean bag beanbag 1 797 39 6 bean bag otherfurniture Furniture n02816656 beanbag.n.01 3 chair +909 kitchen extractor kitchen extractor 1 40 7 otherprop Objects 40 misc +910 toilet bin toilet bin 1 307 40 7 bin otherprop Objects 40 misc +911 wall in other room wall /otherroom 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +912 plumbing plumbing 1 40 7 otherprop Objects n03969041 plumbing.n.01 39 objects +913 moose head / sculpture / hunting trophy moose head/sculpture/hunting trophy 1 547 40 7 trophy otherprop Objects 40 misc +914 ceiling dome ceiling dome 1 40 7 otherprop Objects 40 misc +915 cabinet counter cabinet counter 1 7 12 6 counter counter Furniture 40 misc +916 cabinet \other room cabinet /otherroom 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +917 flowerbed flowerbed 1 157 4 1 bed bed Bed bed bed n03368352 flowerbed.n.01 39 objects +918 paingitn painting 1 64 11 8 picture picture Picture n03876519 painting.n.01 39 objects +919 bed headboard headboard 1 161 39 6 headboard otherfurniture Furniture n03502200 headboard.n.01 11 bed +920 antique clock antique clock 1 56 40 7 clock otherprop Objects clock 03046257 n03046257 clock.n.01 39 objects +921 rocks rock 1 40 7 otherprop Objects n09416076 rock.n.01 39 objects +922 shower caddy for soap etc. shower caddy 1 40 7 otherprop Objects 40 misc +923 window / frame window frame 1 59 9 13 window window Window n04589593 window_frame.n.01 9 window +924 media console media console 1 40 7 otherprop Objects 40 misc +925 cloths cloth 1 40 7 otherprop Objects n03309808 fabric.n.01 39 objects +926 vaccum cleaner vacuum cleaner 1 306 40 7 vacuum cleaner otherprop Objects n04517823 vacuum.n.04 37 appliances +927 door frame fireplace wall door frame 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +928 fruit dish fruit dish 1 40 7 otherprop Objects 40 misc +929 risers for theater seating risers for theater seating 1 40 7 otherprop Objects 40 misc +930 interior bathroom wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +931 hall wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +932 drawer sink table drawer sink table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +933 keyboard piano keyboard piano 1 298 39 6 piano otherfurniture Furniture piano piano 03928116 n03928116 piano.n.01 39 objects +934 tree branch tree branch 1 40 7 otherprop Objects n13163803 limb.n.02 39 objects +935 tiled floor tiled floor 1 11 2 5 floor floor Floor 40 misc +936 ceiling bedroom ceiling bedroom 1 40 7 otherprop Objects 40 misc +937 chandellier chandelier 1 342 38 7 chandelier otherstructure Objects n03005285 chandelier.n.01 28 lighting +938 ceilin ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +939 hearst hearst 1 40 7 otherprop Objects n11037278 hearst.n.01 39 objects +940 random wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +941 condiments condiment 1 40 7 otherprop Objects 40 misc +942 shelf cubbies shelf cubby 1 40 7 otherprop Objects 40 misc +943 book display book display 1 40 7 otherprop Objects 40 misc +944 endtable end table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +945 antique telephone antique telephone 1 32 40 7 telephone otherprop Objects telephone 04401088 n04401088 telephone.n.01 39 objects +946 clothes rack clothes rack 1 40 7 otherprop Objects 40 misc +947 doore door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +948 heater piping heater piping 1 40 7 otherprop Objects 40 misc +949 tissue box stand tissue box stand 1 50 39 6 stand otherfurniture Furniture 40 misc +950 utencils utensil 1 267 40 7 utensil otherprop Objects n04516672 utensil.n.01 39 objects +951 foodtray food tray 1 179 40 7 tray otherprop Objects n04476259 tray.n.01 39 objects +952 shelves /otherroom shelves /otherroom 1 42 15 6 shelves shelves Furniture 40 misc +953 cailing ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +954 cluttered chair chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +955 countertop object object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +956 art frames art frame 1 40 7 otherprop Objects 40 misc +957 soap bottle soap bottle 1 2 40 7 bottle otherprop Objects bottle bottle 02876657 n02876657 bottle.n.01 39 objects +958 small wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +959 watch watch 1 384 40 7 watch otherprop Objects n04555897 watch.n.01 39 objects +960 ceil rail 1 497 38 7 railing otherstructure Objects n04047401 railing.n.01 30 railing +961 fuse box fuse box 1 26 29 7 box box Objects 40 misc +962 painting board painting 1 64 11 8 picture picture Picture n03876519 painting.n.01 39 objects +963 door locker handle door locker handle 1 758 40 7 handle otherprop Objects 40 misc +964 box with clutter box 1 26 29 7 box box Objects n02883344 box.n.01 39 objects +965 knive holder knife holder 1 40 7 otherprop Objects 40 misc +966 computer mouse computer mouse 1 103 40 7 mouse otherprop Objects n03793489 mouse.n.04 39 objects +967 mirror / other room mirror /otherroom 1 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +968 door back side door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +969 ceiling bath ceiling bath 1 40 7 otherprop Objects 40 misc +970 chandelier? chandelier 1 342 38 7 chandelier otherstructure Objects n03005285 chandelier.n.01 28 lighting +971 table tray table tray 1 179 40 7 tray otherprop Objects 40 misc +972 aquarium aquarium 1 263 40 7 vessel otherprop Objects n02732072 aquarium.n.01 40 misc +973 wheelbarrow wheelbarrow 1 305 40 7 cart otherprop Objects n02797295 barrow.n.03 39 objects +974 curtain rail curtain rail 1 40 7 otherprop Objects 40 misc +975 rods / table rods/table 1 40 7 otherprop Objects 40 misc +976 counter /otherroom counter /otherroom 1 7 12 6 counter counter Furniture table table table 04379243 n03116530 counter.n.01 26 counter +977 gable gable 1 21 1 12 wall wall Wall n03409393 gable.n.01 1 wall +978 bothroom wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +979 balustrade balustrade 1 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +980 bathroom bathroom 1 40 7 otherprop Objects toilet toilet 40 misc +981 three three 1 40 7 otherprop Objects 40 misc +982 handcloth hand cloth 1 40 7 otherprop Objects 40 misc +983 wall curved wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +984 teapoy coffee table 1 356 39 6 coffee table otherfurniture Furniture table table table 04379243 n03063968 coffee_table.n.01 5 table +985 table with clutter table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +986 back door door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +987 rocky ground rocky ground 1 40 7 otherprop Objects 40 misc +988 doormat doormat 2 143 20 5 floor mat floor mat Floor n03223299 doormat.n.02 2 floor +989 backrest backrest 1 5 5 4 chair chair Chair n02767433 back.n.08 39 objects +990 cabinet with desk cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +991 faucet handles faucet 1 9 40 7 faucet otherprop Objects faucet 03325088 n03325088 faucet.n.01 39 objects +992 floor trim floor trim 1 868 38 7 floor trim otherstructure Objects 40 misc +993 toilet sliding door toilet sliding door 1 28 8 12 door door Wall door 40 misc +994 gym stuff gym equipment 1 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +995 clothes container clothes container 1 140 40 7 container otherprop Objects 40 misc +996 basketball hoop basketball hoop 1 162 40 7 basketball hoop otherprop Objects n02802215 basket.n.03 33 gym_equipment +997 cooktop stovetop 1 40 7 otherprop Objects 40 misc +998 both tub bathtub 1 136 36 7 bathtub bathtub Objects bathtub bathtub tub 02808440 n02808440 bathtub.n.01 25 bathtub +999 fireplace heart fireplace 1 372 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 27 fireplace +1000 spice racks spice rack 1 241 38 7 spice rack otherstructure Objects n04275175 spice_rack.n.01 31 shelving +1001 wall fireplace wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1002 cabinet /w cluttered art cabinet /w cluttered art 1 40 7 otherprop Objects 40 misc +1003 inside wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1004 transformer transformer 1 40 7 otherprop Objects n04471315 transformer.n.01 39 objects +1005 wall light wall lamp 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +1006 bathroom shelf bathroom shelf 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1007 glass french door door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1008 lamp / other room lamp /otherroom 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +1009 picutre picture 1 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +1010 gift gift 1 40 7 otherprop Objects 40 misc +1011 mirror door mirror door 1 28 8 12 door door Wall door 40 misc +1012 deorative object decoration 1 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +1013 ceiling door ceiling door 1 28 8 12 door door Wall door 40 misc +1014 stack of papers stack of papers 1 483 40 7 papers otherprop Objects 40 misc +1015 holy cross holy cross 1 40 7 otherprop Objects 40 misc +1016 door window door window 1 59 9 13 window window Window 40 misc +1017 computer screen monitor 1 49 40 7 monitor otherprop Objects monitor monitor tv or monitor 03211117 n03782190 monitor.n.04 22 tv_monitor +1018 while bottles bottle 1 2 40 7 bottle otherprop Objects bottle bottle 02876657 n02876657 bottle.n.01 39 objects +1019 arcade game arcade game 1 40 7 otherprop Objects 40 misc +1020 unknown - probably part of trellis -- maybe blinds unknown - probably part of trellis -- maybe blinds 1 80 13 13 blinds blinds Window 40 misc +1021 compound wall compound wall 1 21 1 12 wall wall Wall 40 misc +1022 lamps lamp 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +1023 plug plug 1 40 7 otherprop Objects 40 misc +1024 round cushion round cushion 1 40 7 otherprop Objects 40 misc +1025 ceiling frame molding 1 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +1026 magazine magazine 1 71 40 7 magazine otherprop Objects n06595351 magazine.n.01 39 objects +1027 stair case staircase 1 215 38 7 stairs otherstructure Objects n04298308 stairway.n.01 16 stairs +1028 rolling pin rolling pin 1 267 40 7 utensil otherprop Objects n04103206 rolling_pin.n.01 39 objects +1029 shower knob shower knob 1 651 40 7 shower knob otherprop Objects 40 misc +1030 wall statue wall statue 1 40 7 otherprop Objects 40 misc +1031 wal wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1032 sink / basin sink/basin 1 40 7 otherprop Objects 40 misc +1033 celling borader ceiling boarder 1 40 7 otherprop Objects 40 misc +1034 clothes bag clothes bag 1 55 37 7 bag bag Objects 40 misc +1035 perfumes perfume 1 655 40 7 perfume otherprop Objects n03916031 perfume.n.02 39 objects +1036 heat heat 1 40 7 otherprop Objects n03509025 heating_system.n.01 39 objects +1037 kitchen counter support leg kitchen counter support 1 40 7 otherprop Objects 40 misc +1038 window frame window frame 1 59 9 13 window window Window n04589593 window_frame.n.01 9 window +1039 glassdoor door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1040 window closet door window closet door 1 28 8 12 door door Wall door 40 misc +1041 sticks stick 1 529 40 7 stick otherprop Objects 40 misc +1042 photo frame picture frame 1 64 11 8 picture picture Picture n03931765 picture_frame.n.01 6 picture +1043 water pump water pump 1 40 7 otherprop Objects n04561965 water_pump.n.01 39 objects +1044 shower doorframe shower door frame 1 40 7 otherprop Objects n04208936 shower.n.01 23 shower +1045 stairs behind stairs 1 215 38 7 stairs otherstructure Objects stairs n04314914 step.n.04 16 stairs +1046 flood floor 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1047 columned perimeter columned perimeter 1 40 7 otherprop Objects 40 misc +1048 shrine shrine 1 40 7 otherprop Objects n04210390 shrine.n.01 40 misc +1049 papers paper 1 15 26 7 paper paper Objects n14974264 paper.n.01 39 objects +1050 fireplace /w art fireplace 1 372 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 27 fireplace +1051 canvas stand canvas stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1052 art / man statue art/man statue 1 40 7 otherprop Objects 40 misc +1053 bath towel bath towel 1 135 27 7 towel towel Objects n02808304 bath_towel.n.01 20 towel +1054 cradenza credenza 1 7 12 6 counter counter Furniture n03129753 credenza.n.01 36 furniture +1055 doorr frame door frame 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1056 column statue column statue 1 40 7 otherprop Objects 40 misc +1057 water basin water basin 1 40 7 otherprop Objects 40 misc +1058 moulidng molding 1 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +1059 tablet computer tablet computer 1 46 40 7 computer otherprop Objects 40 misc +1060 artwork artwork 1 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +1061 towel bar shelf towel bar shelf 1 40 7 otherprop Objects 40 misc +1062 floor carpet carpet 1 130 40 7 rug floor mat Objects n04118021 rug.n.01 2 floor +1063 wine cabinet wine cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1064 shower rail shower rail 1 40 7 otherprop Objects 40 misc +1065 skirting board skirting board 1 408 38 7 board otherstructure Objects n02800354 baseboard.n.01 1 wall +1066 playpen playpen 1 815 39 6 playpen otherfurniture Furniture n03964495 playpen.n.01 40 misc +1067 room door door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1068 doorgrame doorframe 1 615 38 7 door frame otherstructure Objects n03222722 doorframe.n.01 4 door +1069 makeup makeup 1 40 7 otherprop Objects n03714235 makeup.n.01 39 objects +1070 plant / art plant/art 1 40 7 otherprop Objects 40 misc +1071 sauna seating seat 1 524 39 6 furniture otherfurniture Furniture n04161981 seat.n.03 34 seating +1072 bot bot 1 40 7 otherprop Objects n02311879 bot.n.01 39 objects +1073 handrail / other room handrail /otherroom 1 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +1074 rocking horse rocking horse 1 389 40 7 toy otherprop Objects n03523633 hobby.n.02 39 objects +1075 kitchen lower cabinet kitchen lower cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1076 floor /other room floor /otherroom 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1077 chair stand chair stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1078 wall pack wall pack 1 40 7 otherprop Objects 40 misc +1079 towel box towel box 1 26 29 7 box box Objects 40 misc +1080 vessel sink vessel sink 1 24 34 7 sink sink Objects sink 40 misc +1081 can can 1 329 40 7 can otherprop Objects can 02946921 n02946921 can.n.01 39 objects +1082 ceiling lattice ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1083 backslash backsplash 1 40 7 otherprop Objects 40 misc +1084 ire place fireplace 1 372 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 27 fireplace +1085 divider partition 1 21 1 12 wall wall Wall n03894379 partition.n.01 40 misc +1086 pathway pathway 1 40 7 otherprop Objects n03899533 pathway.n.02 40 misc +1087 laundry laundry 1 40 7 otherprop Objects n03648219 laundry.n.01 38 clothes +1088 tablecloth tablecloth 1 292 40 7 tablecloth otherprop Objects n04380143 tablecloth.n.01 39 objects +1089 water dispenser water dispenser 1 507 40 7 water dispenser otherprop Objects n03210683 dispenser.n.01 39 objects +1090 ;photo photo 1 508 40 7 photo otherprop Objects n03925226 photograph.n.01 6 picture +1091 door frame wall entry door frame 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1092 plastic tarp tarp 1 40 7 otherprop Objects n04395024 tarpaulin.n.01 39 objects +1093 clothing stand clothing stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1094 bathtab bathtub 1 136 36 7 bathtub bathtub Objects bathtub bathtub tub 02808440 n02808440 bathtub.n.01 25 bathtub +1095 apron apron 1 40 7 otherprop Objects 40 misc +1096 ceiling bedroom entry ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1097 ashtray ashtray 1 377 40 7 ashtray otherprop Objects n02747802 ashtray.n.01 39 objects +1098 control panel control panel 1 408 38 7 board otherstructure Objects n03098140 control_panel.n.01 39 objects +1099 ironing board ironing board 1 313 39 6 ironing board otherfurniture Furniture n03586090 ironing_board.n.01 39 objects +1100 exhaust pipe exhaust pipe 1 41 40 7 pipe otherprop Objects n03303510 exhaust_pipe.n.01 40 misc +1101 rack of theater electronics rack 1 50 39 6 stand otherfurniture Furniture n04038440 rack.n.05 31 shelving +1102 canvas canvas 1 559 40 7 sheet otherprop Objects 40 misc +1103 alarm clock alarm clock 1 156 40 7 alarm clock otherprop Objects clock 03046257 n02694662 alarm_clock.n.01 39 objects +1104 ceiling lower ceiling lower 1 40 7 otherprop Objects 40 misc +1105 yarn machine yarn machine 1 220 40 7 machine otherprop Objects 40 misc +1106 tarrace door terrace door 1 28 8 12 door door Wall door 40 misc +1107 stand/table stand/table 1 40 7 otherprop Objects 40 misc +1108 statue base statue 1 294 40 7 sculpture otherprop Objects n04306847 statue.n.01 39 objects +1109 swing swing 1 389 40 7 toy otherprop Objects n04371774 swing.n.02 39 objects +1110 extractor fan extractor fan 1 74 40 7 fan otherprop Objects 40 misc +1111 crib crib 1 485 39 6 crib otherfurniture Furniture 40 misc +1112 hallway wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1113 side wall 2 wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1114 side wall 5 wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1115 mens urinal urinal 1 40 7 otherprop Objects n04515991 urinal.n.01 39 objects +1116 stacked chairs stacked chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1117 patio floor patio floor 1 11 2 5 floor floor Floor 40 misc +1118 stair steps stair step 1 40 7 otherprop Objects 40 misc +1119 electrical controller electrical controller 1 40 7 otherprop Objects 40 misc +1120 kitchencounter kitchen counter 1 7 12 6 counter counter Furniture table table table 04379243 n03116530 counter.n.01 26 counter +1121 rotunda railing rotunda railing 1 497 38 7 railing otherstructure Objects 40 misc +1122 lamp table lamp table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +1123 cabinet top cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1124 tray with bottles of soap and lotion tray 1 179 40 7 tray otherprop Objects n04476259 tray.n.01 39 objects +1125 speakers speaker 1 54 40 7 speaker otherprop Objects speaker 03691459 n03691459 loudspeaker.n.01 39 objects +1126 fancy cabinet cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1127 air vent air vent 1 25 38 7 air vent otherstructure Objects n04526241 vent.n.01 40 misc +1128 sofa couch couch 1 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +1129 channel channel 1 40 7 otherprop Objects 40 misc +1130 wall sign wall sign 1 208 40 7 sign otherprop Objects 40 misc +1131 bench back bench 1 204 39 6 bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +1132 baby changing station baby changing station 1 40 7 otherprop Objects 40 misc +1133 paip pip 1 286 40 7 fruit otherprop Objects n11685091 pip.n.03 39 objects +1134 bed arm pillows pillow 1 119 18 7 pillow pillow Objects pillow 03938244 n03938244 pillow.n.01 8 cushion +1135 shower tap shower tap 1 132 40 7 shower cap otherprop Objects 40 misc +1136 axe axe 1 40 7 otherprop Objects n02764044 ax.n.01 39 objects +1137 table shelf shelf 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1138 washcloth washcloth 1 40 7 otherprop Objects n04554523 washcloth.n.01 39 objects +1139 coffee mug coffee mug 1 263 40 7 vessel otherprop Objects cup or mug 03797390 n03063599 coffee_mug.n.01 39 objects +1140 iron panel panel 1 559 40 7 sheet otherprop Objects n03882058 panel.n.01 35 board_panel +1141 door or window frame door/window frame 1 40 7 otherprop Objects 40 misc +1142 coffe table coffee table 1 356 39 6 coffee table otherfurniture Furniture table table table 04379243 n03063968 coffee_table.n.01 5 table +1143 dice dice 1 40 7 otherprop Objects n03191029 die.n.01 39 objects +1144 title title 1 40 7 otherprop Objects 40 misc +1145 toilet seat toilet 1 124 33 7 toilet toilet Objects toilet toilet n04446276 toilet.n.01 18 toilet +1146 coffee table leg coffee table 1 356 39 6 coffee table otherfurniture Furniture table table table 04379243 n03063968 coffee_table.n.01 5 table +1147 soft chair soft chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1148 icebox icebox 1 17 24 6 refridgerator refridgerator Furniture n04070727 refrigerator.n.01 37 appliances +1149 showerfloor shower floor 1 11 2 5 floor floor Floor n04208936 shower.n.01 23 shower +1150 floor outside floor /outside 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1151 framing framing 1 40 7 otherprop Objects n03390983 frame.n.10 40 misc +1152 wall patch wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1153 decorative bowl decorative bowl 1 826 40 7 decorative bowl otherprop Objects bowl bowl 02880940 n02880940 bowl.n.03 39 objects +1154 dog bed dog bed 1 858 39 6 dog bed otherfurniture Furniture bed bed bed 02818832 n02818832 bed.n.01 11 bed +1155 pool sticks pool stick 1 529 40 7 stick otherprop Objects n03145522 cue.n.04 39 objects +1156 exericse equipment exercise equipment 1 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +1157 remove floor behind floor /otherroom 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1158 doorhandle door handle 1 758 40 7 handle otherprop Objects 40 misc +1159 dishrag dishrag 1 40 7 otherprop Objects n03207743 dishrag.n.01 39 objects +1160 behind behind 1 40 7 otherprop Objects 40 misc +1161 drinking fountain drinking fountain 1 40 7 otherprop Objects n03241335 drinking_fountain.n.01 40 misc +1162 bureau bureau 1 524 39 6 furniture otherfurniture Furniture dresser dresser n03015254 chest_of_drawers.n.01 13 chest_of_drawers +1163 night stand object object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1164 tub bathtub 1 136 36 7 bathtub bathtub Objects bathtub bathtub tub 02808440 n02808440 bathtub.n.01 25 bathtub +1165 parapet parapet 1 40 7 otherprop Objects 40 misc +1166 attic door attic door 1 28 8 12 door door Wall door 40 misc +1167 wall object object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1168 closet door knob closet door knob 1 652 40 7 knob otherprop Objects 40 misc +1169 bathroom accessory bathroom accessory 1 40 7 otherprop Objects n02671780 accessory.n.01 38 clothes +1170 teddy teddy bear 1 389 40 7 toy otherprop Objects n04399382 teddy.n.01 39 objects +1171 wall toilet paper wall toilet paper 1 15 26 7 paper paper Objects 40 misc +1172 coffee machine coffee machine 1 234 40 7 coffee machine otherprop Objects n03063338 coffee_maker.n.01 37 appliances +1173 excercise equipment / other room exercise equipment /otherroom 1 457 39 6 excercise equipment otherfurniture Furniture 40 misc +1174 storage cabinet storage cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1175 toilet stall door toilet stall door 1 28 8 12 door door Wall door 40 misc +1176 bin of posters bin 1 307 40 7 bin otherprop Objects n02839910 bin.n.01 39 objects +1177 photos picture 1 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +1178 push broom broom 1 328 40 7 broom otherprop Objects n02906734 broom.n.01 39 objects +1179 art stand art stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1180 shelf /w art shelf /w art 1 40 7 otherprop Objects 40 misc +1181 spiderman spiderman statue 1 40 7 otherprop Objects 40 misc +1182 mount mount 1 40 7 otherprop Objects 40 misc +1183 ac unit air conditioning 1 40 7 otherprop Objects n02686379 air_conditioner.n.01 39 objects +1184 newspaper newspaper 1 873 40 7 newspapers otherprop Objects 40 misc +1185 towel basket towel basket 1 39 40 7 basket otherprop Objects basket 02801938 n02801938 basket.n.01 39 objects +1186 base rail base rail 1 40 7 otherprop Objects 40 misc +1187 show pack show pack 1 40 7 otherprop Objects 40 misc +1188 coutertop countertop 1 7 12 6 counter counter Furniture n03118245 countertop.n.01 26 counter +1189 stair stepper stair stepper 1 40 7 otherprop Objects 40 misc +1190 car batteries car battery 1 40 7 otherprop Objects n02961225 car_battery.n.01 39 objects +1191 pad pad 1 40 7 otherprop Objects n03195485 diggings.n.02 40 misc +1192 old fireplace fireplace 1 372 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 27 fireplace +1193 desk lamp desk lamp 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +1194 easychair easy chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03262932 easy_chair.n.01 3 chair +1195 wees table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +1196 tabletop box tabletop box 1 26 29 7 box box Objects 40 misc +1197 canopy canopy 1 40 7 otherprop Objects 40 misc +1198 wall behind door wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1199 cabinet / other room cabinet /otherroom 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1200 objects / other room objects /otherroom 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1201 bar soap bar soap 1 133 40 7 soap otherprop Objects 40 misc +1202 column /otherroom column /otherroom 1 94 38 7 column otherstructure Objects n03074380 column.n.06 40 misc +1203 outer wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1204 wall control wall control 1 40 7 otherprop Objects 40 misc +1205 desktop computer tower 1 46 40 7 computer otherprop Objects n03082979 computer.n.01 39 objects +1206 fire screen fire screen 1 40 7 otherprop Objects n03347037 fire_screen.n.01 39 objects +1207 fireplace counter mantel 1 58 38 7 mantel otherstructure Objects mantel n03719343 mantel.n.01 27 fireplace +1208 bread bread 1 246 40 7 bread otherprop Objects n07679356 bread.n.01 40 misc +1209 piano bench piano bench 1 460 39 6 piano bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +1210 window shutter window shutter 1 40 7 otherprop Objects n04211356 shutter.n.02 39 objects +1211 draw draw 1 40 7 otherprop Objects n09269882 draw.n.01 39 objects +1212 flower wage flower vase 1 78 40 7 vase otherprop Objects vase jar 03593526 n04522168 vase.n.01 39 objects +1213 ceiling/west wall ceiling/west wall 1 21 1 12 wall wall Wall 40 misc +1214 decorative quilt decorative quilt 1 575 40 7 quilt otherprop Objects 40 misc +1215 knife knife 1 259 40 7 knife otherprop Objects knife 03624134 n03624134 knife.n.02 39 objects +1216 projector opening projector opening 1 40 7 otherprop Objects 40 misc +1217 boader boarder 1 40 7 otherprop Objects 40 misc +1218 lights / deco lights/deco 1 40 7 otherprop Objects 40 misc +1219 tv3 wall tv 1 40 7 otherprop Objects 40 misc +1220 desk clutter desk clutter 1 40 7 otherprop Objects 40 misc +1221 curatain curtain 1 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +1222 shoe shelves shelving 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1223 mixer mixer 1 40 7 otherprop Objects 40 misc +1224 ceiling fixture ceiling fixture 1 40 7 otherprop Objects 40 misc +1225 scuplture sculpture 1 294 40 7 sculpture otherprop Objects n04157320 sculpture.n.01 39 objects +1226 countertop /otherroom countertop /otherroom 1 7 12 6 counter counter Furniture n03118245 countertop.n.01 26 counter +1227 work bench workbench 1 204 39 6 bench otherfurniture Furniture bench table 04379243 n04600486 workbench.n.01 5 table +1228 wall desk wall desk 1 36 14 10 desk desk Table desk desk 40 misc +1229 shelf of cloth shelf 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1230 telescope telescope 1 467 40 7 telescope otherprop Objects n04403638 telescope.n.01 39 objects +1231 shower hoses shower hose 1 669 40 7 shower hose otherprop Objects 40 misc +1232 step; step 1 38 7 otherstructure Objects n04314914 step.n.04 16 stairs +1233 bathtub platform bathtub platform 1 40 7 otherprop Objects 40 misc +1234 sauna seats seat 1 524 39 6 furniture otherfurniture Furniture n04161981 seat.n.03 34 seating +1235 bucker bucket 1 427 40 7 bucket otherprop Objects n02909870 bucket.n.01 39 objects +1236 ac air conditioner 1 79 38 7 air conditioner otherstructure Objects n02686379 air_conditioner.n.01 39 objects +1237 unknown / remove unknown/remove 1 40 7 otherprop Objects 40 misc +1238 art / statue art/statue 1 40 7 otherprop Objects 40 misc +1239 dinner table dinner table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +1240 storage storage 1 n03744276 memory.n.04 39 objects +1241 unknown kitchen stuff unknown kitchen stuff 1 40 7 otherprop Objects 40 misc +1242 alamari wardrobe 1 772 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 36 furniture +1243 cement drum cement drum 1 145 40 7 drum otherprop Objects 40 misc +1244 kitchen cupboard kitchen cabinet 1 3 3 6 cabinet cabinet Furniture n02933112 cabinet.n.01 7 cabinet +1245 platter platter 1 129 40 7 platter otherprop Objects 40 misc +1246 large chunk of art chunk of art 1 40 7 otherprop Objects 40 misc +1247 dushbin dustbin 1 307 40 7 bin otherprop Objects trash_bin 02747177 n02747177 ashcan.n.01 39 objects +1248 couch pillows pillow 1 119 18 7 pillow pillow Objects pillow 03938244 n03938244 pillow.n.01 8 cushion +1249 ceiling /other room ceiling /otherroom 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1250 object on cutting board object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1251 show wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1252 robes robe 1 40 7 otherprop Objects n04097866 robe.n.01 38 clothes +1253 cusion cushion 1 119 18 7 pillow pillow Objects n03151500 cushion.n.03 8 cushion +1254 column /outside column /outside 1 94 38 7 column otherstructure Objects n03074380 column.n.06 40 misc +1255 hanging clothes hanging clothes 1 141 21 7 clothes clothes Objects n02728440 apparel.n.01 38 clothes +1256 newspaper basket newspaper basket 1 39 40 7 basket otherprop Objects basket 02801938 n02801938 basket.n.01 39 objects +1257 towels in a bowl towel 1 135 27 7 towel towel Objects n04459362 towel.n.01 20 towel +1258 detached door door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1259 big door door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1260 wall molding molding 1 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +1261 bed / other room bed /otherroom 1 157 4 1 bed bed Bed bed bed bed 02818832 n02818832 bed.n.01 11 bed +1262 window /w pictures window /w pictures 1 40 7 otherprop Objects 40 misc +1263 ceiling corridor ceiling corridor 1 40 7 otherprop Objects 40 misc +1264 rotunda base rotunda 1 40 7 otherprop Objects 40 misc +1265 fire extinguisher' fire extinguisher 1 10 40 7 fire extinguisher otherprop Objects n03345837 fire_extinguisher.n.01 39 objects +1266 unicycle unicycle 1 40 7 otherprop Objects n04509417 unicycle.n.01 39 objects +1267 ceiling under balcony ceiling /otherroom 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1268 laundary machines laundry machine 1 220 40 7 machine otherprop Objects 40 misc +1269 nightstand /reflection nightstand /reflection 1 158 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 13 chest_of_drawers +1270 pile of magazines pile of magazines 1 40 7 otherprop Objects 40 misc +1271 wall of statue inset wall of statue inset 1 40 7 otherprop Objects 40 misc +1272 cosmetic cosmetic 1 40 7 otherprop Objects n03113152 cosmetic.n.01 39 objects +1273 unknown item object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1274 side steps 4 step 1 38 7 otherstructure Objects n04314914 step.n.04 16 stairs +1275 kitchen table kitchen table 1 19 7 10 table table Table table table table 04379243 n03620967 kitchen_table.n.01 5 table +1276 round footstool round footstool 1 40 7 otherprop Objects 40 misc +1277 firewood holder firewood holder 1 40 7 otherprop Objects 40 misc +1278 flag flag 1 405 40 7 flag otherprop Objects 40 misc +1279 window frame and shelves window frame and shelves 1 42 15 6 shelves shelves Furniture 40 misc +1280 towel ring towel ring 1 40 7 otherprop Objects n04460038 towel_ring.n.01 39 objects +1281 window frame /reflection window frame /reflection 1 477 38 7 window frame otherstructure Objects n04589593 window_frame.n.01 9 window +1282 basket of something basket of something 1 40 7 otherprop Objects 40 misc +1283 towel rod towel rod 1 134 38 7 towel rod otherstructure Objects 40 misc +1284 window /reflection window /reflection 1 59 9 13 window window Window n04587648 window.n.01 9 window +1285 paper storage paper storage 1 40 7 otherprop Objects 40 misc +1286 t.v tv 1 172 25 11 television television TV tv or monitor 03211117 n03211117 display.n.06 22 tv_monitor +1287 kitchen bell kitchen bell 1 40 7 otherprop Objects 40 misc +1288 ceiling from another room ceiling /otherroom 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1289 throw blanket throw blanket 1 312 40 7 blanket otherprop Objects 40 misc +1290 wall post wall post 1 40 7 otherprop Objects 40 misc +1291 sink pipe sink pipe 1 41 40 7 pipe otherprop Objects 40 misc +1292 workstation workstation 1 46 40 7 computer otherprop Objects n04603399 workstation.n.01 39 objects +1293 room 1 wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1294 storage space storage space 1 645 38 7 storage space otherstructure Objects n04328946 storage_space.n.01 40 misc +1295 mattress bed 1 157 4 1 bed bed Bed bed bed bed 02818832 n02818832 bed.n.01 11 bed +1296 kitchen handle kitchen handle 1 758 40 7 handle otherprop Objects n03485997 handle.n.01 39 objects +1297 bed side table nightstand 1 158 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 13 chest_of_drawers +1298 lookout lookout 1 40 7 otherprop Objects n03688943 lookout.n.03 40 misc +1299 bag of sand bag of sand 1 40 7 otherprop Objects 40 misc +1300 walll wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1301 kitchen countertop object' object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1302 bed rest headboard 1 161 39 6 headboard otherfurniture Furniture n03502200 headboard.n.01 11 bed +1303 furniture 2 furniture 1 524 39 6 furniture otherfurniture Furniture n03405725 furniture.n.01 36 furniture +1304 wall ac vent vent 1 25 38 7 air vent otherstructure Objects n04526241 vent.n.01 40 misc +1305 dumbbells dumbbell 1 40 7 otherprop Objects n03255030 dumbbell.n.01 33 gym_equipment +1306 weight weight 1 40 7 otherprop Objects n04571292 weight.n.02 33 gym_equipment +1307 fixture fixture 1 40 7 otherprop Objects n03354613 fixture.n.01 39 objects +1308 llight light 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +1309 show space show space 1 40 7 otherprop Objects 40 misc +1310 recessed cubby recessed cubby 1 40 7 otherprop Objects 40 misc +1311 oven hood range hood 1 380 38 7 range hood otherstructure Objects range_hood n04053677 range_hood.n.01 39 objects +1312 counter doors counter door 1 28 8 12 door door Wall door 40 misc +1313 roof grill roof grill 1 700 38 7 grill otherstructure Objects 40 misc +1314 shower dial shower dial 1 40 7 otherprop Objects 40 misc +1315 riser riser 1 40 7 otherprop Objects 40 misc +1316 plant ornament plant ornament 1 40 7 otherprop Objects 40 misc +1317 floor stand floor stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1318 fruit fruit 1 286 40 7 fruit otherprop Objects n13134947 fruit.n.01 39 objects +1319 frige refrigerator 1 17 24 6 refridgerator refridgerator Furniture n04070727 refrigerator.n.01 37 appliances +1320 stairwell stairwell 1 40 7 otherprop Objects n04298661 stairwell.n.01 16 stairs +1321 trash bag trash bag 1 55 37 7 bag bag Objects 40 misc +1322 cups cup 1 35 40 7 cup otherprop Objects cup cup or mug 03797390 n03797390 mug.n.04 39 objects +1323 photo mount photo mount 1 40 7 otherprop Objects 40 misc +1324 drawers for clothes drawers for clothes 1 141 21 7 clothes clothes Objects 40 misc +1325 strange ceiling ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1326 bilow billow 1 40 7 otherprop Objects 40 misc +1327 ceilng ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1328 bathroom towel bathroom towel 1 135 27 7 towel towel Objects n04459362 towel.n.01 20 towel +1329 john john 1 40 7 otherprop Objects toilet toilet n04446276 toilet.n.01 18 toilet +1330 refrigerator cabinet refrigerator cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1331 fish picture fish picture 1 64 11 8 picture picture Picture 40 misc +1332 painting frame painting frame 1 40 7 otherprop Objects 40 misc +1333 display table display table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +1334 round seat round seat 1 40 7 otherprop Objects 40 misc +1335 lower floor floor 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1336 lid lid 1 533 40 7 lid otherprop Objects 40 misc +1337 bonsai tree bonsai tree 1 40 7 otherprop Objects 40 misc +1338 window3 window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1339 window2 window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1340 window1 window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1341 window5 window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1342 shower floor surround shower floor 1 11 2 5 floor floor Floor n04208936 shower.n.01 23 shower +1343 folders folder 1 69 40 7 folder otherprop Objects n03376279 folder.n.02 39 objects +1344 decorative plant decorative plant 1 82 40 7 plant otherprop Objects plant 40 misc +1345 cooker unit cooker unit 1 40 7 otherprop Objects 40 misc +1346 towel holder towel holder 1 40 7 otherprop Objects 40 misc +1347 laundrybag laundry bag 1 55 37 7 bag bag Objects 40 misc +1348 bathroom utencil bathroom utensil 1 267 40 7 utensil otherprop Objects n04516672 utensil.n.01 39 objects +1349 wine bottles wine bottle 1 333 40 7 wine bottle otherprop Objects bottle wine bottle 04591713 n04591713 wine_bottle.n.01 39 objects +1350 high shelf high shelf 1 40 7 otherprop Objects 40 misc +1351 loveseat couch 1 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +1352 couch pillow pillow 1 119 18 7 pillow pillow Objects pillow 03938244 n03938244 pillow.n.01 8 cushion +1353 ceiling molding ceiling molding 1 40 7 otherprop Objects 40 misc +1354 firewook firewood 1 40 7 otherprop Objects n15100644 firewood.n.01 40 misc +1355 door /otherroom door /otherroom 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1356 toaster? toaster 1 251 40 7 toaster otherprop Objects n04442312 toaster.n.02 37 appliances +1357 foodstand food stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1358 weight bench weight bench 1 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +1359 floor / room above floor /otherroom 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1360 mini fridge mini fridge 1 17 24 6 refridgerator refridgerator Furniture n03273913 electric_refrigerator.n.01 37 appliances +1361 bars bar 1 51 38 7 bar otherstructure Objects n02788689 bar.n.03 39 objects +1362 lion lion 1 594 40 7 cat otherprop Objects n02129165 lion.n.01 39 objects +1363 cuddly toy cuddly toy 1 389 40 7 toy otherprop Objects 40 misc +1364 toilet stall toilet stall 1 40 7 otherprop Objects 40 misc +1365 copier machine copier machine 1 220 40 7 machine otherprop Objects 40 misc +1366 raised platform platform 1 38 7 otherstructure Objects 40 misc +1367 bell bell 1 40 7 otherprop Objects 40 misc +1368 fireextinctioms fire extinguisher 1 10 40 7 fire extinguisher otherprop Objects n03345837 fire_extinguisher.n.01 39 objects +1369 table vase table vase 1 78 40 7 vase otherprop Objects vase 40 misc +1370 bedroom ceiling bedroom ceiling 1 4 22 3 ceiling ceiling Ceiling 40 misc +1371 ceiling \other room ceiling /otherroom 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1372 chair parts chair part 1 40 7 otherprop Objects 40 misc +1373 door / window door/window 1 40 7 otherprop Objects 40 misc +1374 closet mirror wall closet mirror wall 1 21 1 12 wall wall Wall 40 misc +1375 beam across top of arch beam across top of arch 1 40 7 otherprop Objects 40 misc +1376 fireplace utensils fireplace utensil 1 267 40 7 utensil otherprop Objects 40 misc +1377 liight light 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +1378 self side self side 1 40 7 otherprop Objects 40 misc +1379 kitchen faucet faucet 1 9 40 7 faucet otherprop Objects faucet 03325088 n03325088 faucet.n.01 39 objects +1380 candle holder candle holder 1 148 40 7 candlestick otherprop Objects n02948557 candlestick.n.01 39 objects +1381 mirror/other room mirror /otherroom 1 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +1382 hall top hall top 1 40 7 otherprop Objects 40 misc +1383 decoration / other room decoration /otherroom 1 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +1384 shower hose shower hose 1 669 40 7 shower hose otherprop Objects 40 misc +1385 bottle of detergent bottle of detergent 1 40 7 otherprop Objects 40 misc +1386 box opening box opening 1 40 7 otherprop Objects 40 misc +1387 hunting trohpy hunting trophy 1 547 40 7 trophy otherprop Objects 40 misc +1388 rack with pool cues rack 1 50 39 6 stand otherfurniture Furniture n04038440 rack.n.05 31 shelving +1389 stack stack 1 40 7 otherprop Objects 40 misc +1390 bed stand bed stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1391 planter curb planter curb 1 40 7 otherprop Objects 40 misc +1392 garage door motor garage door motor 1 40 7 otherprop Objects 40 misc +1393 entry entry 1 40 7 otherprop Objects n03290771 entrance.n.01 40 misc +1394 a-frame sign a-frame sign 1 208 40 7 sign otherprop Objects 40 misc +1395 cuboid cuboid 1 40 7 otherprop Objects 40 misc +1396 shelf with jars shelf 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1397 wall top wall top 1 40 7 otherprop Objects 40 misc +1398 window4 window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1399 bath faucet bath faucet 1 9 40 7 faucet otherprop Objects faucet 03325088 n03325088 faucet.n.01 39 objects +1400 statue / art statue/art 1 40 7 otherprop Objects 40 misc +1401 cabinet /w clutter cabinet /w clutter 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1402 workout bike workout bike 1 40 7 otherprop Objects 40 misc +1403 closet area for hanging clothes closet area for hanging clothes 1 141 21 7 clothes clothes Objects 40 misc +1404 vase \other room vase /otherroom 1 78 40 7 vase otherprop Objects vase jar 03593526 n04522168 vase.n.01 39 objects +1405 art / muscle shell art/muscle shell 1 40 7 otherprop Objects 40 misc +1406 center island island 1 456 38 7 kitchen island otherstructure Objects n03620600 kitchen_island.n.01 26 counter +1407 dedore decor 1 40 7 otherprop Objects n03579355 interior_decoration.n.01 39 objects +1408 dome roof dome roof 1 40 7 otherprop Objects 40 misc +1409 chair /w clutter chair /w clutter 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1410 basket of objects (popcorn?) basket 1 39 40 7 basket otherprop Objects basket 02801938 n02801938 basket.n.01 39 objects +1411 outer side wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1412 top floor floor 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1413 pelt pelt 1 40 7 otherprop Objects n01895735 hide.n.02 39 objects +1414 washer-dryer washer-dryer 1 40 7 otherprop Objects 40 misc +1415 yellow egg shaped vase yellow egg shaped vase 1 78 40 7 vase otherprop Objects vase 40 misc +1416 entry table table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +1417 domino decoration domino decoration 1 40 7 otherprop Objects 40 misc +1418 pc tower pc tower 1 40 7 otherprop Objects tower 04460130 n04460130 tower.n.01 40 misc +1419 unknown/ other room unknown /otherroom 1 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +1420 bathroom glass bathroom glass 1 612 38 7 glass otherstructure Objects n03438257 glass.n.02 39 objects +1421 chanellier chandelier 1 342 38 7 chandelier otherstructure Objects n03005285 chandelier.n.01 28 lighting +1422 makeup area makeup area 1 40 7 otherprop Objects 40 misc +1423 fireplace mirror fireplace mirror 1 122 19 7 mirror mirror Objects 40 misc +1424 jug jug 1 687 40 7 jug otherprop Objects bottle bottle 02876657 n03603722 jug.n.01 39 objects +1425 bathroom window bathroom window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1426 falling light falling light 1 62 38 7 light otherstructure Objects 40 misc +1427 snow globe snow globe 1 347 40 7 globe otherprop Objects 40 misc +1428 unknown kitchen appliance unknown kitchen appliance 1 40 7 otherprop Objects 40 misc +1429 boxes /w books boxes /w books 1 85 23 2 books books Books 40 misc +1430 alarm alarm 1 525 40 7 alarm otherprop Objects clock 03046257 n02694662 alarm_clock.n.01 39 objects +1431 electirc outlet electric outlet 1 98 40 7 electrical outlet otherprop Objects n04548771 wall_socket.n.01 39 objects +1432 sauna floor floor 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1433 picture /otherroom picture /otherroom 1 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +1434 garage door railing garage door railing 1 497 38 7 railing otherstructure Objects 40 misc +1435 other ceiling ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1436 lightbox frame light box frame 1 40 7 otherprop Objects 40 misc +1437 air hole vent 1 25 38 7 air vent otherstructure Objects n04526241 vent.n.01 40 misc +1438 mantel mantel 1 58 38 7 mantel otherstructure Objects mantel n03719343 mantel.n.01 27 fireplace +1439 cooker cooker 4 267 40 7 utensil otherprop Objects n03101156 cooker.n.01 39 objects +1440 partial partial 1 40 7 otherprop Objects 40 misc +1441 amplifier amplifier 1 40 7 otherprop Objects n02705944 amplifier.n.01 39 objects +1442 emergency exit emergency exit 1 40 7 otherprop Objects n03345658 fire_escape.n.01 16 stairs +1443 barbecue barbecue 1 40 7 otherprop Objects n02790669 barbecue.n.03 40 misc +1444 unkown / other room unknown /otherroom 1 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +1445 bath sink bath sink 1 24 34 7 sink sink Objects sink 40 misc +1446 spa bench spa bench 1 204 39 6 bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +1447 handsoap hand soap 1 133 40 7 soap otherprop Objects n04253437 soap.n.01 39 objects +1448 folding stand folding stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1449 table plant table plant 1 82 40 7 plant otherprop Objects plant 40 misc +1450 hose outlet hose outlet 1 40 7 otherprop Objects 40 misc +1451 theater stage theater stage 1 40 7 otherprop Objects n04418818 theater_stage.n.01 39 objects +1452 bar cabinet bar cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1453 washer and dryer washing machine and dryer 1 40 7 otherprop Objects 37 appliances +1454 door trim molding molding 1 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +1455 paintings picture 1 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +1456 unknown wall part unknown wall 1 21 1 12 wall wall Wall 40 misc +1457 wooden balcony balcony 1 40 7 otherprop Objects 40 misc +1458 shelves with shoes shelving 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1459 stair landing landing 1 40 7 otherprop Objects n03638511 landing.n.01 2 floor +1460 rack of excercise weights rack 1 50 39 6 stand otherfurniture Furniture n04038440 rack.n.05 31 shelving +1461 unknown - hot tub? unknown - hot tub? 1 40 7 otherprop Objects 40 misc +1462 wall window window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1463 ceiling smoke detector smoke detector 1 40 7 otherprop Objects 40 misc +1464 couch type chair sofa chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1465 scales scale 1 639 40 7 scale otherprop Objects n04141975 scale.n.07 39 objects +1466 chair rail chair rail 1 40 7 otherprop Objects 40 misc +1467 kitchen backsplash backsplash 1 40 7 otherprop Objects 40 misc +1468 shelves / other room shelves /otherroom 1 42 15 6 shelves shelves Furniture 40 misc +1469 tools tool 1 40 7 otherprop Objects n04451818 tool.n.01 39 objects +1470 hoses for chemical tank hoses for chemical tank 1 40 7 otherprop Objects 40 misc +1471 wall behind / remove wall /otherroom 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1472 sauna heat rocks sauna heat rocks 1 40 7 otherprop Objects 40 misc +1473 camera camera 1 40 40 7 camera otherprop Objects camera 02942699 n02942699 camera.n.01 39 objects +1474 curtain lighter curtain lighter 1 40 7 otherprop Objects 40 misc +1475 table decor decoration 1 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +1476 brochure brochure 1 69 40 7 folder otherprop Objects n06413889 booklet.n.01 39 objects +1477 flowerwage flower vase 1 78 40 7 vase otherprop Objects vase jar 03593526 n04522168 vase.n.01 39 objects +1478 drill drill 1 40 7 otherprop Objects 40 misc +1479 book stand bookshelf 1 88 10 6 bookshelf bookshelf Furniture bookshelf bookshelf 02871439 n02871439 bookshelf.n.01 31 shelving +1480 china cabinet cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1481 remove / behind remove 1 0 0 0 void void void 0 void +1482 rotunda wall rotunda wall 1 21 1 12 wall wall Wall 40 misc +1483 stone support structure stone support structure 1 40 7 otherprop Objects 40 misc +1484 dresser decor decoration 1 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +1485 roof / other room ceiling /otherroom 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1486 elliptical elliptical 1 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +1487 furnace furnace 1 551 39 6 furnace otherfurniture Furniture n03404449 furnace.n.01 40 misc +1488 foosball table foosball table 1 510 39 6 foosball table otherfurniture Furniture table table table 04379243 n04379243 table.n.02 5 table +1489 maracas maraca 1 40 7 otherprop Objects n03720891 maraca.n.01 39 objects +1490 box of fruit box of fruit 1 286 40 7 fruit otherprop Objects 40 misc +1491 window screen window screen 1 40 7 otherprop Objects n04589890 window_screen.n.01 9 window +1492 door tag door tag 1 218 40 7 tag otherprop Objects 40 misc +1493 fire pit fire pit 1 40 7 otherprop Objects n09280113 fire_pit.n.01 39 objects +1494 box of tissues box of tissues 1 40 7 otherprop Objects 40 misc +1495 portal portal 1 46 40 7 computer otherprop Objects n06359657 portal_site.n.01 39 objects +1496 yoga mat yoga mat 1 205 40 7 yoga mat otherprop Objects 40 misc +1497 toy duck toy duck 1 887 40 7 duck otherprop Objects 40 misc +1498 motorcycle motorcycle 1 40 7 otherprop Objects motorcycle 03790512 n03790512 motorcycle.n.01 39 objects +1499 computer tower computer tower 1 46 40 7 computer otherprop Objects n03082979 computer.n.01 39 objects +1500 frame for door frame for door 1 28 8 12 door door Wall door 40 misc +1501 theater screen theater screen 1 40 7 otherprop Objects 40 misc +1502 swimming pool swimming pool 1 40 7 otherprop Objects n04371225 swimming_pool.n.01 40 misc +1503 shelf with clutter shelf 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1504 spice rack spice rack 1 241 38 7 spice rack otherstructure Objects n04275175 spice_rack.n.01 31 shelving +1505 bed small bed small 1 40 7 otherprop Objects 40 misc +1506 object/other room object /otherroom 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1507 dinnerware dinnerware 1 40 7 otherprop Objects n03202622 dinnerware.n.01 39 objects +1508 storage shelves shelving 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1509 tanning bed tanning bed 1 157 4 1 bed bed Bed bed bed bed 02818832 n02818832 bed.n.01 11 bed +1510 grid door door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1511 background chair chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1512 boarder boarder 1 40 7 otherprop Objects 40 misc +1513 closte closet 1 40 7 otherprop Objects n03148324 cupboard.n.01 7 cabinet +1514 doorframe / upstairs room doorframe /otherroom 1 615 38 7 door frame otherstructure Objects n03222722 doorframe.n.01 4 door +1515 rolling cart rolling cart 1 305 40 7 cart otherprop Objects 40 misc +1516 box of tissue box of tissue 1 648 40 7 tissue otherprop Objects 40 misc +1517 door nob door knob 1 27 40 7 door knob otherprop Objects 40 misc +1518 floor \other room floor /otherroom 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1519 table w/ clutter table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +1520 table vase soil table vase soil 1 40 7 otherprop Objects 40 misc +1521 light box light box 1 26 29 7 box box Objects 40 misc +1522 wines wine 1 766 40 7 wine otherprop Objects 40 misc +1523 sliding glass door sliding glass door 1 28 8 12 door door Wall door 40 misc +1524 toilet handle toilet handle 1 758 40 7 handle otherprop Objects 40 misc +1525 bath deco bath deco 1 40 7 otherprop Objects 40 misc +1526 shower hose/head shower hose/head 1 40 7 otherprop Objects 40 misc +1527 shower case shower case 1 851 40 7 case otherprop Objects 40 misc +1528 mortar mortar 1 40 7 otherprop Objects 40 misc +1529 watering can watering can 1 329 40 7 can otherprop Objects can 02946921 n02946921 can.n.01 39 objects +1530 scarf scarf 1 240 40 7 scarf otherprop Objects n04143897 scarf.n.01 38 clothes +1531 softer softer 1 40 7 otherprop Objects 40 misc +1532 soap dish cubby soap dish cubby 1 40 7 otherprop Objects 40 misc +1533 paintintg picture 1 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +1534 cleaning clutter cleaning clutter 1 40 7 otherprop Objects 40 misc +1535 fireplace tool set fireplace tool set 1 40 7 otherprop Objects 40 misc +1536 table stand table stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1537 shower valve shower valve 1 40 7 otherprop Objects 40 misc +1538 long sofa couch 1 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +1539 wall soap shelf wall soap shelf 1 40 7 otherprop Objects 40 misc +1540 ceiling inset for fan ceiling inset for fan 1 74 40 7 fan otherprop Objects 40 misc +1541 beanbag chair beanbag chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1542 door bottom rail door bottom rail 1 40 7 otherprop Objects 40 misc +1543 sowing machine sowing machine 1 220 40 7 machine otherprop Objects 40 misc +1544 couch chair sofa chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1545 tabletop trinket tabletop trinket 1 844 40 7 trinket otherprop Objects 40 misc +1546 ceilling ceiling 3 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1547 flowerstand flower stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1548 wall stand wall stand 1 295 38 7 wall stand otherstructure Objects 40 misc +1549 shower cabin shower cabin 1 40 7 otherprop Objects 40 misc +1550 shower-bath cabinet shower-bath cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1551 teddy bear stuffed animal 1 177 40 7 stuffed animal otherprop Objects n04399382 teddy.n.01 39 objects +1552 foor lamp floor lamp 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03367059 floor_lamp.n.01 28 lighting +1553 water fountain water fountain 1 339 38 7 water fountain otherstructure Objects n03241335 drinking_fountain.n.01 40 misc +1554 elephant sculpture elephant sculpture 1 294 40 7 sculpture otherprop Objects 40 misc +1555 grate grate 1 40 7 otherprop Objects 40 misc +1556 chicken chicken 1 40 7 otherprop Objects n01791625 chicken.n.02 39 objects +1557 bathroom fan bathroom fan 1 74 40 7 fan otherprop Objects n03320046 fan.n.01 39 objects +1558 roomba roomba 1 40 7 otherprop Objects 40 misc +1559 planter pot 1 16 40 7 pot otherprop Objects n03991062 pot.n.04 39 objects +1560 controls control 1 40 7 otherprop Objects n03096960 control.n.09 39 objects +1561 nightstand / other room nightstand /otherroom 1 158 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 13 chest_of_drawers +1562 worktop worktop 1 40 7 otherprop Objects 40 misc +1563 steps wall steps wall 1 21 1 12 wall wall Wall 40 misc +1564 door fireplace wall door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1565 floor / upstairs room floor /otherroom 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1566 oven and stove oven and stove 1 242 38 7 stove otherstructure Objects stove 04330267 40 misc +1567 chest drawer chest drawer 1 174 39 6 drawer otherfurniture Furniture 40 misc +1568 door outside door /outside 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1569 fireplace \other room fireplace /otherroom 1 372 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 27 fireplace +1570 hammock hammock 1 157 4 1 bed bed Bed bed bed bed 02818832 n03482252 hammock.n.02 11 bed +1571 dartboard dartboard 1 408 38 7 board otherstructure Objects n03162940 dartboard.n.01 39 objects +1572 fireplace brush fireplace brush 1 40 7 otherprop Objects 40 misc +1573 drum drum 1 145 40 7 drum otherprop Objects n03249569 drum.n.01 39 objects +1574 rotunda ceiling ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1575 ceiling light fixture connection ceiling light fixture connection 1 40 7 otherprop Objects 40 misc +1576 stair fencing banister 1 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +1577 piano stool piano stool 1 150 40 7 stool otherprop Objects stool n03801880 music_stool.n.01 19 stool +1578 swing door swing door 1 28 8 12 door door Wall door n04371979 swing_door.n.01 4 door +1579 bathtub utencils bathtub utensil 1 267 40 7 utensil otherprop Objects 40 misc +1580 access area access area 1 40 7 otherprop Objects 40 misc +1581 exercise mat roll exercise mat roll 1 40 7 otherprop Objects 40 misc +1582 ceiling duct ceiling duct 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1583 motorbike motorbike 1 40 7 otherprop Objects motorcycle 03790512 n03769722 minibike.n.01 39 objects +1584 floor elevator floor elevator 1 40 7 otherprop Objects 40 misc +1585 kitchen countertop items kitchen countertop items 1 40 7 otherprop Objects 40 misc +1586 entertainment set entertainment set 1 40 7 otherprop Objects 40 misc +1587 shower curtain bar shower curtain bar 1 51 38 7 bar otherstructure Objects 40 misc +1588 bedroom entry walls wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1589 oven vent oven vent 1 40 7 otherprop Objects 40 misc +1590 shower floor /otherroom shower floor /otherroom 1 11 2 5 floor floor Floor n04208936 shower.n.01 23 shower +1591 washing bowl sink 1 24 34 7 sink sink Objects sink n04223580 sink.n.01 15 sink +1592 desert plate desert plate 1 233 40 7 plate otherprop Objects 40 misc +1593 fuse panel fuse panel 1 40 7 otherprop Objects 40 misc +1594 barbers chair barbers chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1595 diploma diploma 1 40 7 otherprop Objects 40 misc +1596 wall behind stove wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1597 window6 window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1598 ceiling /otherroom ceiling /otherroom 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1599 ventilation hood ventilation hood 1 40 7 otherprop Objects 40 misc +1600 dirt ground dirt ground 1 40 7 otherprop Objects 40 misc +1601 awning awning 1 40 7 otherprop Objects n02763901 awning.n.01 39 objects +1602 stiarcase step step 1 38 7 otherstructure Objects n04314914 step.n.04 16 stairs +1603 rack of weights rack of weights 1 40 7 otherprop Objects 40 misc +1604 shower tub shower tub 1 675 40 7 shower tube otherprop Objects bathtub bathtub tub 02808440 n02808440 bathtub.n.01 25 bathtub +1605 washing powder washing powder 1 40 7 otherprop Objects 40 misc +1606 toilet cleaner toilet cleaner 1 548 40 7 cleaner otherprop Objects 40 misc +1607 soap displensor shelf in shower soap dispenser shelf in shower 1 40 7 otherprop Objects 40 misc +1608 window or door window/door 1 40 7 otherprop Objects 40 misc +1609 storage boxes storage box 1 26 29 7 box box Objects 40 misc +1610 closet storage area closet storage area 1 40 7 otherprop Objects 40 misc +1611 stationary bike exercise bike 1 457 39 6 excercise equipment otherfurniture Furniture n03302671 exercise_bike.n.01 33 gym_equipment +1612 bedding bedding 1 40 7 otherprop Objects n02820210 bedclothes.n.01 39 objects +1613 stair handrail banister 1 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +1614 room divider partition 1 21 1 12 wall wall Wall n03894379 partition.n.01 40 misc +1615 window frames window frame 1 59 9 13 window window Window n04589593 window_frame.n.01 9 window +1616 wall electrics wall electronics 1 40 7 otherprop Objects 40 misc +1617 other step step 1 38 7 otherstructure Objects n04314914 step.n.04 16 stairs +1618 tabletop objects object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1619 foosball game table foosball game table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +1620 wall showcase wall showcase 1 40 7 otherprop Objects 40 misc +1621 kitchen utencil kitchen utensil 1 267 40 7 utensil otherprop Objects n03621049 kitchen_utensil.n.01 39 objects +1622 towels in a basket towels in a basket 1 39 40 7 basket otherprop Objects basket 02801938 n02801938 basket.n.01 39 objects +1623 lamp stand lamp stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1624 toy giraffe toy giraffe 1 40 7 otherprop Objects 40 misc +1625 big door frame door frame 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1626 unknown / probably shrubbery unknown/probably shrubbery 1 40 7 otherprop Objects 40 misc +1627 wooden wall paneling paneling 1 21 1 12 wall wall Wall n03882611 paneling.n.01 1 wall +1628 bedstead bedstead 1 157 4 1 bed bed Bed n02822579 bedstead.n.01 11 bed +1629 fire dish fire dish 1 40 7 otherprop Objects 40 misc +1630 tray with tea cups tray 1 179 40 7 tray otherprop Objects n04476259 tray.n.01 39 objects +1631 television table tv stand 1 291 39 6 tv stand otherfurniture Furniture tv_stand n03290653 entertainment_center.n.01 36 furniture +1632 tissues tissue 1 648 40 7 tissue otherprop Objects 40 misc +1633 sitting area sitting area 1 40 7 otherprop Objects 40 misc +1634 wall vent vent 1 25 38 7 air vent otherstructure Objects n04526241 vent.n.01 40 misc +1635 utensils utensil 1 267 40 7 utensil otherprop Objects n04516672 utensil.n.01 39 objects +1636 pip pip 1 286 40 7 fruit otherprop Objects n11685091 pip.n.03 39 objects +1637 fireplace wall fireplace wall 1 21 1 12 wall wall Wall 40 misc +1638 stonework stonework 1 40 7 otherprop Objects n04326799 stonework.n.01 40 misc +1639 bottom of stairs bottom of stairs 1 215 38 7 stairs otherstructure Objects stairs 40 misc +1640 star star 1 40 7 otherprop Objects n09444783 star.n.03 39 objects +1641 art / deer statue statue 1 294 40 7 sculpture otherprop Objects n04306847 statue.n.01 39 objects +1642 unknown objects object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1643 wall side wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1644 shower glass /shifted shower glass /shifted 1 612 38 7 glass otherstructure Objects 40 misc +1645 toilet stall partition partition 1 21 1 12 wall wall Wall n03894379 partition.n.01 40 misc +1646 ceiling design 1 40 7 otherprop Objects 40 misc +1647 conference table conference table 1 19 7 10 table table Table table table table 04379243 n03090000 conference_table.n.01 5 table +1648 edge edge 1 40 7 otherprop Objects 40 misc +1649 electrical box electrical box 1 26 29 7 box box Objects n03034244 circuit_breaker.n.01 39 objects +1650 entrance entrance 1 40 7 otherprop Objects n03290771 entrance.n.01 40 misc +1651 gap gap 2 40 7 otherprop Objects n09249034 col.n.01 39 objects +1652 garage door opener motor garage door opener motor 1 40 7 otherprop Objects 40 misc +1653 lamp base lamp 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +1654 racks rack 2 50 39 6 stand otherfurniture Furniture n04038440 rack.n.05 31 shelving +1655 stringer stringer 1 40 7 otherprop Objects 40 misc +1656 supporting structure supporting structure 1 40 7 otherprop Objects n04361095 supporting_structure.n.01 40 misc +1657 typewriter typewriter 1 376 40 7 typewriter otherprop Objects printer 04004475 n04505036 typewriter.n.01 39 objects +1658 walls wall 4 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1659 washbasin top washbasin 1 24 34 7 sink sink Objects sink n04553920 washbasin.n.01 15 sink diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/meta_data/scenes_test.txt b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/meta_data/scenes_test.txt new file mode 100644 index 0000000000000000000000000000000000000000..e378f66e2ed767778f9e62826da0ddac058c1682 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/meta_data/scenes_test.txt @@ -0,0 +1,18 @@ +2t7WUuJeko7 +5ZKStnWn8Zo +ARNzJeq3xxb +fzynW3qQPVF +jtcxE69GiFV +pa4otMbVnkk +q9vSo1VnCiC +rqfALeAoiTq +UwV83HsGsw3 +wc2JMjhGNzB +WYY7iVyf5p8 +YFuZgdQ5vWj +yqstnuAEVhm +YVUC4YcDtcY +gxdoqLR6rwA +gYvKGZ5eRqb +RPmz2sHmrrY +Vt2qJdWjCF2 \ No newline at end of file diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/meta_data/scenes_train.txt b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/meta_data/scenes_train.txt new file mode 100644 index 0000000000000000000000000000000000000000..64afe8d80d754176a0b6c68791da55b16ea36c56 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/meta_data/scenes_train.txt @@ -0,0 +1,61 @@ +17DRP5sb8fy +1LXtFkjw3qL +1pXnuDYAj8r +29hnd4uzFmX +5LpN3gDmAk7 +5q7pvUzZiYa +759xd9YjKW5 +7y3sRwLe3Va +82sE5b5pLXE +8WUmhLawc2A +aayBHfsNo7d +ac26ZMwG7aT +B6ByNegPMKs +b8cTxDM8gDG +cV4RVeZvu5T +D7N2EKCX4Sj +e9zR4mvMWw7 +EDJbREhghzL +GdvgFV5R1Z5 +gTV8FGcVJC9 +HxpKQynjfin +i5noydFURQK +JeFG25nYj2p +JF19kD82Mey +jh4fc5c5qoQ +kEZ7cmS4wCh +mJXqzFtmKg4 +p5wJjkQkbXX +Pm6F8kyY3z2 +pRbA3pwrgk9 +PuKPg4mmafe +PX4nDJXEHrG +qoiz87JEwZ2 +rPc6DW4iMge +s8pcmisQ38h +S9hNv5qa7GM +sKLMLpTHeUy +SN83YJsR3w2 +sT4fr6TAbpF +ULsKaCPVFJR +uNb9QFRL6hY +Uxmj2M2itWa +V2XKFyX4ASd +VFuaQ6m2Qom +VVfe2KiqLaN +Vvot9Ly1tCj +vyrNrziPKCB +VzqfbhrpDEA +XcA2TqTSSAj +2n8kARJN3HM +D7G3Y4RVNrH +dhjEzFoUFzH +E9uDoFAP3SH +gZ6f7yhEvPG +JmbYfDe2QKZ +r1Q1Z4BcV1o +r47D5H71a5s +ur6pFq6Qu1A +VLzqgDo317F +YmJkqBEsHnH +ZMojNkEp431 \ No newline at end of file diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/meta_data/scenes_val.txt b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/meta_data/scenes_val.txt new file mode 100644 index 0000000000000000000000000000000000000000..bcec005da85cc9f481208f9730a23da9bb9acc96 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/meta_data/scenes_val.txt @@ -0,0 +1,11 @@ +2azQ1b91cZZ +8194nk5LbLH +EU6Fwq7SyZv +oLBMNvg9in8 +QUCTc6BB5sX +TbHJrupSAjP +X7HyMhZNoso +pLe4wQe7qrG +x8F5xyUWy9e +Z6MFQCViBuw +zsNo4HB9uLZ \ No newline at end of file diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/preprocess_matterport3d_mesh.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/preprocess_matterport3d_mesh.py new file mode 100644 index 0000000000000000000000000000000000000000..82d2850b73602924e6ce93d23976a35bbe8a2ae7 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/preprocess_matterport3d_mesh.py @@ -0,0 +1,240 @@ +""" +Preprocessing Script for Matterport3D (Unzipping) +adatpted from https://github.com/pengsongyou/openscene/blob/main/scripts/preprocess/preprocess_3d_matterport.py + +Author: Chongjie Ye (chongjieye@link.cuhk.edu.cn) +Modified by: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import argparse +import glob +import plyfile +import numpy as np +import pandas as pd +import multiprocessing as mp +from concurrent.futures import ProcessPoolExecutor +from itertools import repeat +from pathlib import Path +import torch + +MATTERPORT_CLASS_REMAP = np.zeros(41) +MATTERPORT_CLASS_REMAP[1] = 1 +MATTERPORT_CLASS_REMAP[2] = 2 +MATTERPORT_CLASS_REMAP[3] = 3 +MATTERPORT_CLASS_REMAP[4] = 4 +MATTERPORT_CLASS_REMAP[5] = 5 +MATTERPORT_CLASS_REMAP[6] = 6 +MATTERPORT_CLASS_REMAP[7] = 7 +MATTERPORT_CLASS_REMAP[8] = 8 +MATTERPORT_CLASS_REMAP[9] = 9 +MATTERPORT_CLASS_REMAP[10] = 10 +MATTERPORT_CLASS_REMAP[11] = 11 +MATTERPORT_CLASS_REMAP[12] = 12 +MATTERPORT_CLASS_REMAP[14] = 13 +MATTERPORT_CLASS_REMAP[16] = 14 +MATTERPORT_CLASS_REMAP[22] = 21 # DIFFERENCE TO SCANNET! +MATTERPORT_CLASS_REMAP[24] = 15 +MATTERPORT_CLASS_REMAP[28] = 16 +MATTERPORT_CLASS_REMAP[33] = 17 +MATTERPORT_CLASS_REMAP[34] = 18 +MATTERPORT_CLASS_REMAP[36] = 19 +MATTERPORT_CLASS_REMAP[39] = 20 + +MATTERPORT_LABELS_21 = ( + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refrigerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "other", + "ceiling", +) +MATTERPORT_ALLOWED_NYU_CLASSES = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 14, + 16, + 22, + 24, + 28, + 33, + 34, + 36, + 39, +] + + +def handle_process(mesh_path, output_path, mapping, train_scenes, val_scenes): + # Get the scene id and region name from the mesh path + scene_id = Path(mesh_path).parent.parent.name + region_id = Path(mesh_path).stem.removeprefix("region") + data_name = f"{scene_id}_{int(region_id):02d}" + + output_path = Path(output_path) + # Check which split the scene belongs to (train, val, or test) + if scene_id in train_scenes: + output_folder = output_path / "train" / data_name + split = "train" + elif scene_id in val_scenes: + output_folder = output_path / "val" / data_name + split = "val" + else: + output_folder = output_path / "test" / data_name + split = "test" + + # Create the output directory if it doesn't exist + os.makedirs(output_folder, exist_ok=True) + print(f"Processing: {data_name} in {split}") + + # Load the vertex data + with open(mesh_path, "rb") as f: + plydata = plyfile.PlyData.read(f) + vertex_data = plydata["vertex"].data + + # Get the coordinates, colors, and normals from the vertex data + coords = np.vstack([vertex_data["x"], vertex_data["y"], vertex_data["z"]]).T + colors = np.vstack( + [vertex_data["red"], vertex_data["green"], vertex_data["blue"]] + ).T + normals = np.vstack([vertex_data["nx"], vertex_data["ny"], vertex_data["nz"]]).T + + # Load the face data + face_data = plydata["face"].data + category_id = face_data["category_id"] + + # Replace -1 with 0 in category_id + category_id[category_id == -1] = 0 + + # Map the labels according to NYU40ID + mapped_labels = mapping[category_id] + + # Replace labels not in MATTERPORT_ALLOWED_NYU_CLASSES with 0 + mapped_labels[ + np.logical_not(np.isin(mapped_labels, MATTERPORT_ALLOWED_NYU_CLASSES)) + ] = 0 + + # Remap the labels to ScanNet 20 categories + ceiling + remapped_labels = MATTERPORT_CLASS_REMAP[mapped_labels].astype(int) + + # Calculate per-vertex labels + triangles = face_data["vertex_indices"] + vertex_labels = np.zeros((coords.shape[0], 22), dtype=np.int32) + # calculate per-vertex labels + for row_id in range(triangles.shape[0]): + for i in range(3): + vertex_labels[triangles[row_id][i], remapped_labels[row_id]] += 1 + + # Get the most frequent label for each vertex + vertex_labels = np.argmax(vertex_labels, axis=1) + vertex_labels -= 1 + + # Add the vertex labels to the data to be saved + # Prepare the data to be saved + data_dict = dict( + coord=coords.astype("float32"), + color=colors.astype("uint8"), + normal=normals.astype("float32"), + segment=vertex_labels.astype("int16"), + ) + + # Save processed data + for key in data_dict.keys(): + np.save(output_folder / f"{key}.npy", data_dict[key]) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset_root", + required=True, + help="Path to the Matterport3D dataset containing scene folders", + ) + parser.add_argument( + "--output_root", + required=True, + help="Output path where train/val folders will be located", + ) + parser.add_argument( + "--num_workers", + default=mp.cpu_count(), + type=int, + help="Num workers for preprocessing.", + ) + opt = parser.parse_args() + meta_root = Path(os.path.dirname(__file__)) / "meta_data" + + # Load label map + category_mapping = pd.read_csv( + meta_root / "category_mapping.tsv", + sep="\t", + header=0, + ) + mapping = np.insert( + category_mapping[["nyu40id"]].to_numpy().astype(int).flatten(), 0, 0, axis=0 + ) + + # Load train/val splits + with open(meta_root / "scenes_train.txt") as train_file: + train_scenes = train_file.read().splitlines() + with open(meta_root / "scenes_val.txt") as val_file: + val_scenes = val_file.read().splitlines() + with open(meta_root / "scenes_test.txt") as test_file: + test_scenes = test_file.read().splitlines() + + # Create output directories + os.makedirs(opt.output_root, exist_ok=True) + train_output_dir = os.path.join(opt.output_root, "train") + os.makedirs(train_output_dir, exist_ok=True) + val_output_dir = os.path.join(opt.output_root, "val") + os.makedirs(val_output_dir, exist_ok=True) + test_output_dir = os.path.join(opt.output_root, "test") + os.makedirs(test_output_dir, exist_ok=True) + + # Load scene paths + scene_paths = sorted( + glob.glob( + os.path.join( + opt.dataset_root, "v1", "scans", "*", "region_segmentations", "*.ply" + ) + ) + ) + + # Preprocess data. + pool = ProcessPoolExecutor(max_workers=opt.num_workers) + print("Processing scenes...") + _ = list( + pool.map( + handle_process, + scene_paths, + repeat(opt.output_root), + repeat(mapping), + repeat(train_scenes), + repeat(val_scenes), + ) + ) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/unzip_matterport3d_region_segmentation.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/unzip_matterport3d_region_segmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..eee79908262e09aeb428a5e176cf799e8647a01a --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/matterport3d/unzip_matterport3d_region_segmentation.py @@ -0,0 +1,66 @@ +""" +Preprocessing Script for Matterport3D (Unzipping) +modified from official preprocess code. + +Author: Chongjie Ye (chongjieye@link.cuhk.edu.cn) +Modified by: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import argparse +import os +import zipfile +import glob +import multiprocessing as mp +from concurrent.futures import ProcessPoolExecutor +from itertools import repeat + + +def unzip_file(input_path, output_path): + print(f"Unzipping {input_path} ...") + os.makedirs(os.path.dirname(output_path), exist_ok=True) + with zipfile.ZipFile(input_path, "r") as zip_ref: + zip_ref.extractall(output_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='Unzip all "region_segmentations.zip" files in a directory' + ) + parser.add_argument( + "--dataset_root", + type=str, + help="Path to input directory containing ZIP files", + required=True, + ) + parser.add_argument( + "--output_root", + type=str, + help="Path to output directory for extracted files", + default=None, + ) + parser.add_argument( + "--num_workers", + default=mp.cpu_count(), + type=int, + help="Num workers for preprocessing.", + ) + args = parser.parse_args() + if args.output_root is None: + args.output_root = args.dataset_root + args.output_root = os.path.join(args.output_root, "v1", "scans") + + file_list = glob.glob( + os.path.join(args.dataset_root, "v1", "scans", "*", "region_segmentations.zip") + ) + + # Preprocess data. + print("Unzipping region_segmentations.zip in Matterport3D...") + pool = ProcessPoolExecutor(max_workers=args.num_workers) + _ = list( + pool.map( + unzip_file, + file_list, + repeat(args.output_root), + ) + ) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/nuscenes/preprocess_nuscenes_info.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/nuscenes/preprocess_nuscenes_info.py new file mode 100644 index 0000000000000000000000000000000000000000..7ed106f193a488aa76385157aa33fb65e5944a6f --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/nuscenes/preprocess_nuscenes_info.py @@ -0,0 +1,607 @@ +""" +Preprocessing Script for nuScenes Informantion +modified from OpenPCDet (https://github.com/open-mmlab/OpenPCDet) + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +from pathlib import Path +import numpy as np +import argparse +import tqdm +import pickle +from functools import reduce +from pyquaternion import Quaternion +from nuscenes.nuscenes import NuScenes +from nuscenes.utils import splits +from nuscenes.utils.geometry_utils import transform_matrix + + +map_name_from_general_to_detection = { + "human.pedestrian.adult": "pedestrian", + "human.pedestrian.child": "pedestrian", + "human.pedestrian.wheelchair": "ignore", + "human.pedestrian.stroller": "ignore", + "human.pedestrian.personal_mobility": "ignore", + "human.pedestrian.police_officer": "pedestrian", + "human.pedestrian.construction_worker": "pedestrian", + "animal": "ignore", + "vehicle.car": "car", + "vehicle.motorcycle": "motorcycle", + "vehicle.bicycle": "bicycle", + "vehicle.bus.bendy": "bus", + "vehicle.bus.rigid": "bus", + "vehicle.truck": "truck", + "vehicle.construction": "construction_vehicle", + "vehicle.emergency.ambulance": "ignore", + "vehicle.emergency.police": "ignore", + "vehicle.trailer": "trailer", + "movable_object.barrier": "barrier", + "movable_object.trafficcone": "traffic_cone", + "movable_object.pushable_pullable": "ignore", + "movable_object.debris": "ignore", + "static_object.bicycle_rack": "ignore", +} + + +cls_attr_dist = { + "barrier": { + "cycle.with_rider": 0, + "cycle.without_rider": 0, + "pedestrian.moving": 0, + "pedestrian.sitting_lying_down": 0, + "pedestrian.standing": 0, + "vehicle.moving": 0, + "vehicle.parked": 0, + "vehicle.stopped": 0, + }, + "bicycle": { + "cycle.with_rider": 2791, + "cycle.without_rider": 8946, + "pedestrian.moving": 0, + "pedestrian.sitting_lying_down": 0, + "pedestrian.standing": 0, + "vehicle.moving": 0, + "vehicle.parked": 0, + "vehicle.stopped": 0, + }, + "bus": { + "cycle.with_rider": 0, + "cycle.without_rider": 0, + "pedestrian.moving": 0, + "pedestrian.sitting_lying_down": 0, + "pedestrian.standing": 0, + "vehicle.moving": 9092, + "vehicle.parked": 3294, + "vehicle.stopped": 3881, + }, + "car": { + "cycle.with_rider": 0, + "cycle.without_rider": 0, + "pedestrian.moving": 0, + "pedestrian.sitting_lying_down": 0, + "pedestrian.standing": 0, + "vehicle.moving": 114304, + "vehicle.parked": 330133, + "vehicle.stopped": 46898, + }, + "construction_vehicle": { + "cycle.with_rider": 0, + "cycle.without_rider": 0, + "pedestrian.moving": 0, + "pedestrian.sitting_lying_down": 0, + "pedestrian.standing": 0, + "vehicle.moving": 882, + "vehicle.parked": 11549, + "vehicle.stopped": 2102, + }, + "ignore": { + "cycle.with_rider": 307, + "cycle.without_rider": 73, + "pedestrian.moving": 0, + "pedestrian.sitting_lying_down": 0, + "pedestrian.standing": 0, + "vehicle.moving": 165, + "vehicle.parked": 400, + "vehicle.stopped": 102, + }, + "motorcycle": { + "cycle.with_rider": 4233, + "cycle.without_rider": 8326, + "pedestrian.moving": 0, + "pedestrian.sitting_lying_down": 0, + "pedestrian.standing": 0, + "vehicle.moving": 0, + "vehicle.parked": 0, + "vehicle.stopped": 0, + }, + "pedestrian": { + "cycle.with_rider": 0, + "cycle.without_rider": 0, + "pedestrian.moving": 157444, + "pedestrian.sitting_lying_down": 13939, + "pedestrian.standing": 46530, + "vehicle.moving": 0, + "vehicle.parked": 0, + "vehicle.stopped": 0, + }, + "traffic_cone": { + "cycle.with_rider": 0, + "cycle.without_rider": 0, + "pedestrian.moving": 0, + "pedestrian.sitting_lying_down": 0, + "pedestrian.standing": 0, + "vehicle.moving": 0, + "vehicle.parked": 0, + "vehicle.stopped": 0, + }, + "trailer": { + "cycle.with_rider": 0, + "cycle.without_rider": 0, + "pedestrian.moving": 0, + "pedestrian.sitting_lying_down": 0, + "pedestrian.standing": 0, + "vehicle.moving": 3421, + "vehicle.parked": 19224, + "vehicle.stopped": 1895, + }, + "truck": { + "cycle.with_rider": 0, + "cycle.without_rider": 0, + "pedestrian.moving": 0, + "pedestrian.sitting_lying_down": 0, + "pedestrian.standing": 0, + "vehicle.moving": 21339, + "vehicle.parked": 55626, + "vehicle.stopped": 11097, + }, +} + + +def get_available_scenes(nusc): + available_scenes = [] + for scene in nusc.scene: + scene_token = scene["token"] + scene_rec = nusc.get("scene", scene_token) + sample_rec = nusc.get("sample", scene_rec["first_sample_token"]) + sd_rec = nusc.get("sample_data", sample_rec["data"]["LIDAR_TOP"]) + has_more_frames = True + scene_not_exist = False + while has_more_frames: + lidar_path, boxes, _ = nusc.get_sample_data(sd_rec["token"]) + if not Path(lidar_path).exists(): + scene_not_exist = True + break + else: + break + if scene_not_exist: + continue + available_scenes.append(scene) + return available_scenes + + +def get_sample_data(nusc, sample_data_token, selected_anntokens=None): + """ + Returns the data path as well as all annotations related to that sample_data. + Note that the boxes are transformed into the current sensor"s coordinate frame. + Args: + nusc: + sample_data_token: Sample_data token. + selected_anntokens: If provided only return the selected annotation. + + Returns: + + """ + # Retrieve sensor & pose records + sd_record = nusc.get("sample_data", sample_data_token) + cs_record = nusc.get("calibrated_sensor", sd_record["calibrated_sensor_token"]) + sensor_record = nusc.get("sensor", cs_record["sensor_token"]) + pose_record = nusc.get("ego_pose", sd_record["ego_pose_token"]) + + data_path = nusc.get_sample_data_path(sample_data_token) + + if sensor_record["modality"] == "camera": + cam_intrinsic = np.array(cs_record["camera_intrinsic"]) + else: + cam_intrinsic = None + + # Retrieve all sample annotations and map to sensor coordinate system. + if selected_anntokens is not None: + boxes = list(map(nusc.get_box, selected_anntokens)) + else: + boxes = nusc.get_boxes(sample_data_token) + + # Make list of Box objects including coord system transforms. + box_list = [] + for box in boxes: + box.velocity = nusc.box_velocity(box.token) + # Move box to ego vehicle coord system + box.translate(-np.array(pose_record["translation"])) + box.rotate(Quaternion(pose_record["rotation"]).inverse) + + # Move box to sensor coord system + box.translate(-np.array(cs_record["translation"])) + box.rotate(Quaternion(cs_record["rotation"]).inverse) + + box_list.append(box) + + return data_path, box_list, cam_intrinsic + + +def quaternion_yaw(q: Quaternion) -> float: + """ + Calculate the yaw angle from a quaternion. + Note that this only works for a quaternion that represents a box in lidar or global coordinate frame. + It does not work for a box in the camera frame. + :param q: Quaternion of interest. + :return: Yaw angle in radians. + """ + + # Project into xy plane. + v = np.dot(q.rotation_matrix, np.array([1, 0, 0])) + + # Measure yaw using arctan. + yaw = np.arctan2(v[1], v[0]) + + return yaw + + +def obtain_sensor2top( + nusc, sensor_token, l2e_t, l2e_r_mat, e2g_t, e2g_r_mat, sensor_type="lidar" +): + """Obtain the info with RT matric from general sensor to Top LiDAR. + + Args: + nusc (class): Dataset class in the nuScenes dataset. + sensor_token (str): Sample data token corresponding to the + specific sensor type. + l2e_t (np.ndarray): Translation from lidar to ego in shape (1, 3). + l2e_r_mat (np.ndarray): Rotation matrix from lidar to ego + in shape (3, 3). + e2g_t (np.ndarray): Translation from ego to global in shape (1, 3). + e2g_r_mat (np.ndarray): Rotation matrix from ego to global + in shape (3, 3). + sensor_type (str): Sensor to calibrate. Default: "lidar". + + Returns: + sweep (dict): Sweep information after transformation. + """ + sd_rec = nusc.get("sample_data", sensor_token) + cs_record = nusc.get("calibrated_sensor", sd_rec["calibrated_sensor_token"]) + pose_record = nusc.get("ego_pose", sd_rec["ego_pose_token"]) + data_path = str(nusc.get_sample_data_path(sd_rec["token"])) + # if os.getcwd() in data_path: # path from lyftdataset is absolute path + # data_path = data_path.split(f"{os.getcwd()}/")[-1] # relative path + sweep = { + "data_path": data_path, + "type": sensor_type, + "sample_data_token": sd_rec["token"], + "sensor2ego_translation": cs_record["translation"], + "sensor2ego_rotation": cs_record["rotation"], + "ego2global_translation": pose_record["translation"], + "ego2global_rotation": pose_record["rotation"], + "timestamp": sd_rec["timestamp"], + } + l2e_r_s = sweep["sensor2ego_rotation"] + l2e_t_s = sweep["sensor2ego_translation"] + e2g_r_s = sweep["ego2global_rotation"] + e2g_t_s = sweep["ego2global_translation"] + + # obtain the RT from sensor to Top LiDAR + # sweep->ego->global->ego'->lidar + l2e_r_s_mat = Quaternion(l2e_r_s).rotation_matrix + e2g_r_s_mat = Quaternion(e2g_r_s).rotation_matrix + R = (l2e_r_s_mat.T @ e2g_r_s_mat.T) @ ( + np.linalg.inv(e2g_r_mat).T @ np.linalg.inv(l2e_r_mat).T + ) + T = (l2e_t_s @ e2g_r_s_mat.T + e2g_t_s) @ ( + np.linalg.inv(e2g_r_mat).T @ np.linalg.inv(l2e_r_mat).T + ) + T -= ( + e2g_t @ (np.linalg.inv(e2g_r_mat).T @ np.linalg.inv(l2e_r_mat).T) + + l2e_t @ np.linalg.inv(l2e_r_mat).T + ).squeeze(0) + sweep["sensor2lidar_rotation"] = R.T # points @ R.T + T + sweep["sensor2lidar_translation"] = T + return sweep + + +def fill_trainval_infos( + data_path, nusc, train_scenes, test=False, max_sweeps=10, with_camera=False +): + train_nusc_infos = [] + val_nusc_infos = [] + progress_bar = tqdm.tqdm( + total=len(nusc.sample), desc="create_info", dynamic_ncols=True + ) + + ref_chan = "LIDAR_TOP" # The radar channel from which we track back n sweeps to aggregate the point cloud. + chan = "LIDAR_TOP" # The reference channel of the current sample_rec that the point clouds are mapped to. + + for index, sample in enumerate(nusc.sample): + progress_bar.update() + + ref_sd_token = sample["data"][ref_chan] + ref_sd_rec = nusc.get("sample_data", ref_sd_token) + ref_cs_rec = nusc.get( + "calibrated_sensor", ref_sd_rec["calibrated_sensor_token"] + ) + ref_pose_rec = nusc.get("ego_pose", ref_sd_rec["ego_pose_token"]) + ref_time = 1e-6 * ref_sd_rec["timestamp"] + + ref_lidar_path, ref_boxes, _ = get_sample_data(nusc, ref_sd_token) + + ref_cam_front_token = sample["data"]["CAM_FRONT"] + ref_cam_path, _, ref_cam_intrinsic = nusc.get_sample_data(ref_cam_front_token) + + # Homogeneous transform from ego car frame to reference frame + ref_from_car = transform_matrix( + ref_cs_rec["translation"], Quaternion(ref_cs_rec["rotation"]), inverse=True + ) + + # Homogeneous transformation matrix from global to _current_ ego car frame + car_from_global = transform_matrix( + ref_pose_rec["translation"], + Quaternion(ref_pose_rec["rotation"]), + inverse=True, + ) + info = { + "lidar_path": Path(ref_lidar_path).relative_to(data_path).__str__(), + "lidar_token": ref_sd_token, + "cam_front_path": Path(ref_cam_path).relative_to(data_path).__str__(), + "cam_intrinsic": ref_cam_intrinsic, + "token": sample["token"], + "sweeps": [], + "ref_from_car": ref_from_car, + "car_from_global": car_from_global, + "timestamp": ref_time, + } + if with_camera: + info["cams"] = dict() + l2e_r = ref_cs_rec["rotation"] + l2e_t = (ref_cs_rec["translation"],) + e2g_r = ref_pose_rec["rotation"] + e2g_t = ref_pose_rec["translation"] + l2e_r_mat = Quaternion(l2e_r).rotation_matrix + e2g_r_mat = Quaternion(e2g_r).rotation_matrix + + # obtain 6 image's information per frame + camera_types = [ + "CAM_FRONT", + "CAM_FRONT_RIGHT", + "CAM_FRONT_LEFT", + "CAM_BACK", + "CAM_BACK_LEFT", + "CAM_BACK_RIGHT", + ] + for cam in camera_types: + cam_token = sample["data"][cam] + cam_path, _, camera_intrinsics = nusc.get_sample_data(cam_token) + cam_info = obtain_sensor2top( + nusc, cam_token, l2e_t, l2e_r_mat, e2g_t, e2g_r_mat, cam + ) + cam_info["data_path"] = ( + Path(cam_info["data_path"]).relative_to(data_path).__str__() + ) + cam_info.update(camera_intrinsics=camera_intrinsics) + info["cams"].update({cam: cam_info}) + + sample_data_token = sample["data"][chan] + curr_sd_rec = nusc.get("sample_data", sample_data_token) + sweeps = [] + while len(sweeps) < max_sweeps - 1: + if curr_sd_rec["prev"] == "": + if len(sweeps) == 0: + sweep = { + "lidar_path": Path(ref_lidar_path) + .relative_to(data_path) + .__str__(), + "sample_data_token": curr_sd_rec["token"], + "transform_matrix": None, + "time_lag": curr_sd_rec["timestamp"] * 0, + } + sweeps.append(sweep) + else: + sweeps.append(sweeps[-1]) + else: + curr_sd_rec = nusc.get("sample_data", curr_sd_rec["prev"]) + + # Get past pose + current_pose_rec = nusc.get("ego_pose", curr_sd_rec["ego_pose_token"]) + global_from_car = transform_matrix( + current_pose_rec["translation"], + Quaternion(current_pose_rec["rotation"]), + inverse=False, + ) + + # Homogeneous transformation matrix from sensor coordinate frame to ego car frame. + current_cs_rec = nusc.get( + "calibrated_sensor", curr_sd_rec["calibrated_sensor_token"] + ) + car_from_current = transform_matrix( + current_cs_rec["translation"], + Quaternion(current_cs_rec["rotation"]), + inverse=False, + ) + + tm = reduce( + np.dot, + [ref_from_car, car_from_global, global_from_car, car_from_current], + ) + + lidar_path = nusc.get_sample_data_path(curr_sd_rec["token"]) + + time_lag = ref_time - 1e-6 * curr_sd_rec["timestamp"] + + sweep = { + "lidar_path": Path(lidar_path).relative_to(data_path).__str__(), + "sample_data_token": curr_sd_rec["token"], + "transform_matrix": tm, + "global_from_car": global_from_car, + "car_from_current": car_from_current, + "time_lag": time_lag, + } + sweeps.append(sweep) + + info["sweeps"] = sweeps + + assert len(info["sweeps"]) == max_sweeps - 1, ( + f"sweep {curr_sd_rec['token']} only has {len(info['sweeps'])} sweeps, " + f"you should duplicate to sweep num {max_sweeps - 1}" + ) + + if not test: + # processing gt bbox + annotations = [ + nusc.get("sample_annotation", token) for token in sample["anns"] + ] + + # the filtering gives 0.5~1 map improvement + num_lidar_pts = np.array([anno["num_lidar_pts"] for anno in annotations]) + num_radar_pts = np.array([anno["num_radar_pts"] for anno in annotations]) + mask = num_lidar_pts + num_radar_pts > 0 + + locs = np.array([b.center for b in ref_boxes]).reshape(-1, 3) + dims = np.array([b.wlh for b in ref_boxes]).reshape(-1, 3)[ + :, [1, 0, 2] + ] # wlh == > dxdydz (lwh) + velocity = np.array([b.velocity for b in ref_boxes]).reshape(-1, 3) + rots = np.array([quaternion_yaw(b.orientation) for b in ref_boxes]).reshape( + -1, 1 + ) + names = np.array([b.name for b in ref_boxes]) + tokens = np.array([b.token for b in ref_boxes]) + gt_boxes = np.concatenate([locs, dims, rots, velocity[:, :2]], axis=1) + + assert len(annotations) == len(gt_boxes) == len(velocity) + + info["gt_boxes"] = gt_boxes[mask, :] + info["gt_boxes_velocity"] = velocity[mask, :] + info["gt_names"] = np.array( + [map_name_from_general_to_detection[name] for name in names] + )[mask] + info["gt_boxes_token"] = tokens[mask] + info["num_lidar_pts"] = num_lidar_pts[mask] + info["num_radar_pts"] = num_radar_pts[mask] + + # processing gt segment + segment_path = nusc.get("lidarseg", ref_sd_token)["filename"] + info["gt_segment_path"] = segment_path + + if sample["scene_token"] in train_scenes: + train_nusc_infos.append(info) + else: + val_nusc_infos.append(info) + + progress_bar.close() + return train_nusc_infos, val_nusc_infos + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset_root", required=True, help="Path to the nuScenes dataset." + ) + parser.add_argument( + "--output_root", + required=True, + help="Output path where processed information located.", + ) + parser.add_argument( + "--max_sweeps", default=10, type=int, help="Max number of sweeps. Default: 10." + ) + parser.add_argument( + "--with_camera", + action="store_true", + default=False, + help="Whether use camera or not.", + ) + config = parser.parse_args() + + print(f"Loading nuScenes tables for version v1.0-trainval...") + nusc_trainval = NuScenes( + version="v1.0-trainval", dataroot=config.dataset_root, verbose=False + ) + available_scenes_trainval = get_available_scenes(nusc_trainval) + available_scene_names_trainval = [s["name"] for s in available_scenes_trainval] + print("total scene num:", len(nusc_trainval.scene)) + print("exist scene num:", len(available_scenes_trainval)) + assert len(available_scenes_trainval) == len(nusc_trainval.scene) == 850 + + print(f"Loading nuScenes tables for version v1.0-test...") + nusc_test = NuScenes( + version="v1.0-test", dataroot=config.dataset_root, verbose=False + ) + available_scenes_test = get_available_scenes(nusc_test) + available_scene_names_test = [s["name"] for s in available_scenes_test] + print("total scene num:", len(nusc_test.scene)) + print("exist scene num:", len(available_scenes_test)) + assert len(available_scenes_test) == len(nusc_test.scene) == 150 + + train_scenes = splits.train + train_scenes = set( + [ + available_scenes_trainval[available_scene_names_trainval.index(s)]["token"] + for s in train_scenes + ] + ) + test_scenes = splits.test + test_scenes = set( + [ + available_scenes_test[available_scene_names_test.index(s)]["token"] + for s in test_scenes + ] + ) + print(f"Filling trainval information...") + train_nusc_infos, val_nusc_infos = fill_trainval_infos( + config.dataset_root, + nusc_trainval, + train_scenes, + test=False, + max_sweeps=config.max_sweeps, + with_camera=config.with_camera, + ) + print(f"Filling test information...") + test_nusc_infos, _ = fill_trainval_infos( + config.dataset_root, + nusc_test, + test_scenes, + test=True, + max_sweeps=config.max_sweeps, + with_camera=config.with_camera, + ) + + print(f"Saving nuScenes information...") + os.makedirs(os.path.join(config.output_root, "info"), exist_ok=True) + print( + f"train sample: {len(train_nusc_infos)}, val sample: {len(val_nusc_infos)}, test sample: {len(test_nusc_infos)}" + ) + with open( + os.path.join( + config.output_root, + "info", + f"nuscenes_infos_{config.max_sweeps}sweeps_train.pkl", + ), + "wb", + ) as f: + pickle.dump(train_nusc_infos, f) + with open( + os.path.join( + config.output_root, + "info", + f"nuscenes_infos_{config.max_sweeps}sweeps_val.pkl", + ), + "wb", + ) as f: + pickle.dump(val_nusc_infos, f) + with open( + os.path.join( + config.output_root, + "info", + f"nuscenes_infos_{config.max_sweeps}sweeps_test.pkl", + ), + "wb", + ) as f: + pickle.dump(test_nusc_infos, f) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/s3dis/preprocess_s3dis.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/s3dis/preprocess_s3dis.py new file mode 100644 index 0000000000000000000000000000000000000000..d770ad6317996c8a53cd13b2e12af3a536b6dca4 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/s3dis/preprocess_s3dis.py @@ -0,0 +1,233 @@ +""" +Preprocessing Script for S3DIS +Parsing normal vectors has a large consumption of memory. Please reduce max_workers if memory is limited. + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import argparse +import glob +import numpy as np + +try: + import open3d +except ImportError: + import warnings + + warnings.warn("Please install open3d for parsing normal") + +try: + import trimesh +except ImportError: + import warnings + + warnings.warn("Please install trimesh for parsing normal") + +from concurrent.futures import ProcessPoolExecutor +from itertools import repeat + +area_mesh_dict = {} + + +def parse_room( + room, angle, dataset_root, output_root, align_angle=True, parse_normal=False +): + print("Parsing: {}".format(room)) + classes = [ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", + ] + class2label = {cls: i for i, cls in enumerate(classes)} + source_dir = os.path.join(dataset_root, room) + save_path = os.path.join(output_root, room) + os.makedirs(save_path, exist_ok=True) + object_path_list = sorted(glob.glob(os.path.join(source_dir, "Annotations/*.txt"))) + + room_coords = [] + room_colors = [] + room_normals = [] + room_semantic_gt = [] + room_instance_gt = [] + + for object_id, object_path in enumerate(object_path_list): + object_name = os.path.basename(object_path).split("_")[0] + obj = np.loadtxt(object_path) + coords = obj[:, :3] + colors = obj[:, 3:6] + # note: in some room there is 'stairs' class + class_name = object_name if object_name in classes else "clutter" + semantic_gt = np.repeat(class2label[class_name], coords.shape[0]) + semantic_gt = semantic_gt.reshape([-1, 1]) + instance_gt = np.repeat(object_id, coords.shape[0]) + instance_gt = instance_gt.reshape([-1, 1]) + + room_coords.append(coords) + room_colors.append(colors) + room_semantic_gt.append(semantic_gt) + room_instance_gt.append(instance_gt) + + room_coords = np.ascontiguousarray(np.vstack(room_coords)) + + if parse_normal: + x_min, z_max, y_min = np.min(room_coords, axis=0) + x_max, z_min, y_max = np.max(room_coords, axis=0) + z_max = -z_max + z_min = -z_min + max_bound = np.array([x_max, y_max, z_max]) + 0.1 + min_bound = np.array([x_min, y_min, z_min]) - 0.1 + bbox = open3d.geometry.AxisAlignedBoundingBox( + min_bound=min_bound, max_bound=max_bound + ) + # crop room + room_mesh = ( + area_mesh_dict[os.path.dirname(room)] + .crop(bbox) + .transform( + np.array([[1, 0, 0, 0], [0, 0, -1, 0], [0, 1, 0, 0], [0, 0, 0, 1]]) + ) + ) + vertices = np.array(room_mesh.vertices) + faces = np.array(room_mesh.triangles) + vertex_normals = np.array(room_mesh.vertex_normals) + room_mesh = trimesh.Trimesh( + vertices=vertices, faces=faces, vertex_normals=vertex_normals + ) + (closest_points, distances, face_id) = room_mesh.nearest.on_surface(room_coords) + room_normals = room_mesh.face_normals[face_id] + + if align_angle: + angle = (2 - angle / 180) * np.pi + rot_cos, rot_sin = np.cos(angle), np.sin(angle) + rot_t = np.array([[rot_cos, -rot_sin, 0], [rot_sin, rot_cos, 0], [0, 0, 1]]) + room_center = (np.max(room_coords, axis=0) + np.min(room_coords, axis=0)) / 2 + room_coords = (room_coords - room_center) @ np.transpose(rot_t) + room_center + if parse_normal: + room_normals = room_normals @ np.transpose(rot_t) + + room_colors = np.ascontiguousarray(np.vstack(room_colors)) + room_semantic_gt = np.ascontiguousarray(np.vstack(room_semantic_gt)) + room_instance_gt = np.ascontiguousarray(np.vstack(room_instance_gt)) + np.save(os.path.join(save_path, "coord.npy"), room_coords.astype(np.float32)) + np.save(os.path.join(save_path, "color.npy"), room_colors.astype(np.uint8)) + np.save(os.path.join(save_path, "segment.npy"), room_semantic_gt.astype(np.int16)) + np.save(os.path.join(save_path, "instance.npy"), room_instance_gt.astype(np.int16)) + + if parse_normal: + np.save(os.path.join(save_path, "normal.npy"), room_normals.astype(np.float32)) + + +def main_process(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--splits", + required=True, + nargs="+", + choices=["Area_1", "Area_2", "Area_3", "Area_4", "Area_5", "Area_6"], + help="Splits need to process ([Area_1, Area_2, Area_3, Area_4, Area_5, Area_6]).", + ) + parser.add_argument( + "--dataset_root", required=True, help="Path to Stanford3dDataset_v1.2 dataset" + ) + parser.add_argument( + "--output_root", + required=True, + help="Output path where area folders will be located", + ) + parser.add_argument( + "--raw_root", + default=None, + help="Path to Stanford2d3dDataset_noXYZ dataset (optional)", + ) + parser.add_argument( + "--align_angle", action="store_true", help="Whether align room angles" + ) + parser.add_argument( + "--parse_normal", action="store_true", help="Whether process normal" + ) + parser.add_argument( + "--num_workers", default=1, type=int, help="Num workers for preprocessing." + ) + args = parser.parse_args() + + if args.parse_normal: + assert args.raw_root is not None + + room_list = [] + angle_list = [] + + # Load room information + print("Loading room information ...") + for split in args.splits: + area_info = np.loadtxt( + os.path.join( + args.dataset_root, + split, + f"{split}_alignmentAngle.txt", + ), + dtype=str, + ) + room_list += [os.path.join(split, room_info[0]) for room_info in area_info] + angle_list += [int(room_info[1]) for room_info in area_info] + + if args.parse_normal: + # load raw mesh file to extract normal + print("Loading raw mesh file ...") + for split in args.splits: + if split != "Area_5": + mesh_dir = os.path.join(args.raw_root, split, "3d", "rgb.obj") + mesh = open3d.io.read_triangle_mesh(mesh_dir) + mesh.triangle_uvs.clear() + else: + mesh_a_dir = os.path.join(args.raw_root, f"{split}a", "3d", "rgb.obj") + mesh_b_dir = os.path.join(args.raw_root, f"{split}b", "3d", "rgb.obj") + mesh_a = open3d.io.read_triangle_mesh(mesh_a_dir) + mesh_a.triangle_uvs.clear() + mesh_b = open3d.io.read_triangle_mesh(mesh_b_dir) + mesh_b.triangle_uvs.clear() + mesh_b = mesh_b.transform( + np.array( + [ + [0, 0, -1, -4.09703582], + [0, 1, 0, 0], + [1, 0, 0, -6.22617759], + [0, 0, 0, 1], + ] + ) + ) + mesh = mesh_a + mesh_b + area_mesh_dict[split] = mesh + print(f"{split} mesh is loaded") + + # Preprocess data. + print("Processing scenes...") + pool = ProcessPoolExecutor( + max_workers=args.num_workers + ) # peak 110G memory when parsing normal. + _ = list( + pool.map( + parse_room, + room_list, + angle_list, + repeat(args.dataset_root), + repeat(args.output_root), + repeat(args.align_angle), + repeat(args.parse_normal), + ) + ) + + +if __name__ == "__main__": + main_process() diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/sampling_chunking_data.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/sampling_chunking_data.py new file mode 100644 index 0000000000000000000000000000000000000000..96536d415370bf28c0f1cc89312b2fde719c9a58 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/sampling_chunking_data.py @@ -0,0 +1,149 @@ +""" +Chunking Data + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import argparse +import numpy as np +import multiprocessing as mp +from concurrent.futures import ProcessPoolExecutor +from itertools import repeat +from pathlib import Path + + +def chunking_scene( + name, + dataset_root, + split, + grid_size=None, + chunk_range=(6, 6), + chunk_stride=(3, 3), + chunk_minimum_size=10000, +): + print(f"Chunking scene {name} in {split} split") + dataset_root = Path(dataset_root) + scene_path = dataset_root / split / name + assets = os.listdir(scene_path) + data_dict = dict() + for asset in assets: + if not asset.endswith(".npy"): + continue + data_dict[asset[:-4]] = np.load(scene_path / asset) + coord = data_dict["coord"] - data_dict["coord"].min(axis=0) + + if grid_size is not None: + grid_coord = np.floor(coord / grid_size).astype(int) + _, idx = np.unique(grid_coord, axis=0, return_index=True) + coord = coord[idx] + for key in data_dict.keys(): + data_dict[key] = data_dict[key][idx] + + bev_range = coord.max(axis=0)[:2] + x, y = np.meshgrid( + np.arange(0, bev_range[0] + chunk_stride[0] - chunk_range[0], chunk_stride[0]), + np.arange(0, bev_range[0] + chunk_stride[0] - chunk_range[0], chunk_stride[0]), + indexing="ij", + ) + chunks = np.concatenate([x.reshape([-1, 1]), y.reshape([-1, 1])], axis=-1) + chunk_idx = 0 + for chunk in chunks: + mask = ( + (coord[:, 0] >= chunk[0]) + & (coord[:, 0] < chunk[0] + chunk_range[0]) + & (coord[:, 1] >= chunk[1]) + & (coord[:, 1] < chunk[1] + chunk_range[1]) + ) + if np.sum(mask) < chunk_minimum_size: + continue + + chunk_data_name = f"{name}_{chunk_idx}" + if grid_size is not None: + chunk_split_name = ( + f"{split}_" + f"grid{grid_size * 100:.0f}mm_" + f"chunk{chunk_range[0]}x{chunk_range[1]}_" + f"stride{chunk_stride[0]}x{chunk_stride[1]}" + ) + else: + chunk_split_name = ( + f"{split}_" + f"chunk{chunk_range[0]}x{chunk_range[1]}_" + f"stride{chunk_stride[0]}x{chunk_stride[1]}" + ) + + chunk_save_path = dataset_root / chunk_split_name / chunk_data_name + chunk_save_path.mkdir(parents=True, exist_ok=True) + for key in data_dict.keys(): + np.save(chunk_save_path / f"{key}.npy", data_dict[key][mask]) + chunk_idx += 1 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset_root", + required=True, + help="Path to the Pointcept processed ScanNet++ dataset.", + ) + parser.add_argument( + "--split", + required=True, + default="train", + type=str, + help="Split need to process.", + ) + parser.add_argument( + "--grid_size", + default=None, + type=float, + help="Grid size for initial grid sampling", + ) + parser.add_argument( + "--chunk_range", + default=[6, 6], + type=int, + nargs="+", + help="Range of each chunk, e.g. --chunk_range 6 6", + ) + parser.add_argument( + "--chunk_stride", + default=[3, 3], + type=int, + nargs="+", + help="Stride of each chunk, e.g. --chunk_stride 3 3", + ) + parser.add_argument( + "--chunk_minimum_size", + default=10000, + type=int, + help="Minimum number of points in each chunk", + ) + parser.add_argument( + "--num_workers", + default=mp.cpu_count(), + type=int, + help="Num workers for preprocessing.", + ) + + config = parser.parse_args() + config.dataset_root = Path(config.dataset_root) + data_list = os.listdir(config.dataset_root / config.split) + + print("Processing scenes...") + pool = ProcessPoolExecutor(max_workers=config.num_workers) + _ = list( + pool.map( + chunking_scene, + data_list, + repeat(config.dataset_root), + repeat(config.split), + repeat(config.grid_size), + repeat(config.chunk_range), + repeat(config.chunk_stride), + repeat(config.chunk_minimum_size), + ) + ) + pool.shutdown() diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/classes_ObjClassification-ShapeNetCore55.txt b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/classes_ObjClassification-ShapeNetCore55.txt new file mode 100644 index 0000000000000000000000000000000000000000..e53f5bcb2c1480f42ee9327940246258aa434f88 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/classes_ObjClassification-ShapeNetCore55.txt @@ -0,0 +1,17 @@ +1 trash +3 basket +4 bathtub +5 bed +9 shelf +13 cabinet +18 chair +20 keyboard +22 tv +30 lamp +31 laptop +35 microwave +39 pillow +42 printer +47 sofa +48 stove +49 table diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/classes_SemVoxLabel-nyu40id.txt b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/classes_SemVoxLabel-nyu40id.txt new file mode 100644 index 0000000000000000000000000000000000000000..48e228766391e0f0234c2eed086e31f738068a4b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/classes_SemVoxLabel-nyu40id.txt @@ -0,0 +1,20 @@ +1 wall +2 floor +3 cabinet +4 bed +5 chair +6 sofa +7 table +8 door +9 window +10 bookshelf +11 picture +12 counter +14 desk +16 curtain +24 refridgerator +28 shower curtain +33 toilet +34 sink +36 bathtub +39 otherfurniture \ No newline at end of file diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannet200_constants.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannet200_constants.py new file mode 100644 index 0000000000000000000000000000000000000000..0404fd6aa8ad14ad729354ce184d4b51834bfd1b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannet200_constants.py @@ -0,0 +1,704 @@ +# ScanNet Benchmark constants +VALID_CLASS_IDS_20 = ( + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 14, + 16, + 24, + 28, + 33, + 34, + 36, + 39, +) + +CLASS_LABELS_20 = ( + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "bookshelf", + "picture", + "counter", + "desk", + "curtain", + "refrigerator", + "shower curtain", + "toilet", + "sink", + "bathtub", + "otherfurniture", +) + +SCANNET_COLOR_MAP_20 = { + 0: (0.0, 0.0, 0.0), + 1: (174.0, 199.0, 232.0), + 2: (152.0, 223.0, 138.0), + 3: (31.0, 119.0, 180.0), + 4: (255.0, 187.0, 120.0), + 5: (188.0, 189.0, 34.0), + 6: (140.0, 86.0, 75.0), + 7: (255.0, 152.0, 150.0), + 8: (214.0, 39.0, 40.0), + 9: (197.0, 176.0, 213.0), + 10: (148.0, 103.0, 189.0), + 11: (196.0, 156.0, 148.0), + 12: (23.0, 190.0, 207.0), + 14: (247.0, 182.0, 210.0), + 15: (66.0, 188.0, 102.0), + 16: (219.0, 219.0, 141.0), + 17: (140.0, 57.0, 197.0), + 18: (202.0, 185.0, 52.0), + 19: (51.0, 176.0, 203.0), + 20: (200.0, 54.0, 131.0), + 21: (92.0, 193.0, 61.0), + 22: (78.0, 71.0, 183.0), + 23: (172.0, 114.0, 82.0), + 24: (255.0, 127.0, 14.0), + 25: (91.0, 163.0, 138.0), + 26: (153.0, 98.0, 156.0), + 27: (140.0, 153.0, 101.0), + 28: (158.0, 218.0, 229.0), + 29: (100.0, 125.0, 154.0), + 30: (178.0, 127.0, 135.0), + 32: (146.0, 111.0, 194.0), + 33: (44.0, 160.0, 44.0), + 34: (112.0, 128.0, 144.0), + 35: (96.0, 207.0, 209.0), + 36: (227.0, 119.0, 194.0), + 37: (213.0, 92.0, 176.0), + 38: (94.0, 106.0, 211.0), + 39: (82.0, 84.0, 163.0), + 40: (100.0, 85.0, 144.0), +} + +# ScanNet200 Benchmark constants +VALID_CLASS_IDS_200 = ( + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 21, + 22, + 23, + 24, + 26, + 27, + 28, + 29, + 31, + 32, + 33, + 34, + 35, + 36, + 38, + 39, + 40, + 41, + 42, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 54, + 55, + 56, + 57, + 58, + 59, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 82, + 84, + 86, + 87, + 88, + 89, + 90, + 93, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 110, + 112, + 115, + 116, + 118, + 120, + 121, + 122, + 125, + 128, + 130, + 131, + 132, + 134, + 136, + 138, + 139, + 140, + 141, + 145, + 148, + 154, + 155, + 156, + 157, + 159, + 161, + 163, + 165, + 166, + 168, + 169, + 170, + 177, + 180, + 185, + 188, + 191, + 193, + 195, + 202, + 208, + 213, + 214, + 221, + 229, + 230, + 232, + 233, + 242, + 250, + 261, + 264, + 276, + 283, + 286, + 300, + 304, + 312, + 323, + 325, + 331, + 342, + 356, + 370, + 392, + 395, + 399, + 408, + 417, + 488, + 540, + 562, + 570, + 572, + 581, + 609, + 748, + 776, + 1156, + 1163, + 1164, + 1165, + 1166, + 1167, + 1168, + 1169, + 1170, + 1171, + 1172, + 1173, + 1174, + 1175, + 1176, + 1178, + 1179, + 1180, + 1181, + 1182, + 1183, + 1184, + 1185, + 1186, + 1187, + 1188, + 1189, + 1190, + 1191, +) + +CLASS_LABELS_200 = ( + "wall", + "chair", + "floor", + "table", + "door", + "couch", + "cabinet", + "shelf", + "desk", + "office chair", + "bed", + "pillow", + "sink", + "picture", + "window", + "toilet", + "bookshelf", + "monitor", + "curtain", + "book", + "armchair", + "coffee table", + "box", + "refrigerator", + "lamp", + "kitchen cabinet", + "towel", + "clothes", + "tv", + "nightstand", + "counter", + "dresser", + "stool", + "cushion", + "plant", + "ceiling", + "bathtub", + "end table", + "dining table", + "keyboard", + "bag", + "backpack", + "toilet paper", + "printer", + "tv stand", + "whiteboard", + "blanket", + "shower curtain", + "trash can", + "closet", + "stairs", + "microwave", + "stove", + "shoe", + "computer tower", + "bottle", + "bin", + "ottoman", + "bench", + "board", + "washing machine", + "mirror", + "copier", + "basket", + "sofa chair", + "file cabinet", + "fan", + "laptop", + "shower", + "paper", + "person", + "paper towel dispenser", + "oven", + "blinds", + "rack", + "plate", + "blackboard", + "piano", + "suitcase", + "rail", + "radiator", + "recycling bin", + "container", + "wardrobe", + "soap dispenser", + "telephone", + "bucket", + "clock", + "stand", + "light", + "laundry basket", + "pipe", + "clothes dryer", + "guitar", + "toilet paper holder", + "seat", + "speaker", + "column", + "bicycle", + "ladder", + "bathroom stall", + "shower wall", + "cup", + "jacket", + "storage bin", + "coffee maker", + "dishwasher", + "paper towel roll", + "machine", + "mat", + "windowsill", + "bar", + "toaster", + "bulletin board", + "ironing board", + "fireplace", + "soap dish", + "kitchen counter", + "doorframe", + "toilet paper dispenser", + "mini fridge", + "fire extinguisher", + "ball", + "hat", + "shower curtain rod", + "water cooler", + "paper cutter", + "tray", + "shower door", + "pillar", + "ledge", + "toaster oven", + "mouse", + "toilet seat cover dispenser", + "furniture", + "cart", + "storage container", + "scale", + "tissue box", + "light switch", + "crate", + "power outlet", + "decoration", + "sign", + "projector", + "closet door", + "vacuum cleaner", + "candle", + "plunger", + "stuffed animal", + "headphones", + "dish rack", + "broom", + "guitar case", + "range hood", + "dustpan", + "hair dryer", + "water bottle", + "handicap bar", + "purse", + "vent", + "shower floor", + "water pitcher", + "mailbox", + "bowl", + "paper bag", + "alarm clock", + "music stand", + "projector screen", + "divider", + "laundry detergent", + "bathroom counter", + "object", + "bathroom vanity", + "closet wall", + "laundry hamper", + "bathroom stall door", + "ceiling light", + "trash bin", + "dumbbell", + "stair rail", + "tube", + "bathroom cabinet", + "cd case", + "closet rod", + "coffee kettle", + "structure", + "shower head", + "keyboard piano", + "case of water bottles", + "coat rack", + "storage organizer", + "folded chair", + "fire alarm", + "power strip", + "calendar", + "poster", + "potted plant", + "luggage", + "mattress", +) + +SCANNET_COLOR_MAP_200 = { + 0: (0.0, 0.0, 0.0), + 1: (174.0, 199.0, 232.0), + 2: (188.0, 189.0, 34.0), + 3: (152.0, 223.0, 138.0), + 4: (255.0, 152.0, 150.0), + 5: (214.0, 39.0, 40.0), + 6: (91.0, 135.0, 229.0), + 7: (31.0, 119.0, 180.0), + 8: (229.0, 91.0, 104.0), + 9: (247.0, 182.0, 210.0), + 10: (91.0, 229.0, 110.0), + 11: (255.0, 187.0, 120.0), + 13: (141.0, 91.0, 229.0), + 14: (112.0, 128.0, 144.0), + 15: (196.0, 156.0, 148.0), + 16: (197.0, 176.0, 213.0), + 17: (44.0, 160.0, 44.0), + 18: (148.0, 103.0, 189.0), + 19: (229.0, 91.0, 223.0), + 21: (219.0, 219.0, 141.0), + 22: (192.0, 229.0, 91.0), + 23: (88.0, 218.0, 137.0), + 24: (58.0, 98.0, 137.0), + 26: (177.0, 82.0, 239.0), + 27: (255.0, 127.0, 14.0), + 28: (237.0, 204.0, 37.0), + 29: (41.0, 206.0, 32.0), + 31: (62.0, 143.0, 148.0), + 32: (34.0, 14.0, 130.0), + 33: (143.0, 45.0, 115.0), + 34: (137.0, 63.0, 14.0), + 35: (23.0, 190.0, 207.0), + 36: (16.0, 212.0, 139.0), + 38: (90.0, 119.0, 201.0), + 39: (125.0, 30.0, 141.0), + 40: (150.0, 53.0, 56.0), + 41: (186.0, 197.0, 62.0), + 42: (227.0, 119.0, 194.0), + 44: (38.0, 100.0, 128.0), + 45: (120.0, 31.0, 243.0), + 46: (154.0, 59.0, 103.0), + 47: (169.0, 137.0, 78.0), + 48: (143.0, 245.0, 111.0), + 49: (37.0, 230.0, 205.0), + 50: (14.0, 16.0, 155.0), + 51: (196.0, 51.0, 182.0), + 52: (237.0, 80.0, 38.0), + 54: (138.0, 175.0, 62.0), + 55: (158.0, 218.0, 229.0), + 56: (38.0, 96.0, 167.0), + 57: (190.0, 77.0, 246.0), + 58: (208.0, 49.0, 84.0), + 59: (208.0, 193.0, 72.0), + 62: (55.0, 220.0, 57.0), + 63: (10.0, 125.0, 140.0), + 64: (76.0, 38.0, 202.0), + 65: (191.0, 28.0, 135.0), + 66: (211.0, 120.0, 42.0), + 67: (118.0, 174.0, 76.0), + 68: (17.0, 242.0, 171.0), + 69: (20.0, 65.0, 247.0), + 70: (208.0, 61.0, 222.0), + 71: (162.0, 62.0, 60.0), + 72: (210.0, 235.0, 62.0), + 73: (45.0, 152.0, 72.0), + 74: (35.0, 107.0, 149.0), + 75: (160.0, 89.0, 237.0), + 76: (227.0, 56.0, 125.0), + 77: (169.0, 143.0, 81.0), + 78: (42.0, 143.0, 20.0), + 79: (25.0, 160.0, 151.0), + 80: (82.0, 75.0, 227.0), + 82: (253.0, 59.0, 222.0), + 84: (240.0, 130.0, 89.0), + 86: (123.0, 172.0, 47.0), + 87: (71.0, 194.0, 133.0), + 88: (24.0, 94.0, 205.0), + 89: (134.0, 16.0, 179.0), + 90: (159.0, 32.0, 52.0), + 93: (213.0, 208.0, 88.0), + 95: (64.0, 158.0, 70.0), + 96: (18.0, 163.0, 194.0), + 97: (65.0, 29.0, 153.0), + 98: (177.0, 10.0, 109.0), + 99: (152.0, 83.0, 7.0), + 100: (83.0, 175.0, 30.0), + 101: (18.0, 199.0, 153.0), + 102: (61.0, 81.0, 208.0), + 103: (213.0, 85.0, 216.0), + 104: (170.0, 53.0, 42.0), + 105: (161.0, 192.0, 38.0), + 106: (23.0, 241.0, 91.0), + 107: (12.0, 103.0, 170.0), + 110: (151.0, 41.0, 245.0), + 112: (133.0, 51.0, 80.0), + 115: (184.0, 162.0, 91.0), + 116: (50.0, 138.0, 38.0), + 118: (31.0, 237.0, 236.0), + 120: (39.0, 19.0, 208.0), + 121: (223.0, 27.0, 180.0), + 122: (254.0, 141.0, 85.0), + 125: (97.0, 144.0, 39.0), + 128: (106.0, 231.0, 176.0), + 130: (12.0, 61.0, 162.0), + 131: (124.0, 66.0, 140.0), + 132: (137.0, 66.0, 73.0), + 134: (250.0, 253.0, 26.0), + 136: (55.0, 191.0, 73.0), + 138: (60.0, 126.0, 146.0), + 139: (153.0, 108.0, 234.0), + 140: (184.0, 58.0, 125.0), + 141: (135.0, 84.0, 14.0), + 145: (139.0, 248.0, 91.0), + 148: (53.0, 200.0, 172.0), + 154: (63.0, 69.0, 134.0), + 155: (190.0, 75.0, 186.0), + 156: (127.0, 63.0, 52.0), + 157: (141.0, 182.0, 25.0), + 159: (56.0, 144.0, 89.0), + 161: (64.0, 160.0, 250.0), + 163: (182.0, 86.0, 245.0), + 165: (139.0, 18.0, 53.0), + 166: (134.0, 120.0, 54.0), + 168: (49.0, 165.0, 42.0), + 169: (51.0, 128.0, 133.0), + 170: (44.0, 21.0, 163.0), + 177: (232.0, 93.0, 193.0), + 180: (176.0, 102.0, 54.0), + 185: (116.0, 217.0, 17.0), + 188: (54.0, 209.0, 150.0), + 191: (60.0, 99.0, 204.0), + 193: (129.0, 43.0, 144.0), + 195: (252.0, 100.0, 106.0), + 202: (187.0, 196.0, 73.0), + 208: (13.0, 158.0, 40.0), + 213: (52.0, 122.0, 152.0), + 214: (128.0, 76.0, 202.0), + 221: (187.0, 50.0, 115.0), + 229: (180.0, 141.0, 71.0), + 230: (77.0, 208.0, 35.0), + 232: (72.0, 183.0, 168.0), + 233: (97.0, 99.0, 203.0), + 242: (172.0, 22.0, 158.0), + 250: (155.0, 64.0, 40.0), + 261: (118.0, 159.0, 30.0), + 264: (69.0, 252.0, 148.0), + 276: (45.0, 103.0, 173.0), + 283: (111.0, 38.0, 149.0), + 286: (184.0, 9.0, 49.0), + 300: (188.0, 174.0, 67.0), + 304: (53.0, 206.0, 53.0), + 312: (97.0, 235.0, 252.0), + 323: (66.0, 32.0, 182.0), + 325: (236.0, 114.0, 195.0), + 331: (241.0, 154.0, 83.0), + 342: (133.0, 240.0, 52.0), + 356: (16.0, 205.0, 144.0), + 370: (75.0, 101.0, 198.0), + 392: (237.0, 95.0, 251.0), + 395: (191.0, 52.0, 49.0), + 399: (227.0, 254.0, 54.0), + 408: (49.0, 206.0, 87.0), + 417: (48.0, 113.0, 150.0), + 488: (125.0, 73.0, 182.0), + 540: (229.0, 32.0, 114.0), + 562: (158.0, 119.0, 28.0), + 570: (60.0, 205.0, 27.0), + 572: (18.0, 215.0, 201.0), + 581: (79.0, 76.0, 153.0), + 609: (134.0, 13.0, 116.0), + 748: (192.0, 97.0, 63.0), + 776: (108.0, 163.0, 18.0), + 1156: (95.0, 220.0, 156.0), + 1163: (98.0, 141.0, 208.0), + 1164: (144.0, 19.0, 193.0), + 1165: (166.0, 36.0, 57.0), + 1166: (212.0, 202.0, 34.0), + 1167: (23.0, 206.0, 34.0), + 1168: (91.0, 211.0, 236.0), + 1169: (79.0, 55.0, 137.0), + 1170: (182.0, 19.0, 117.0), + 1171: (134.0, 76.0, 14.0), + 1172: (87.0, 185.0, 28.0), + 1173: (82.0, 224.0, 187.0), + 1174: (92.0, 110.0, 214.0), + 1175: (168.0, 80.0, 171.0), + 1176: (197.0, 63.0, 51.0), + 1178: (175.0, 199.0, 77.0), + 1179: (62.0, 180.0, 98.0), + 1180: (8.0, 91.0, 150.0), + 1181: (77.0, 15.0, 130.0), + 1182: (154.0, 65.0, 96.0), + 1183: (197.0, 152.0, 11.0), + 1184: (59.0, 155.0, 45.0), + 1185: (12.0, 147.0, 145.0), + 1186: (54.0, 35.0, 219.0), + 1187: (210.0, 73.0, 181.0), + 1188: (221.0, 124.0, 77.0), + 1189: (149.0, 214.0, 66.0), + 1190: (72.0, 185.0, 134.0), + 1191: (42.0, 94.0, 198.0), +} + +# For instance segmentation the non-object categories +VALID_PANOPTIC_IDS = (1, 3) + +CLASS_LABELS_PANOPTIC = ("wall", "floor") diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannet200_splits.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannet200_splits.py new file mode 100644 index 0000000000000000000000000000000000000000..39ccc3c60bf289199342332e455fadb5b22129ee --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannet200_splits.py @@ -0,0 +1,625 @@ +# This file contains the HEAD - COMMON - TAIL split category ids for ScanNet 200 + +HEAD_CATS_SCANNET_200 = [ + "tv stand", + "curtain", + "blinds", + "shower curtain", + "bookshelf", + "tv", + "kitchen cabinet", + "pillow", + "lamp", + "dresser", + "monitor", + "object", + "ceiling", + "board", + "stove", + "closet wall", + "couch", + "office chair", + "kitchen counter", + "shower", + "closet", + "doorframe", + "sofa chair", + "mailbox", + "nightstand", + "washing machine", + "picture", + "book", + "sink", + "recycling bin", + "table", + "backpack", + "shower wall", + "toilet", + "copier", + "counter", + "stool", + "refrigerator", + "window", + "file cabinet", + "chair", + "wall", + "plant", + "coffee table", + "stairs", + "armchair", + "cabinet", + "bathroom vanity", + "bathroom stall", + "mirror", + "blackboard", + "trash can", + "stair rail", + "box", + "towel", + "door", + "clothes", + "whiteboard", + "bed", + "floor", + "bathtub", + "desk", + "wardrobe", + "clothes dryer", + "radiator", + "shelf", +] +COMMON_CATS_SCANNET_200 = [ + "cushion", + "end table", + "dining table", + "keyboard", + "bag", + "toilet paper", + "printer", + "blanket", + "microwave", + "shoe", + "computer tower", + "bottle", + "bin", + "ottoman", + "bench", + "basket", + "fan", + "laptop", + "person", + "paper towel dispenser", + "oven", + "rack", + "piano", + "suitcase", + "rail", + "container", + "telephone", + "stand", + "light", + "laundry basket", + "pipe", + "seat", + "column", + "bicycle", + "ladder", + "jacket", + "storage bin", + "coffee maker", + "dishwasher", + "machine", + "mat", + "windowsill", + "bulletin board", + "fireplace", + "mini fridge", + "water cooler", + "shower door", + "pillar", + "ledge", + "furniture", + "cart", + "decoration", + "closet door", + "vacuum cleaner", + "dish rack", + "range hood", + "projector screen", + "divider", + "bathroom counter", + "laundry hamper", + "bathroom stall door", + "ceiling light", + "trash bin", + "bathroom cabinet", + "structure", + "storage organizer", + "potted plant", + "mattress", +] +TAIL_CATS_SCANNET_200 = [ + "paper", + "plate", + "soap dispenser", + "bucket", + "clock", + "guitar", + "toilet paper holder", + "speaker", + "cup", + "paper towel roll", + "bar", + "toaster", + "ironing board", + "soap dish", + "toilet paper dispenser", + "fire extinguisher", + "ball", + "hat", + "shower curtain rod", + "paper cutter", + "tray", + "toaster oven", + "mouse", + "toilet seat cover dispenser", + "storage container", + "scale", + "tissue box", + "light switch", + "crate", + "power outlet", + "sign", + "projector", + "candle", + "plunger", + "stuffed animal", + "headphones", + "broom", + "guitar case", + "dustpan", + "hair dryer", + "water bottle", + "handicap bar", + "purse", + "vent", + "shower floor", + "water pitcher", + "bowl", + "paper bag", + "alarm clock", + "music stand", + "laundry detergent", + "dumbbell", + "tube", + "cd case", + "closet rod", + "coffee kettle", + "shower head", + "keyboard piano", + "case of water bottles", + "coat rack", + "folded chair", + "fire alarm", + "power strip", + "calendar", + "poster", + "luggage", +] + + +# Given the different size of the official train and val sets, not all ScanNet200 categories are present in the validation set. +# Here we list of categories with labels and IDs present in both train and validation set, and the remaining categories those are present in train, but not in val +# We dont evaluate on unseen validation categories in this benchmark + +VALID_CLASS_IDS_200_VALIDATION = ( + "wall", + "chair", + "floor", + "table", + "door", + "couch", + "cabinet", + "shelf", + "desk", + "office chair", + "bed", + "pillow", + "sink", + "picture", + "window", + "toilet", + "bookshelf", + "monitor", + "curtain", + "book", + "armchair", + "coffee table", + "box", + "refrigerator", + "lamp", + "kitchen cabinet", + "towel", + "clothes", + "tv", + "nightstand", + "counter", + "dresser", + "stool", + "cushion", + "plant", + "ceiling", + "bathtub", + "end table", + "dining table", + "keyboard", + "bag", + "backpack", + "toilet paper", + "printer", + "tv stand", + "whiteboard", + "blanket", + "shower curtain", + "trash can", + "closet", + "stairs", + "microwave", + "stove", + "shoe", + "computer tower", + "bottle", + "bin", + "ottoman", + "bench", + "board", + "washing machine", + "mirror", + "copier", + "basket", + "sofa chair", + "file cabinet", + "fan", + "laptop", + "shower", + "paper", + "person", + "paper towel dispenser", + "oven", + "blinds", + "rack", + "plate", + "blackboard", + "piano", + "suitcase", + "rail", + "radiator", + "recycling bin", + "container", + "wardrobe", + "soap dispenser", + "telephone", + "bucket", + "clock", + "stand", + "light", + "laundry basket", + "pipe", + "clothes dryer", + "guitar", + "toilet paper holder", + "seat", + "speaker", + "column", + "ladder", + "bathroom stall", + "shower wall", + "cup", + "jacket", + "storage bin", + "coffee maker", + "dishwasher", + "paper towel roll", + "machine", + "mat", + "windowsill", + "bar", + "toaster", + "bulletin board", + "ironing board", + "fireplace", + "soap dish", + "kitchen counter", + "doorframe", + "toilet paper dispenser", + "mini fridge", + "fire extinguisher", + "ball", + "hat", + "shower curtain rod", + "water cooler", + "paper cutter", + "tray", + "shower door", + "pillar", + "ledge", + "toaster oven", + "mouse", + "toilet seat cover dispenser", + "furniture", + "cart", + "scale", + "tissue box", + "light switch", + "crate", + "power outlet", + "decoration", + "sign", + "projector", + "closet door", + "vacuum cleaner", + "plunger", + "stuffed animal", + "headphones", + "dish rack", + "broom", + "range hood", + "dustpan", + "hair dryer", + "water bottle", + "handicap bar", + "vent", + "shower floor", + "water pitcher", + "mailbox", + "bowl", + "paper bag", + "projector screen", + "divider", + "laundry detergent", + "bathroom counter", + "object", + "bathroom vanity", + "closet wall", + "laundry hamper", + "bathroom stall door", + "ceiling light", + "trash bin", + "dumbbell", + "stair rail", + "tube", + "bathroom cabinet", + "closet rod", + "coffee kettle", + "shower head", + "keyboard piano", + "case of water bottles", + "coat rack", + "folded chair", + "fire alarm", + "power strip", + "calendar", + "poster", + "potted plant", + "mattress", +) + +CLASS_LABELS_200_VALIDATION = ( + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 21, + 22, + 23, + 24, + 26, + 27, + 28, + 29, + 31, + 32, + 33, + 34, + 35, + 36, + 38, + 39, + 40, + 41, + 42, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 54, + 55, + 56, + 57, + 58, + 59, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 82, + 84, + 86, + 87, + 88, + 89, + 90, + 93, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 110, + 112, + 115, + 116, + 118, + 120, + 122, + 125, + 128, + 130, + 131, + 132, + 134, + 136, + 138, + 139, + 140, + 141, + 145, + 148, + 154, + 155, + 156, + 157, + 159, + 161, + 163, + 165, + 166, + 168, + 169, + 170, + 177, + 180, + 185, + 188, + 191, + 193, + 195, + 202, + 208, + 213, + 214, + 229, + 230, + 232, + 233, + 242, + 250, + 261, + 264, + 276, + 283, + 300, + 304, + 312, + 323, + 325, + 342, + 356, + 370, + 392, + 395, + 408, + 417, + 488, + 540, + 562, + 570, + 609, + 748, + 776, + 1156, + 1163, + 1164, + 1165, + 1166, + 1167, + 1168, + 1169, + 1170, + 1171, + 1172, + 1173, + 1175, + 1176, + 1179, + 1180, + 1181, + 1182, + 1184, + 1185, + 1186, + 1187, + 1188, + 1189, + 1191, +) + +VALID_CLASS_IDS_200_TRAIN_ONLY = ( + "bicycle", + "storage container", + "candle", + "guitar case", + "purse", + "alarm clock", + "music stand", + "cd case", + "structure", + "storage organizer", + "luggage", +) + +CLASS_LABELS_200_TRAIN_ONLY = ( + 121, + 221, + 286, + 331, + 399, + 572, + 581, + 1174, + 1178, + 1183, + 1190, +) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannet_means.npz b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannet_means.npz new file mode 100644 index 0000000000000000000000000000000000000000..d9bbb4f7c3b72dbe81fbeb86f594066b883fafaf --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannet_means.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df5c2bd40e8518e982c7d7b4b39020b07ac774695038bf49cb28b44e5760457e +size 676 diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv1_test.txt b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv1_test.txt new file mode 100644 index 0000000000000000000000000000000000000000..b9e7d9205321e8ca047a527466f4b7100c9c9d2c --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv1_test.txt @@ -0,0 +1,312 @@ +scene0568_00 +scene0568_01 +scene0568_02 +scene0304_00 +scene0488_00 +scene0488_01 +scene0412_00 +scene0412_01 +scene0217_00 +scene0019_00 +scene0019_01 +scene0414_00 +scene0575_00 +scene0575_01 +scene0575_02 +scene0426_00 +scene0426_01 +scene0426_02 +scene0426_03 +scene0549_00 +scene0549_01 +scene0578_00 +scene0578_01 +scene0578_02 +scene0665_00 +scene0665_01 +scene0050_00 +scene0050_01 +scene0050_02 +scene0257_00 +scene0025_00 +scene0025_01 +scene0025_02 +scene0583_00 +scene0583_01 +scene0583_02 +scene0701_00 +scene0701_01 +scene0701_02 +scene0580_00 +scene0580_01 +scene0565_00 +scene0169_00 +scene0169_01 +scene0655_00 +scene0655_01 +scene0655_02 +scene0063_00 +scene0221_00 +scene0221_01 +scene0591_00 +scene0591_01 +scene0591_02 +scene0678_00 +scene0678_01 +scene0678_02 +scene0462_00 +scene0427_00 +scene0595_00 +scene0193_00 +scene0193_01 +scene0164_00 +scene0164_01 +scene0164_02 +scene0164_03 +scene0598_00 +scene0598_01 +scene0598_02 +scene0599_00 +scene0599_01 +scene0599_02 +scene0328_00 +scene0300_00 +scene0300_01 +scene0354_00 +scene0458_00 +scene0458_01 +scene0423_00 +scene0423_01 +scene0423_02 +scene0307_00 +scene0307_01 +scene0307_02 +scene0606_00 +scene0606_01 +scene0606_02 +scene0432_00 +scene0432_01 +scene0608_00 +scene0608_01 +scene0608_02 +scene0651_00 +scene0651_01 +scene0651_02 +scene0430_00 +scene0430_01 +scene0689_00 +scene0357_00 +scene0357_01 +scene0574_00 +scene0574_01 +scene0574_02 +scene0329_00 +scene0329_01 +scene0329_02 +scene0153_00 +scene0153_01 +scene0616_00 +scene0616_01 +scene0671_00 +scene0671_01 +scene0618_00 +scene0382_00 +scene0382_01 +scene0490_00 +scene0621_00 +scene0607_00 +scene0607_01 +scene0149_00 +scene0695_00 +scene0695_01 +scene0695_02 +scene0695_03 +scene0389_00 +scene0377_00 +scene0377_01 +scene0377_02 +scene0342_00 +scene0139_00 +scene0629_00 +scene0629_01 +scene0629_02 +scene0496_00 +scene0633_00 +scene0633_01 +scene0518_00 +scene0652_00 +scene0406_00 +scene0406_01 +scene0406_02 +scene0144_00 +scene0144_01 +scene0494_00 +scene0278_00 +scene0278_01 +scene0316_00 +scene0609_00 +scene0609_01 +scene0609_02 +scene0609_03 +scene0084_00 +scene0084_01 +scene0084_02 +scene0696_00 +scene0696_01 +scene0696_02 +scene0351_00 +scene0351_01 +scene0643_00 +scene0644_00 +scene0645_00 +scene0645_01 +scene0645_02 +scene0081_00 +scene0081_01 +scene0081_02 +scene0647_00 +scene0647_01 +scene0535_00 +scene0353_00 +scene0353_01 +scene0353_02 +scene0559_00 +scene0559_01 +scene0559_02 +scene0593_00 +scene0593_01 +scene0246_00 +scene0653_00 +scene0653_01 +scene0064_00 +scene0064_01 +scene0356_00 +scene0356_01 +scene0356_02 +scene0030_00 +scene0030_01 +scene0030_02 +scene0222_00 +scene0222_01 +scene0338_00 +scene0338_01 +scene0338_02 +scene0378_00 +scene0378_01 +scene0378_02 +scene0660_00 +scene0553_00 +scene0553_01 +scene0553_02 +scene0527_00 +scene0663_00 +scene0663_01 +scene0663_02 +scene0664_00 +scene0664_01 +scene0664_02 +scene0334_00 +scene0334_01 +scene0334_02 +scene0046_00 +scene0046_01 +scene0046_02 +scene0203_00 +scene0203_01 +scene0203_02 +scene0088_00 +scene0088_01 +scene0088_02 +scene0088_03 +scene0086_00 +scene0086_01 +scene0086_02 +scene0670_00 +scene0670_01 +scene0256_00 +scene0256_01 +scene0256_02 +scene0249_00 +scene0441_00 +scene0658_00 +scene0704_00 +scene0704_01 +scene0187_00 +scene0187_01 +scene0131_00 +scene0131_01 +scene0131_02 +scene0207_00 +scene0207_01 +scene0207_02 +scene0461_00 +scene0011_00 +scene0011_01 +scene0343_00 +scene0251_00 +scene0077_00 +scene0077_01 +scene0684_00 +scene0684_01 +scene0550_00 +scene0686_00 +scene0686_01 +scene0686_02 +scene0208_00 +scene0500_00 +scene0500_01 +scene0552_00 +scene0552_01 +scene0648_00 +scene0648_01 +scene0435_00 +scene0435_01 +scene0435_02 +scene0435_03 +scene0690_00 +scene0690_01 +scene0693_00 +scene0693_01 +scene0693_02 +scene0700_00 +scene0700_01 +scene0700_02 +scene0699_00 +scene0231_00 +scene0231_01 +scene0231_02 +scene0697_00 +scene0697_01 +scene0697_02 +scene0697_03 +scene0474_00 +scene0474_01 +scene0474_02 +scene0474_03 +scene0474_04 +scene0474_05 +scene0355_00 +scene0355_01 +scene0146_00 +scene0146_01 +scene0146_02 +scene0196_00 +scene0702_00 +scene0702_01 +scene0702_02 +scene0314_00 +scene0277_00 +scene0277_01 +scene0277_02 +scene0095_00 +scene0095_01 +scene0015_00 +scene0100_00 +scene0100_01 +scene0100_02 +scene0558_00 +scene0558_01 +scene0558_02 +scene0685_00 +scene0685_01 +scene0685_02 diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv1_train.txt b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv1_train.txt new file mode 100644 index 0000000000000000000000000000000000000000..7520948c8170df9ae1a9e8a40bc444fcc7cc0772 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv1_train.txt @@ -0,0 +1,1045 @@ +scene0191_00 +scene0191_01 +scene0191_02 +scene0119_00 +scene0230_00 +scene0528_00 +scene0528_01 +scene0705_00 +scene0705_01 +scene0705_02 +scene0415_00 +scene0415_01 +scene0415_02 +scene0007_00 +scene0141_00 +scene0141_01 +scene0141_02 +scene0515_00 +scene0515_01 +scene0515_02 +scene0447_00 +scene0447_01 +scene0447_02 +scene0531_00 +scene0503_00 +scene0285_00 +scene0069_00 +scene0584_00 +scene0584_01 +scene0584_02 +scene0581_00 +scene0581_01 +scene0581_02 +scene0620_00 +scene0620_01 +scene0263_00 +scene0263_01 +scene0481_00 +scene0481_01 +scene0020_00 +scene0020_01 +scene0291_00 +scene0291_01 +scene0291_02 +scene0469_00 +scene0469_01 +scene0469_02 +scene0659_00 +scene0659_01 +scene0024_00 +scene0024_01 +scene0024_02 +scene0564_00 +scene0117_00 +scene0027_00 +scene0027_01 +scene0027_02 +scene0028_00 +scene0330_00 +scene0418_00 +scene0418_01 +scene0418_02 +scene0233_00 +scene0233_01 +scene0673_00 +scene0673_01 +scene0673_02 +scene0673_03 +scene0673_04 +scene0673_05 +scene0585_00 +scene0585_01 +scene0362_00 +scene0362_01 +scene0362_02 +scene0362_03 +scene0035_00 +scene0035_01 +scene0358_00 +scene0358_01 +scene0358_02 +scene0037_00 +scene0194_00 +scene0321_00 +scene0293_00 +scene0293_01 +scene0623_00 +scene0623_01 +scene0592_00 +scene0592_01 +scene0569_00 +scene0569_01 +scene0413_00 +scene0313_00 +scene0313_01 +scene0313_02 +scene0480_00 +scene0480_01 +scene0401_00 +scene0517_00 +scene0517_01 +scene0517_02 +scene0032_00 +scene0032_01 +scene0613_00 +scene0613_01 +scene0613_02 +scene0306_00 +scene0306_01 +scene0052_00 +scene0052_01 +scene0052_02 +scene0053_00 +scene0444_00 +scene0444_01 +scene0055_00 +scene0055_01 +scene0055_02 +scene0560_00 +scene0589_00 +scene0589_01 +scene0589_02 +scene0610_00 +scene0610_01 +scene0610_02 +scene0364_00 +scene0364_01 +scene0383_00 +scene0383_01 +scene0383_02 +scene0006_00 +scene0006_01 +scene0006_02 +scene0275_00 +scene0451_00 +scene0451_01 +scene0451_02 +scene0451_03 +scene0451_04 +scene0451_05 +scene0135_00 +scene0065_00 +scene0065_01 +scene0065_02 +scene0104_00 +scene0674_00 +scene0674_01 +scene0448_00 +scene0448_01 +scene0448_02 +scene0502_00 +scene0502_01 +scene0502_02 +scene0440_00 +scene0440_01 +scene0440_02 +scene0071_00 +scene0072_00 +scene0072_01 +scene0072_02 +scene0509_00 +scene0509_01 +scene0509_02 +scene0649_00 +scene0649_01 +scene0602_00 +scene0694_00 +scene0694_01 +scene0101_00 +scene0101_01 +scene0101_02 +scene0101_03 +scene0101_04 +scene0101_05 +scene0218_00 +scene0218_01 +scene0579_00 +scene0579_01 +scene0579_02 +scene0039_00 +scene0039_01 +scene0493_00 +scene0493_01 +scene0242_00 +scene0242_01 +scene0242_02 +scene0083_00 +scene0083_01 +scene0127_00 +scene0127_01 +scene0662_00 +scene0662_01 +scene0662_02 +scene0018_00 +scene0087_00 +scene0087_01 +scene0087_02 +scene0332_00 +scene0332_01 +scene0332_02 +scene0628_00 +scene0628_01 +scene0628_02 +scene0134_00 +scene0134_01 +scene0134_02 +scene0238_00 +scene0238_01 +scene0092_00 +scene0092_01 +scene0092_02 +scene0092_03 +scene0092_04 +scene0022_00 +scene0022_01 +scene0467_00 +scene0392_00 +scene0392_01 +scene0392_02 +scene0424_00 +scene0424_01 +scene0424_02 +scene0646_00 +scene0646_01 +scene0646_02 +scene0098_00 +scene0098_01 +scene0044_00 +scene0044_01 +scene0044_02 +scene0510_00 +scene0510_01 +scene0510_02 +scene0571_00 +scene0571_01 +scene0166_00 +scene0166_01 +scene0166_02 +scene0563_00 +scene0172_00 +scene0172_01 +scene0388_00 +scene0388_01 +scene0215_00 +scene0215_01 +scene0252_00 +scene0287_00 +scene0668_00 +scene0572_00 +scene0572_01 +scene0572_02 +scene0026_00 +scene0224_00 +scene0113_00 +scene0113_01 +scene0551_00 +scene0381_00 +scene0381_01 +scene0381_02 +scene0371_00 +scene0371_01 +scene0460_00 +scene0118_00 +scene0118_01 +scene0118_02 +scene0417_00 +scene0008_00 +scene0634_00 +scene0521_00 +scene0123_00 +scene0123_01 +scene0123_02 +scene0045_00 +scene0045_01 +scene0511_00 +scene0511_01 +scene0114_00 +scene0114_01 +scene0114_02 +scene0070_00 +scene0029_00 +scene0029_01 +scene0029_02 +scene0129_00 +scene0103_00 +scene0103_01 +scene0002_00 +scene0002_01 +scene0132_00 +scene0132_01 +scene0132_02 +scene0124_00 +scene0124_01 +scene0143_00 +scene0143_01 +scene0143_02 +scene0604_00 +scene0604_01 +scene0604_02 +scene0507_00 +scene0105_00 +scene0105_01 +scene0105_02 +scene0428_00 +scene0428_01 +scene0311_00 +scene0140_00 +scene0140_01 +scene0182_00 +scene0182_01 +scene0182_02 +scene0142_00 +scene0142_01 +scene0399_00 +scene0399_01 +scene0012_00 +scene0012_01 +scene0012_02 +scene0060_00 +scene0060_01 +scene0370_00 +scene0370_01 +scene0370_02 +scene0310_00 +scene0310_01 +scene0310_02 +scene0661_00 +scene0650_00 +scene0152_00 +scene0152_01 +scene0152_02 +scene0158_00 +scene0158_01 +scene0158_02 +scene0482_00 +scene0482_01 +scene0600_00 +scene0600_01 +scene0600_02 +scene0393_00 +scene0393_01 +scene0393_02 +scene0562_00 +scene0174_00 +scene0174_01 +scene0157_00 +scene0157_01 +scene0161_00 +scene0161_01 +scene0161_02 +scene0159_00 +scene0254_00 +scene0254_01 +scene0115_00 +scene0115_01 +scene0115_02 +scene0162_00 +scene0163_00 +scene0163_01 +scene0523_00 +scene0523_01 +scene0523_02 +scene0459_00 +scene0459_01 +scene0175_00 +scene0085_00 +scene0085_01 +scene0279_00 +scene0279_01 +scene0279_02 +scene0201_00 +scene0201_01 +scene0201_02 +scene0283_00 +scene0456_00 +scene0456_01 +scene0429_00 +scene0043_00 +scene0043_01 +scene0419_00 +scene0419_01 +scene0419_02 +scene0368_00 +scene0368_01 +scene0348_00 +scene0348_01 +scene0348_02 +scene0442_00 +scene0178_00 +scene0380_00 +scene0380_01 +scene0380_02 +scene0165_00 +scene0165_01 +scene0165_02 +scene0181_00 +scene0181_01 +scene0181_02 +scene0181_03 +scene0333_00 +scene0614_00 +scene0614_01 +scene0614_02 +scene0404_00 +scene0404_01 +scene0404_02 +scene0185_00 +scene0126_00 +scene0126_01 +scene0126_02 +scene0519_00 +scene0236_00 +scene0236_01 +scene0189_00 +scene0075_00 +scene0267_00 +scene0192_00 +scene0192_01 +scene0192_02 +scene0281_00 +scene0420_00 +scene0420_01 +scene0420_02 +scene0195_00 +scene0195_01 +scene0195_02 +scene0597_00 +scene0597_01 +scene0597_02 +scene0041_00 +scene0041_01 +scene0111_00 +scene0111_01 +scene0111_02 +scene0666_00 +scene0666_01 +scene0666_02 +scene0200_00 +scene0200_01 +scene0200_02 +scene0536_00 +scene0536_01 +scene0536_02 +scene0390_00 +scene0280_00 +scene0280_01 +scene0280_02 +scene0344_00 +scene0344_01 +scene0205_00 +scene0205_01 +scene0205_02 +scene0484_00 +scene0484_01 +scene0009_00 +scene0009_01 +scene0009_02 +scene0302_00 +scene0302_01 +scene0209_00 +scene0209_01 +scene0209_02 +scene0210_00 +scene0210_01 +scene0395_00 +scene0395_01 +scene0395_02 +scene0683_00 +scene0601_00 +scene0601_01 +scene0214_00 +scene0214_01 +scene0214_02 +scene0477_00 +scene0477_01 +scene0439_00 +scene0439_01 +scene0468_00 +scene0468_01 +scene0468_02 +scene0546_00 +scene0466_00 +scene0466_01 +scene0220_00 +scene0220_01 +scene0220_02 +scene0122_00 +scene0122_01 +scene0130_00 +scene0110_00 +scene0110_01 +scene0110_02 +scene0327_00 +scene0156_00 +scene0266_00 +scene0266_01 +scene0001_00 +scene0001_01 +scene0228_00 +scene0199_00 +scene0219_00 +scene0464_00 +scene0232_00 +scene0232_01 +scene0232_02 +scene0299_00 +scene0299_01 +scene0530_00 +scene0363_00 +scene0453_00 +scene0453_01 +scene0570_00 +scene0570_01 +scene0570_02 +scene0183_00 +scene0239_00 +scene0239_01 +scene0239_02 +scene0373_00 +scene0373_01 +scene0241_00 +scene0241_01 +scene0241_02 +scene0188_00 +scene0622_00 +scene0622_01 +scene0244_00 +scene0244_01 +scene0691_00 +scene0691_01 +scene0206_00 +scene0206_01 +scene0206_02 +scene0247_00 +scene0247_01 +scene0061_00 +scene0061_01 +scene0082_00 +scene0250_00 +scene0250_01 +scene0250_02 +scene0501_00 +scene0501_01 +scene0501_02 +scene0320_00 +scene0320_01 +scene0320_02 +scene0320_03 +scene0631_00 +scene0631_01 +scene0631_02 +scene0255_00 +scene0255_01 +scene0255_02 +scene0047_00 +scene0265_00 +scene0265_01 +scene0265_02 +scene0004_00 +scene0336_00 +scene0336_01 +scene0058_00 +scene0058_01 +scene0260_00 +scene0260_01 +scene0260_02 +scene0243_00 +scene0603_00 +scene0603_01 +scene0093_00 +scene0093_01 +scene0093_02 +scene0109_00 +scene0109_01 +scene0434_00 +scene0434_01 +scene0434_02 +scene0290_00 +scene0627_00 +scene0627_01 +scene0470_00 +scene0470_01 +scene0137_00 +scene0137_01 +scene0137_02 +scene0270_00 +scene0270_01 +scene0270_02 +scene0271_00 +scene0271_01 +scene0504_00 +scene0274_00 +scene0274_01 +scene0274_02 +scene0036_00 +scene0036_01 +scene0276_00 +scene0276_01 +scene0272_00 +scene0272_01 +scene0499_00 +scene0698_00 +scene0698_01 +scene0051_00 +scene0051_01 +scene0051_02 +scene0051_03 +scene0108_00 +scene0245_00 +scene0369_00 +scene0369_01 +scene0369_02 +scene0284_00 +scene0289_00 +scene0289_01 +scene0286_00 +scene0286_01 +scene0286_02 +scene0286_03 +scene0031_00 +scene0031_01 +scene0031_02 +scene0545_00 +scene0545_01 +scene0545_02 +scene0557_00 +scene0557_01 +scene0557_02 +scene0533_00 +scene0533_01 +scene0116_00 +scene0116_01 +scene0116_02 +scene0611_00 +scene0611_01 +scene0688_00 +scene0294_00 +scene0294_01 +scene0294_02 +scene0295_00 +scene0295_01 +scene0296_00 +scene0296_01 +scene0596_00 +scene0596_01 +scene0596_02 +scene0532_00 +scene0532_01 +scene0637_00 +scene0638_00 +scene0121_00 +scene0121_01 +scene0121_02 +scene0040_00 +scene0040_01 +scene0197_00 +scene0197_01 +scene0197_02 +scene0410_00 +scene0410_01 +scene0305_00 +scene0305_01 +scene0615_00 +scene0615_01 +scene0703_00 +scene0703_01 +scene0555_00 +scene0297_00 +scene0297_01 +scene0297_02 +scene0582_00 +scene0582_01 +scene0582_02 +scene0023_00 +scene0094_00 +scene0013_00 +scene0013_01 +scene0013_02 +scene0136_00 +scene0136_01 +scene0136_02 +scene0407_00 +scene0407_01 +scene0062_00 +scene0062_01 +scene0062_02 +scene0386_00 +scene0318_00 +scene0554_00 +scene0554_01 +scene0497_00 +scene0213_00 +scene0258_00 +scene0323_00 +scene0323_01 +scene0324_00 +scene0324_01 +scene0016_00 +scene0016_01 +scene0016_02 +scene0681_00 +scene0398_00 +scene0398_01 +scene0227_00 +scene0090_00 +scene0066_00 +scene0262_00 +scene0262_01 +scene0155_00 +scene0155_01 +scene0155_02 +scene0352_00 +scene0352_01 +scene0352_02 +scene0038_00 +scene0038_01 +scene0038_02 +scene0335_00 +scene0335_01 +scene0335_02 +scene0261_00 +scene0261_01 +scene0261_02 +scene0261_03 +scene0640_00 +scene0640_01 +scene0640_02 +scene0080_00 +scene0080_01 +scene0080_02 +scene0403_00 +scene0403_01 +scene0282_00 +scene0282_01 +scene0282_02 +scene0682_00 +scene0173_00 +scene0173_01 +scene0173_02 +scene0522_00 +scene0687_00 +scene0345_00 +scene0345_01 +scene0612_00 +scene0612_01 +scene0411_00 +scene0411_01 +scene0411_02 +scene0625_00 +scene0625_01 +scene0211_00 +scene0211_01 +scene0211_02 +scene0211_03 +scene0676_00 +scene0676_01 +scene0179_00 +scene0498_00 +scene0498_01 +scene0498_02 +scene0547_00 +scene0547_01 +scene0547_02 +scene0269_00 +scene0269_01 +scene0269_02 +scene0366_00 +scene0680_00 +scene0680_01 +scene0588_00 +scene0588_01 +scene0588_02 +scene0588_03 +scene0346_00 +scene0346_01 +scene0359_00 +scene0359_01 +scene0014_00 +scene0120_00 +scene0120_01 +scene0212_00 +scene0212_01 +scene0212_02 +scene0176_00 +scene0049_00 +scene0259_00 +scene0259_01 +scene0586_00 +scene0586_01 +scene0586_02 +scene0309_00 +scene0309_01 +scene0125_00 +scene0455_00 +scene0177_00 +scene0177_01 +scene0177_02 +scene0326_00 +scene0372_00 +scene0171_00 +scene0171_01 +scene0374_00 +scene0654_00 +scene0654_01 +scene0445_00 +scene0445_01 +scene0475_00 +scene0475_01 +scene0475_02 +scene0349_00 +scene0349_01 +scene0234_00 +scene0669_00 +scene0669_01 +scene0375_00 +scene0375_01 +scene0375_02 +scene0387_00 +scene0387_01 +scene0387_02 +scene0312_00 +scene0312_01 +scene0312_02 +scene0384_00 +scene0385_00 +scene0385_01 +scene0385_02 +scene0000_00 +scene0000_01 +scene0000_02 +scene0376_00 +scene0376_01 +scene0376_02 +scene0301_00 +scene0301_01 +scene0301_02 +scene0322_00 +scene0542_00 +scene0079_00 +scene0079_01 +scene0099_00 +scene0099_01 +scene0476_00 +scene0476_01 +scene0476_02 +scene0394_00 +scene0394_01 +scene0147_00 +scene0147_01 +scene0067_00 +scene0067_01 +scene0067_02 +scene0397_00 +scene0397_01 +scene0337_00 +scene0337_01 +scene0337_02 +scene0431_00 +scene0223_00 +scene0223_01 +scene0223_02 +scene0010_00 +scene0010_01 +scene0402_00 +scene0268_00 +scene0268_01 +scene0268_02 +scene0679_00 +scene0679_01 +scene0405_00 +scene0128_00 +scene0408_00 +scene0408_01 +scene0190_00 +scene0107_00 +scene0076_00 +scene0167_00 +scene0361_00 +scene0361_01 +scene0361_02 +scene0216_00 +scene0202_00 +scene0303_00 +scene0303_01 +scene0303_02 +scene0446_00 +scene0446_01 +scene0089_00 +scene0089_01 +scene0089_02 +scene0360_00 +scene0150_00 +scene0150_01 +scene0150_02 +scene0421_00 +scene0421_01 +scene0421_02 +scene0454_00 +scene0626_00 +scene0626_01 +scene0626_02 +scene0186_00 +scene0186_01 +scene0538_00 +scene0479_00 +scene0479_01 +scene0479_02 +scene0656_00 +scene0656_01 +scene0656_02 +scene0656_03 +scene0525_00 +scene0525_01 +scene0525_02 +scene0308_00 +scene0396_00 +scene0396_01 +scene0396_02 +scene0624_00 +scene0292_00 +scene0292_01 +scene0632_00 +scene0253_00 +scene0021_00 +scene0325_00 +scene0325_01 +scene0437_00 +scene0437_01 +scene0438_00 +scene0590_00 +scene0590_01 +scene0400_00 +scene0400_01 +scene0541_00 +scene0541_01 +scene0541_02 +scene0677_00 +scene0677_01 +scene0677_02 +scene0443_00 +scene0315_00 +scene0288_00 +scene0288_01 +scene0288_02 +scene0422_00 +scene0672_00 +scene0672_01 +scene0184_00 +scene0449_00 +scene0449_01 +scene0449_02 +scene0048_00 +scene0048_01 +scene0138_00 +scene0452_00 +scene0452_01 +scene0452_02 +scene0667_00 +scene0667_01 +scene0667_02 +scene0463_00 +scene0463_01 +scene0078_00 +scene0078_01 +scene0078_02 +scene0636_00 +scene0457_00 +scene0457_01 +scene0457_02 +scene0465_00 +scene0465_01 +scene0577_00 +scene0151_00 +scene0151_01 +scene0339_00 +scene0573_00 +scene0573_01 +scene0154_00 +scene0096_00 +scene0096_01 +scene0096_02 +scene0235_00 +scene0168_00 +scene0168_01 +scene0168_02 +scene0594_00 +scene0587_00 +scene0587_01 +scene0587_02 +scene0587_03 +scene0229_00 +scene0229_01 +scene0229_02 +scene0512_00 +scene0106_00 +scene0106_01 +scene0106_02 +scene0472_00 +scene0472_01 +scene0472_02 +scene0489_00 +scene0489_01 +scene0489_02 +scene0425_00 +scene0425_01 +scene0641_00 +scene0526_00 +scene0526_01 +scene0317_00 +scene0317_01 +scene0544_00 +scene0017_00 +scene0017_01 +scene0017_02 +scene0042_00 +scene0042_01 +scene0042_02 +scene0576_00 +scene0576_01 +scene0576_02 +scene0347_00 +scene0347_01 +scene0347_02 +scene0436_00 +scene0226_00 +scene0226_01 +scene0485_00 +scene0486_00 +scene0487_00 +scene0487_01 +scene0619_00 +scene0097_00 +scene0367_00 +scene0367_01 +scene0491_00 +scene0492_00 +scene0492_01 +scene0005_00 +scene0005_01 +scene0543_00 +scene0543_01 +scene0543_02 +scene0657_00 +scene0341_00 +scene0341_01 diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv1_val.txt b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv1_val.txt new file mode 100644 index 0000000000000000000000000000000000000000..965ff258035f857446c30b10e9a6be49f71d3dc7 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv1_val.txt @@ -0,0 +1,156 @@ +scene0534_00 +scene0534_01 +scene0319_00 +scene0273_00 +scene0273_01 +scene0225_00 +scene0198_00 +scene0003_00 +scene0003_01 +scene0003_02 +scene0409_00 +scene0409_01 +scene0331_00 +scene0331_01 +scene0505_00 +scene0505_01 +scene0505_02 +scene0505_03 +scene0505_04 +scene0506_00 +scene0057_00 +scene0057_01 +scene0074_00 +scene0074_01 +scene0074_02 +scene0091_00 +scene0112_00 +scene0112_01 +scene0112_02 +scene0240_00 +scene0102_00 +scene0102_01 +scene0513_00 +scene0514_00 +scene0514_01 +scene0537_00 +scene0516_00 +scene0516_01 +scene0495_00 +scene0617_00 +scene0133_00 +scene0520_00 +scene0520_01 +scene0635_00 +scene0635_01 +scene0054_00 +scene0473_00 +scene0473_01 +scene0524_00 +scene0524_01 +scene0379_00 +scene0471_00 +scene0471_01 +scene0471_02 +scene0566_00 +scene0248_00 +scene0248_01 +scene0248_02 +scene0529_00 +scene0529_01 +scene0529_02 +scene0391_00 +scene0264_00 +scene0264_01 +scene0264_02 +scene0675_00 +scene0675_01 +scene0350_00 +scene0350_01 +scene0350_02 +scene0450_00 +scene0068_00 +scene0068_01 +scene0237_00 +scene0237_01 +scene0365_00 +scene0365_01 +scene0365_02 +scene0605_00 +scene0605_01 +scene0539_00 +scene0539_01 +scene0539_02 +scene0540_00 +scene0540_01 +scene0540_02 +scene0170_00 +scene0170_01 +scene0170_02 +scene0433_00 +scene0340_00 +scene0340_01 +scene0340_02 +scene0160_00 +scene0160_01 +scene0160_02 +scene0160_03 +scene0160_04 +scene0059_00 +scene0059_01 +scene0059_02 +scene0056_00 +scene0056_01 +scene0478_00 +scene0478_01 +scene0548_00 +scene0548_01 +scene0548_02 +scene0204_00 +scene0204_01 +scene0204_02 +scene0033_00 +scene0145_00 +scene0483_00 +scene0508_00 +scene0508_01 +scene0508_02 +scene0180_00 +scene0148_00 +scene0556_00 +scene0556_01 +scene0416_00 +scene0416_01 +scene0416_02 +scene0416_03 +scene0416_04 +scene0073_00 +scene0073_01 +scene0073_02 +scene0073_03 +scene0034_00 +scene0034_01 +scene0034_02 +scene0639_00 +scene0561_00 +scene0561_01 +scene0298_00 +scene0692_00 +scene0692_01 +scene0692_02 +scene0692_03 +scene0692_04 +scene0642_00 +scene0642_01 +scene0642_02 +scene0642_03 +scene0630_00 +scene0630_01 +scene0630_02 +scene0630_03 +scene0630_04 +scene0630_05 +scene0630_06 +scene0706_00 +scene0567_00 +scene0567_01 diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2-labels-old.combined.tsv b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2-labels-old.combined.tsv new file mode 100644 index 0000000000000000000000000000000000000000..05c006e98066aa78d126bebcfb3654200d351b93 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2-labels-old.combined.tsv @@ -0,0 +1,608 @@ +id raw_category category count nyu40id eigen13id nyuClass nyu40class eigen13class ModelNet40 ModelNet10 ShapeNetCore55 synsetoffset wnsynsetid wnsynsetkey mpcat40 mpcat40index +1 wall wall 8277 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +2 chair chair 4646 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +22 books book 1678 23 2 book books Books n02870526 book.n.11 objects 39 +3 floor floor 1553 2 5 floor floor Floor n03365592 floor.n.01 floor 2 +5 door door 1483 8 12 door door Wall door n03221720 door.n.01 door 4 +1163 object object 1313 40 7 otherprop Objects objects 39 +16 window window 1209 9 13 window window Window n04587648 window.n.01 window 9 +4 table table 1170 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +56 trash can trash can 1090 39 6 garbage bin otherfurniture Furniture trash_bin 2747177 n02747177 ashcan.n.01 objects 39 +13 pillow pillow 937 18 7 pillow pillow Objects pillow 3938244 n03938244 pillow.n.01 cushion 8 +15 picture picture 862 11 8 picture picture Picture n03931044 picture.n.01 picture 6 +41 ceiling ceiling 806 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 ceiling 17 +26 box box 775 29 7 box box Objects n02883344 box.n.01 objects 39 +161 doorframe doorframe 768 8 12 door door Wall door doorframe.n.01 door 4 +19 monitor monitor 765 40 7 monitor otherprop Objects monitor monitor tv or monitor 3211117 n03782190 monitor.n.04 objects 39 +7 cabinet cabinet 731 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +9 desk desk 680 14 10 desk desk Table desk desk table 4379243 n03179701 desk.n.01 table 5 +8 shelf shelf 641 15 6 shelves shelves Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +10 office chair office chair 595 5 4 chair chair Chair chair chair chair 3001627 n04373704 swivel_chair.n.01 chair 3 +31 towel towel 570 27 7 towel towel Objects n04459362 towel.n.01 towel 20 +6 couch couch 502 6 9 sofa sofa Sofa sofa sofa sofa 4256520 n04256520 sofa.n.01 sofa 10 +14 sink sink 488 34 7 sink sink Objects sink n04223580 sink.n.01 sink 15 +48 backpack backpack 479 40 7 backpack otherprop Objects n02769748 backpack.n.01 objects 39 +28 lamp lamp 419 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +11 bed bed 370 4 1 bed bed Bed bed bed bed 2818832 n02818832 bed.n.01 bed 11 +18 bookshelf bookshelf 360 10 6 bookshelf bookshelf Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +71 mirror mirror 349 19 7 mirror mirror Objects n03773035 mirror.n.01 mirror 21 +21 curtain curtain 347 16 13 curtain curtain Window curtain n03151077 curtain.n.01 curtain 12 +40 plant plant 331 40 7 plant otherprop Objects plant n00017222 plant.n.02 plant 14 +52 whiteboard whiteboard 327 30 7 whiteboard whiteboard Objects n03211616 display_panel.n.01 board_panel 35 +96 radiator radiator 322 39 6 radiator otherfurniture Furniture n04041069 radiator.n.02 misc 40 +22 book book 318 23 2 book books Books n02870526 book.n.11 objects 39 +29 kitchen cabinet kitchen cabinet 310 3 6 cabinet cabinet Furniture n02933112 cabinet.n.01 cabinet 7 +49 toilet paper toilet paper 291 40 7 toilet paper otherprop Objects n15075141 toilet_tissue.n.01 objects 39 +29 kitchen cabinets kitchen cabinet 289 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +23 armchair armchair 281 5 4 chair chair Chair chair chair chair 3001627 n02738535 armchair.n.01 chair 3 +63 shoes shoe 272 40 7 shoe otherprop Objects n04199027 shoe.n.01 clothes 38 +24 coffee table coffee table 258 7 10 coffee table table Table table table table 4379243 n03063968 coffee_table.n.01 table 5 +17 toilet toilet 256 33 7 toilet toilet Objects toilet toilet n04446276 toilet.n.01 toilet 18 +47 bag bag 252 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +32 clothes clothes 248 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +46 keyboard keyboard 246 40 7 keyboard otherprop Objects keyboard computer keyboard 3085013 n03085013 computer_keyboard.n.01 objects 39 +65 bottle bottle 226 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +97 recycling bin recycling bin 225 39 6 garbage bin otherfurniture Furniture trash_bin 2747177 n02747177 ashcan.n.01 objects 39 +34 nightstand nightstand 224 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 chest_of_drawers 13 +38 stool stool 221 40 7 stool otherprop Objects stool n04326896 stool.n.01 stool 19 +33 tv tv 219 25 11 television television TV tv or monitor 3211117 n03211117 display.n.06 tv_monitor 22 +75 file cabinet file cabinet 217 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +36 dresser dresser 213 17 6 dresser dresser Furniture dresser dresser n03015254 chest_of_drawers.n.01 chest_of_drawers 13 +64 computer tower computer tower 203 40 7 computer otherprop Objects n03082979 computer.n.01 objects 39 +32 clothing clothes 165 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +101 telephone telephone 164 40 7 telephone otherprop Objects telephone 4401088 n04401088 telephone.n.01 objects 39 +130 cup cup 157 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 +27 refrigerator refrigerator 154 24 6 refridgerator refridgerator Furniture n04070727 refrigerator.n.01 appliances 37 +44 end table end table 147 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +131 jacket jacket 146 40 7 jacket otherprop Objects n03589791 jacket.n.01 clothes 38 +55 shower curtain shower curtain 144 28 7 shower curtain shower curtain Objects curtain n04209239 shower_curtain.n.01 curtain 12 +42 bathtub bathtub 144 36 7 bathtub bathtub Objects bathtub bathtub tub 2808440 n02808440 bathtub.n.01 bathtub 25 +59 microwave microwave 141 40 7 microwave otherprop Objects microwave 3761084 n03761084 microwave.n.02 appliances 37 +159 kitchen counter kitchen counter 140 12 6 counter counter Furniture table table table 4379243 n03116530 counter.n.01 counter 26 +74 sofa chair sofa chair 129 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +82 paper towel dispenser paper towel dispenser 129 40 7 paper towel dispenser otherprop Objects objects 39 +1164 bathroom vanity bathroom vanity 126 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 table 5 +93 suitcase suitcase 118 40 7 luggage otherprop Objects n02773838 bag.n.06 objects 39 +77 laptop laptop 111 40 7 laptop otherprop Objects laptop laptop 3642806 n03642806 laptop.n.01 objects 39 +67 ottoman ottoman 111 39 6 ottoman otherfurniture Furniture stool n03380724 footstool.n.01 stool 19 +128 shower walls shower wall 109 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +50 printer printer 106 40 7 printer otherprop Objects printer 4004475 n04004475 printer.n.03 appliances 37 +35 counter counter 104 12 6 counter counter Furniture table table table 4379243 n03116530 counter.n.01 counter 26 +69 board board 100 38 7 board otherstructure Objects board_panel 35 +100 soap dispenser soap dispenser 99 40 7 otherprop Objects n04254120 soap_dispenser.n.01 objects 39 +62 stove stove 95 38 7 stove otherstructure Objects stove 4330267 n04330267 stove.n.02 appliances 37 +105 light light 93 38 7 light otherstructure Objects n03665366 light.n.02 lighting 28 +1165 closet wall closet wall 90 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +165 mini fridge mini fridge 87 24 6 refridgerator refridgerator Furniture n03273913 electric_refrigerator.n.01 appliances 37 +7 cabinets cabinet 79 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +5 doors door 76 8 12 door door Wall door n03221720 door.n.01 door 4 +76 fan fan 75 40 7 fan otherprop Objects n03320046 fan.n.01 misc 40 +230 tissue box tissue box 73 40 7 tissue box otherprop Objects n02883344 box.n.01 objects 39 +54 blanket blanket 72 40 7 blanket otherprop Objects n02849154 blanket.n.01 objects 39 +125 bathroom stall bathroom stall 71 38 7 otherstructure Objects n02873839 booth.n.02 misc 40 +72 copier copier 70 40 7 otherprop Objects n03257586 duplicator.n.01 appliances 37 +68 bench bench 66 39 6 bench otherfurniture Furniture bench bench 2828884 n02828884 bench.n.01 seating 34 +145 bar bar 66 38 7 bar otherstructure Objects n02788689 bar.n.03 misc 40 +157 soap dish soap dish 65 40 7 soap dish otherprop Objects n04254009 soap_dish.n.01 objects 39 +1166 laundry hamper laundry hamper 65 40 7 laundry basket otherprop Objects objects 39 +132 storage bin storage bin 63 40 7 storage bin otherprop Objects objects 39 +1167 bathroom stall door bathroom stall door 62 8 12 door door Wall door n03221720 door.n.01 door 4 +232 light switch light switch 61 38 7 light switch otherstructure Objects n04372370 switch.n.01 misc 40 +134 coffee maker coffee maker 61 40 7 otherprop Objects n03063338 coffee_maker.n.01 appliances 37 +51 tv stand tv stand 61 39 6 tv stand otherfurniture Furniture tv_stand n03290653 entertainment_center.n.01 furniture 36 +250 decoration decoration 60 40 7 otherprop Objects n03169390 decoration.n.01 misc 40 +1168 ceiling light ceiling light 59 38 7 light otherstructure Objects n03665366 light.n.02 lighting 28 +342 range hood range hood 59 38 7 range hood otherstructure Objects range_hood n04053677 range_hood.n.01 misc 40 +89 blackboard blackboard 58 38 7 blackboard otherstructure Objects n02846511 blackboard.n.01 board_panel 35 +103 clock clock 58 40 7 clock otherprop Objects clock 3046257 n03046257 clock.n.01 objects 39 +99 wardrobe closet wardrobe 54 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 furniture 36 +95 rail rail 53 38 7 railing otherstructure Objects n04047401 railing.n.01 railing 30 +154 bulletin board bulletin board 53 38 7 board otherstructure Objects n03211616 display_panel.n.01 board_panel 35 +140 mat mat 52 20 5 floor mat floor mat Floor n03727837 mat.n.01 floor 2 +1169 trash bin trash bin 52 39 6 garbage bin otherfurniture Furniture trash_bin 2747177 n02747177 ashcan.n.01 objects 39 +193 ledge ledge 51 38 7 otherstructure Objects n09337253 ledge.n.01 misc 40 +116 seat seat 49 39 6 furniture otherfurniture Furniture n04161981 seat.n.03 furniture 36 +202 mouse mouse 49 40 7 mouse otherprop Objects n03793489 mouse.n.04 objects 39 +73 basket basket 48 40 7 basket otherprop Objects basket 2801938 n02801938 basket.n.01 objects 39 +78 shower shower 48 38 7 otherstructure Objects n04208936 shower.n.01 shower 23 +1170 dumbbell dumbbell 48 40 7 otherprop Objects n03255030 dumbbell.n.01 objects 39 +79 paper paper 46 26 7 paper paper Objects n14974264 paper.n.01 objects 39 +80 person person 46 31 7 person person Objects person n05217688 person.n.02 misc 40 +141 windowsill windowsill 45 38 7 otherstructure Objects n04590263 windowsill.n.01 window 9 +57 closet closet 45 39 6 wardrobe otherfurniture Furniture wardrobe misc 40 +102 bucket bucket 45 40 7 bucket otherprop Objects n02909870 bucket.n.01 misc 40 +261 sign sign 44 40 7 sign otherprop Objects n04217882 signboard.n.01 objects 39 +118 speaker speaker 43 40 7 speaker otherprop Objects speaker 3691459 n03691459 loudspeaker.n.01 objects 39 +136 dishwasher dishwasher 43 38 7 dishwasher otherstructure Objects dishwasher 3207941 n03207941 dishwasher.n.01 appliances 37 +98 container container 43 40 7 container otherprop Objects n03094503 container.n.01 objects 39 +1171 stair rail stair rail 42 38 7 banister otherstructure Objects n02788148 bannister.n.02 railing 30 +170 shower curtain rod shower curtain rod 42 40 7 otherprop Objects curtain 12 +1172 tube tube 41 40 7 otherprop Objects misc 40 +1173 bathroom cabinet bathroom cabinet 39 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +79 papers paper 39 26 7 paper paper Objects n14974264 paper.n.01 objects 39 +221 storage container storage container 39 40 7 container otherprop Objects objects 39 +570 paper bag paper bag 39 37 7 bag bag Objects n04122825 sack.n.01 objects 39 +138 paper towel roll paper towel roll 39 40 7 paper towel otherprop Objects n03887697 paper_towel.n.01 towel 20 +168 ball ball 39 40 7 ball otherprop Objects objects 39 +276 closet doors closet door 38 8 12 door door Wall door n03221720 door.n.01 door 4 +106 laundry basket laundry basket 37 40 7 laundry basket otherprop Objects basket 2801938 n03050864 clothes_hamper.n.01 objects 39 +214 cart cart 37 40 7 cart otherprop Objects n03484083 handcart.n.01 shelving 31 +276 closet door closet door 35 8 12 door door Wall door n03221720 door.n.01 door 4 +323 dish rack dish rack 35 40 7 dish rack otherprop Objects n03207630 dish_rack.n.01 objects 39 +58 stairs stairs 35 38 7 stairs otherstructure Objects n04298308 stairway.n.01 stairs 16 +86 blinds blinds 35 13 13 blinds blinds Window n02851099 blind.n.03 blinds 32 +2 stack of chairs chair 35 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +399 purse purse 34 40 7 purse otherprop Objects n02774152 bag.n.04 objects 39 +121 bicycle bicycle 33 40 7 bicycle otherprop Objects bicycle 2834778 n02834778 bicycle.n.01 objects 39 +185 tray tray 32 40 7 tray otherprop Objects n04476259 tray.n.01 objects 39 +300 plunger plunger 30 40 7 otherprop Objects n03970156 plunger.n.03 objects 39 +180 paper cutter paper cutter 30 40 7 paper cutter otherprop Objects n03886940 paper_cutter.n.01 objects 39 +163 toilet paper dispenser toilet paper dispenser 29 40 7 otherprop Objects objects 39 +26 boxes box 29 29 7 box box Objects n02883344 box.n.01 objects 39 +66 bin bin 28 40 7 bin otherprop Objects n02839910 bin.n.01 objects 39 +208 toilet seat cover dispenser toilet seat cover dispenser 28 40 7 otherprop Objects objects 39 +112 guitar guitar 28 40 7 guitar otherprop Objects guitar guitar 3467517 n03467517 guitar.n.01 objects 39 +540 mailboxes mailbox 28 29 7 box box Objects mailbox 3710193 n03710193 mailbox.n.01 misc 40 +395 handicap bar handicap bar 27 38 7 bar otherstructure Objects misc 40 +166 fire extinguisher fire extinguisher 27 40 7 fire extinguisher otherprop Objects n03345837 fire_extinguisher.n.01 misc 40 +122 ladder ladder 27 39 6 ladder otherfurniture Furniture stairs n03632277 ladder.n.01 stairs 16 +120 column column 26 38 7 column otherstructure Objects n03074380 column.n.06 column 24 +107 pipe pipe 25 40 7 pipe otherprop Objects n03944672 pipe.n.02 misc 40 +283 vacuum cleaner vacuum cleaner 25 40 7 otherprop Objects n04517823 vacuum.n.04 objects 39 +88 plate plate 24 40 7 plate otherprop Objects n03959485 plate.n.04 objects 39 +90 piano piano 24 39 6 piano otherfurniture Furniture piano piano 3928116 n03928116 piano.n.01 furniture 36 +177 water cooler water cooler 24 39 6 water cooler otherfurniture Furniture n04559166 water_cooler.n.01 misc 40 +1174 cd case cd case 24 40 7 otherprop Objects objects 39 +562 bowl bowl 24 40 7 bowl otherprop Objects bowl bowl 2880940 n02880940 bowl.n.03 objects 39 +1175 closet rod closet rod 24 40 7 otherprop Objects n04100174 rod.n.01 misc 40 +1156 bathroom counter bathroom counter 24 12 6 counter counter Furniture table table table 4379243 n03116530 counter.n.01 counter 26 +84 oven oven 23 38 7 oven otherstructure Objects n03862676 oven.n.01 appliances 37 +104 stand stand 23 39 6 stand otherfurniture Furniture table table table 4379243 n04301000 stand.n.04 table 5 +229 scale scale 23 40 7 scale otherprop Objects n04141975 scale.n.07 objects 39 +70 washing machine washing machine 23 39 6 washing machine otherfurniture Furniture washing_machine 4554684 n04554684 washer.n.03 appliances 37 +325 broom broom 22 40 7 broom otherprop Objects n02906734 broom.n.01 objects 39 +169 hat hat 22 40 7 hat otherprop Objects n03497657 hat.n.01 clothes 38 +128 shower wall shower wall 22 1 12 wall wall Wall n04208936 shower.n.01 wall 1 +331 guitar case guitar case 21 40 7 guitar case otherprop Objects objects 39 +87 rack rack 21 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +488 water pitcher water pitcher 21 40 7 pitcher otherprop Objects n03950228 pitcher.n.02 objects 39 +776 laundry detergent laundry detergent 21 40 7 otherprop Objects objects 39 +370 hair dryer hair dryer 21 40 7 hair dryer otherprop Objects n03483316 hand_blower.n.01 objects 39 +191 pillar pillar 21 38 7 column otherstructure Objects n03073977 column.n.07 column 24 +748 divider divider 20 40 7 otherprop Objects wall 1 +242 power outlet power outlet 19 40 7 otherprop Objects misc 40 +45 dining table dining table 19 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +417 shower floor shower floor 19 2 5 floor floor Floor n04208936 shower.n.01 floor 2 +70 washing machines washing machine 19 39 6 washing machine otherfurniture Furniture washing_machine 4554684 n04554684 washer.n.03 appliances 37 +188 shower door shower door 19 8 12 door door Wall door n04208936 shower.n.01 door 4 +1176 coffee kettle coffee kettle 18 40 7 pot otherprop Objects n03612814 kettle.n.01 objects 39 +1177 wardrobe cabinet wardrobe 18 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 furniture 36 +1178 structure structure 18 38 7 otherstructure Objects misc 40 +18 bookshelves bookshelf 17 10 6 bookshelf bookshelf Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +110 clothes dryer clothes dryer 17 39 6 otherfurniture Furniture n03251766 dryer.n.01 appliances 37 +148 toaster toaster 17 40 7 toaster otherprop Objects n04442312 toaster.n.02 appliances 37 +63 shoe shoe 17 40 7 shoe otherprop Objects n04199027 shoe.n.01 clothes 38 +155 ironing board ironing board 16 39 6 ironing board otherfurniture Furniture n03586090 ironing_board.n.01 objects 39 +572 alarm clock alarm clock 16 40 7 alarm clock otherprop Objects clock 3046257 n02694662 alarm_clock.n.01 objects 39 +1179 shower head shower head 15 38 7 otherstructure Objects shower 23 +28 lamp base lamp 15 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +392 water bottle water bottle 15 40 7 bottle otherprop Objects bottle bottle 2876657 n04557648 water_bottle.n.01 objects 39 +1180 keyboard piano keyboard piano 15 39 6 piano otherfurniture Furniture piano piano 3928116 n03928116 piano.n.01 furniture 36 +609 projector screen projector screen 15 38 7 projector screen otherstructure Objects misc 40 +1181 case of water bottles case of water bottles 15 40 7 otherprop Objects objects 39 +195 toaster oven toaster oven 14 40 7 toaster oven otherprop Objects n04442441 toaster_oven.n.01 appliances 37 +581 music stand music stand 14 39 6 music stand otherfurniture Furniture n03801760 music_stand.n.01 furniture 36 +58 staircase stairs 14 38 7 stairs otherstructure Objects n04298308 stairway.n.01 stairs 16 +1182 coat rack coat rack 14 40 7 otherprop Objects n03059103 coatrack.n.01 shelving 3 +1183 storage organizer storage organizer 14 40 7 otherprop Objects shelving 3 +139 machine machine 14 40 7 machine otherprop Objects n03699975 machine.n.01 appliances 37 +1184 folded chair folded chair 14 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +1185 fire alarm fire alarm 14 40 7 otherprop Objects n03343737 fire_alarm.n.02 misc 40 +156 fireplace fireplace 13 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 fireplace 27 +408 vent vent 13 40 7 otherprop Objects n04526241 vent.n.01 misc 40 +213 furniture furniture 13 39 6 furniture otherfurniture Furniture n03405725 furniture.n.01 furniture 36 +1186 power strip power strip 13 40 7 otherprop Objects objects 39 +1187 calendar calendar 13 40 7 otherprop Objects objects 39 +1188 poster poster 13 11 8 picture picture Picture n03931044 picture.n.01 picture 6 +115 toilet paper holder toilet paper holder 13 40 7 toilet paper holder otherprop Objects objects 39 +1189 potted plant potted plant 12 40 7 plant otherprop Objects plant n00017222 plant.n.02 plant 14 +304 stuffed animal stuffed animal 12 40 7 stuffed animal otherprop Objects n04399382 teddy.n.01 objects 39 +1190 luggage luggage 12 40 7 luggage otherprop Objects n02774630 baggage.n.01 objects 39 +21 curtains curtain 12 16 13 curtain curtain Window curtain n03151077 curtain.n.01 curtain 12 +312 headphones headphones 12 40 7 otherprop Objects n03261776 earphone.n.01 objects 39 +233 crate crate 12 39 6 crate otherfurniture Furniture n03127925 crate.n.01 objects 39 +286 candle candle 12 40 7 candle otherprop Objects lamp n02948072 candle.n.01 objects 39 +264 projector projector 12 40 7 projector otherprop Objects n04009552 projector.n.02 objects 39 +110 clothes dryers clothes dryer 12 39 6 otherfurniture Furniture n03251766 dryer.n.01 appliances 37 +1191 mattress mattress 12 4 1 bed bed Bed bed bed bed 2818832 n02818832 bed.n.01 bed 11 +356 dustpan dustpan 12 40 7 otherprop Objects n03259009 dustpan.n.02 objects 39 +25 drawer drawer 11 39 6 drawer otherfurniture Furniture n03233905 drawer.n.01 furniture 36 +750 rod rod 11 40 7 otherprop Objects pistol 3948459 n03427202 gat.n.01 misc 40 +269 globe globe 11 40 7 globe otherprop Objects objects 39 +307 footrest footrest 11 39 6 foot rest otherfurniture Furniture stool n03380724 footstool.n.01 stool 19 +410 piano bench piano bench 11 39 6 piano bench otherfurniture Furniture bench bench 2828884 n02828884 bench.n.01 seating 34 +730 breakfast bar breakfast bar 11 38 7 bar otherstructure Objects counter 26 +216 step stool step stool 11 40 7 step stool otherprop Objects stool n04315713 step_stool.n.01 stool 19 +1192 hand rail hand rail 11 38 7 railing otherstructure Objects railing 30 +119 vending machine vending machine 11 40 7 machine otherprop Objects n04525305 vending_machine.n.01 appliances 37 +682 ceiling fan ceiling fan 11 40 7 fan otherprop Objects n03320046 fan.n.01 misc 40 +434 swiffer swiffer 11 40 7 otherprop Objects objects 39 +126 foosball table foosball table 11 39 6 foosball table otherfurniture Furniture table table table 4379243 n04379243 table.n.02 table 5 +919 jar jar 11 40 7 jar otherprop Objects jar 3593526 n03593526 jar.n.01 objects 39 +85 footstool footstool 11 39 6 ottoman otherfurniture Furniture stool n03380724 footstool.n.01 stool 19 +1193 folded table folded table 10 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +108 round table round table 10 7 10 table table Table table table table 4379243 n04114554 round_table.n.02 table 5 +135 hamper hamper 10 40 7 basket otherprop Objects basket 2801938 n03482405 hamper.n.02 objects 39 +1194 poster tube poster tube 10 40 7 otherprop Objects objects 39 +432 case case 10 40 7 case otherprop Objects objects 39 +53 carpet carpet 10 40 7 rug otherprop Objects n04118021 rug.n.01 floor 2 +1195 thermostat thermostat 10 40 7 otherprop Objects n04422875 thermostat.n.01 misc 40 +111 coat coat 10 40 7 jacket otherprop Objects n03057021 coat.n.01 clothes 38 +305 water fountain water fountain 10 38 7 water fountain otherstructure Objects n03241335 drinking_fountain.n.01 misc 40 +1125 smoke detector smoke detector 10 40 7 otherprop Objects misc 40 +13 pillows pillow 9 18 7 pillow pillow Objects pillow 3938244 n03938244 pillow.n.01 cushion 8 +1196 flip flops flip flops 9 40 7 shoe otherprop Objects n04199027 shoe.n.01 clothes 38 +1197 cloth cloth 9 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +1198 banner banner 9 40 7 otherprop Objects n02788021 banner.n.01 misc 40 +1199 clothes hanger clothes hanger 9 40 7 otherprop Objects n03057920 coat_hanger.n.01 objects 39 +1200 whiteboard eraser whiteboard eraser 9 40 7 otherprop Objects objects 39 +378 iron iron 9 40 7 otherprop Objects n03584829 iron.n.04 objects 39 +591 instrument case instrument case 9 40 7 case otherprop Objects objects 39 +49 toilet paper rolls toilet paper 9 40 7 toilet paper otherprop Objects n15075141 toilet_tissue.n.01 objects 39 +92 soap soap 9 40 7 soap otherprop Objects n04253437 soap.n.01 objects 39 +1098 block block 9 40 7 otherprop Objects misc 40 +291 wall hanging wall hanging 8 40 7 otherprop Objects n03491178 hanging.n.01 picture 6 +1063 kitchen island kitchen island 8 38 7 kitchen island otherstructure Objects n03620600 kitchen_island.n.01 counter 26 +107 pipes pipe 8 38 7 otherstructure Objects misc 40 +1135 toothbrush toothbrush 8 40 7 toothbrush otherprop Objects n04453156 toothbrush.n.01 objects 39 +189 shirt shirt 8 40 7 otherprop Objects n04197391 shirt.n.01 clothes 38 +245 cutting board cutting board 8 40 7 cutting board otherprop Objects n03025513 chopping_board.n.01 objects 39 +194 vase vase 8 40 7 vase otherprop Objects vase jar 3593526 n04522168 vase.n.01 objects 39 +1201 shower control valve shower control valve 8 38 7 otherstructure Objects n04208936 shower.n.01 shower 23 +386 exercise machine exercise machine 8 40 7 machine otherprop Objects gym_equipment 33 +1202 compost bin compost bin 8 39 6 garbage bin otherfurniture Furniture trash_bin 2747177 n02747177 ashcan.n.01 objects 39 +857 shorts shorts 8 40 7 shorts otherprop Objects clothes 38 +452 tire tire 8 40 7 otherprop Objects n04440749 tire.n.01 objects 39 +1203 teddy bear teddy bear 7 40 7 stuffed animal otherprop Objects n04399382 teddy.n.01 objects 39 +346 bathrobe bathrobe 7 40 7 otherprop Objects n02807616 bathrobe.n.01 clothes 38 +152 handrail handrail 7 38 7 railing otherstructure Objects n02788148 bannister.n.02 railing 30 +83 faucet faucet 7 40 7 faucet otherprop Objects faucet 3325088 n03325088 faucet.n.01 misc 40 +1204 pantry wall pantry wall 7 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +726 thermos thermos 7 40 7 flask otherprop Objects bottle bottle 2876657 n04422727 thermos.n.01 objects 39 +61 rug rug 7 40 7 rug otherprop Objects n04118021 rug.n.01 floor 2 +39 couch cushions cushion 7 18 7 pillow pillow Objects n03151500 cushion.n.03 cushion 8 +1117 tripod tripod 7 39 6 stand otherfurniture Furniture n04485082 tripod.n.01 objects 39 +540 mailbox mailbox 7 29 7 box box Objects mailbox 3710193 n03710193 mailbox.n.01 misc 40 +1205 tupperware tupperware 7 40 7 otherprop Objects objects 39 +415 shoe rack shoe rack 7 40 7 shoe rack otherprop Objects shelving 31 +31 towels towel 6 27 7 towel towel Objects n04459362 towel.n.01 towel 20 +1206 beer bottles beer bottle 6 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +153 treadmill treadmill 6 39 6 treadmill otherfurniture Furniture n04477387 treadmill.n.01 gym_equipment 33 +1207 salt salt 6 40 7 otherprop Objects objects 39 +129 chest chest 6 39 6 chest otherfurniture Furniture dresser dresser chest_of_drawers 13 +220 dispenser dispenser 6 40 7 otherprop Objects n03210683 dispenser.n.01 objects 39 +1208 mirror doors mirror door 6 8 12 door door Wall door n03221720 door.n.01 door 4 +231 remote remote 6 40 7 otherprop Objects remote_control 4074963 n04074963 remote_control.n.01 objects 39 +1209 folded ladder folded ladder 6 39 6 ladder otherfurniture Furniture stairs n03632277 ladder.n.01 misc 40 +39 cushion cushion 6 18 7 pillow pillow Objects n03151500 cushion.n.03 cushion 8 +1210 carton carton 6 40 7 otherprop Objects objects 39 +117 step step 6 38 7 otherstructure Objects n04314914 step.n.04 misc 40 +822 drying rack drying rack 6 39 6 drying rack otherfurniture Furniture shelving 31 +238 slippers slipper 6 40 7 shoe otherprop Objects n04241394 slipper.n.01 clothes 38 +143 pool table pool table 6 39 6 pool table otherfurniture Furniture table table table 4379243 n03982430 pool_table.n.01 table 5 +1211 soda stream soda stream 6 40 7 otherprop Objects objects 39 +228 toilet brush toilet brush 6 40 7 toilet brush otherprop Objects objects 39 +494 loft bed loft bed 6 4 1 bed bed Bed bed bed bed 2818832 n02818832 bed.n.01 bed 11 +226 cooking pot cooking pot 6 40 7 pot otherprop Objects objects 39 +91 heater heater 6 39 6 heater otherfurniture Furniture n03508101 heater.n.01 misc 40 +1072 messenger bag messenger bag 6 37 7 bag bag Objects objects 39 +435 stapler stapler 6 40 7 stapler otherprop Objects n04303497 stapler.n.01 objects 39 +1165 closet walls closet wall 5 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +345 scanner scanner 5 40 7 otherprop Objects appliances 37 +893 elliptical machine elliptical machine 5 40 7 machine otherprop Objects gym_equipment 33 +621 kettle kettle 5 40 7 pot otherprop Objects n03612814 kettle.n.01 objects 39 +1212 metronome metronome 5 40 7 otherprop Objects n03757604 metronome.n.01 objects 39 +297 dumbell dumbell 5 40 7 otherprop Objects objects 39 +1213 music book music book 5 23 2 book books Books n02870526 book.n.11 objects 39 +1214 rice cooker rice cooker 5 40 7 otherprop Objects objects 39 +1215 dart board dart board 5 38 7 board otherstructure Objects n03162940 dartboard.n.01 objects 39 +529 sewing machine sewing machine 5 40 7 sewing machine otherprop Objects n04179913 sewing_machine.n.01 objects 39 +1216 grab bar grab bar 5 38 7 railing otherstructure Objects railing 30 +1217 flowerpot flowerpot 5 40 7 vase otherprop Objects vase jar 3593526 n04522168 vase.n.01 objects 39 +1218 painting painting 5 11 8 picture picture Picture n03931044 picture.n.01 picture 6 +1219 railing railing 5 38 7 railing otherstructure Objects n04047401 railing.n.01 railing 30 +1220 stair stair 5 38 7 stairs otherstructure Objects stairs n04314914 step.n.04 stairs 16 +525 toolbox toolbox 5 39 6 chest otherfurniture Furniture n04452615 toolbox.n.01 objects 39 +204 nerf gun nerf gun 5 40 7 otherprop Objects objects 39 +693 binders binder 5 40 7 binder otherprop Objects objects 39 +179 desk lamp desk lamp 5 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +1221 quadcopter quadcopter 5 40 7 otherprop Objects objects 39 +1222 pitcher pitcher 5 40 7 pitcher otherprop Objects n03950228 pitcher.n.02 objects 39 +1223 hanging hanging 5 40 7 otherprop Objects misc 40 +1224 mail mail 5 40 7 otherprop Objects misc 40 +1225 closet ceiling closet ceiling 5 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 ceiling 17 +1226 hoverboard hoverboard 5 40 7 otherprop Objects objects 39 +1227 beanbag chair beanbag chair 5 39 6 bean bag otherfurniture Furniture n02816656 beanbag.n.01 chair 3 +571 water heater water heater 5 40 7 water heater otherprop Objects n04560113 water_heater.n.01 misc 40 +1228 spray bottle spray bottle 5 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +556 rope rope 5 40 7 rope otherprop Objects n04108268 rope.n.01 objects 39 +280 plastic container plastic container 5 40 7 container otherprop Objects objects 39 +1229 soap bottle soap bottle 5 40 7 soap otherprop Objects objects 39 +1230 ikea bag ikea bag 4 37 7 bag bag Objects 2773838 n02773838 bag.n.06 objects 39 +1231 sleeping bag sleeping bag 4 40 7 otherprop Objects n04235860 sleeping_bag.n.01 objects 39 +1232 duffel bag duffel bag 4 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +746 frying pan frying pan 4 40 7 frying pan otherprop Objects n03400231 frying_pan.n.01 objects 39 +1233 oven mitt oven mitt 4 40 7 otherprop Objects objects 39 +1234 pot pot 4 40 7 pot otherprop Objects n04235860 sleeping_bag.n.01 objects 39 +144 hand dryer hand dryer 4 40 7 otherprop Objects objects 39 +282 dollhouse dollhouse 4 39 6 doll house otherfurniture Furniture n03219483 dollhouse.n.01 objects 39 +167 shampoo bottle shampoo bottle 4 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +1235 hair brush hair brush 4 40 7 otherprop Objects n02908217 brush.n.02 objects 39 +1236 tennis racket tennis racket 4 40 7 otherprop Objects n04409806 tennis_racket.n.01 objects 39 +1237 display case display case 4 40 7 case otherprop Objects objects 39 +234 ping pong table ping pong table 4 39 6 ping pong table otherfurniture Furniture table table table 4379243 n04379243 table.n.02 table 5 +563 boiler boiler 4 40 7 otherprop Objects misc 40 +1238 bag of coffee beans bag of coffee beans 4 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +1239 bananas banana 4 40 7 otherprop Objects n00021265 food.n.01 objects 39 +1240 carseat carseat 4 40 7 otherprop Objects misc 40 +366 helmet helmet 4 40 7 otherprop Objects helmet 3513137 n03513137 helmet.n.02 clothes 38 +816 umbrella umbrella 4 40 7 umbrella otherprop Objects n04507155 umbrella.n.01 objects 39 +1241 coffee box coffee box 4 40 7 otherprop Objects objects 39 +719 envelope envelope 4 40 7 envelope otherprop Objects n03291819 envelope.n.01 objects 39 +284 wet floor sign wet floor sign 4 40 7 sign otherprop Objects misc 40 +1242 clothing rack clothing rack 4 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +247 controller controller 4 40 7 otherprop Objects n03096960 control.n.09 objects 39 +1243 bath walls bathroom wall 4 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +1244 podium podium 4 39 6 otherfurniture Furniture n03159640 dais.n.01 furniture 36 +1245 storage box storage box 4 29 7 box box Objects n02883344 box.n.01 objects 39 +1246 dolly dolly 4 40 7 otherprop Objects misc 40 +1247 shampoo shampoo 3 40 7 otherprop Objects n04183516 shampoo.n.01 objects 39 +592 paper tray paper tray 3 40 7 paper tray otherprop Objects objects 39 +385 cabinet door cabinet door 3 8 12 door door Wall door door 4 +1248 changing station changing station 3 40 7 otherprop Objects misc 40 +1249 poster printer poster printer 3 40 7 printer otherprop Objects printer 4004475 n04004475 printer.n.03 appliances 37 +133 screen screen 3 40 7 otherprop Objects n03151077 curtain.n.01 curtain 12 +301 soap bar soap bar 3 38 7 bar otherstructure Objects objects 39 +1250 crutches crutches 3 40 7 otherprop Objects n03141823 crutch.n.01 objects 39 +379 studio light studio light 3 38 7 light otherstructure Objects lighting 28 +130 stack of cups cup 3 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 +1251 toilet flush button toilet flush button 3 40 7 otherprop Objects objects 39 +450 trunk trunk 3 40 7 otherprop Objects misc 40 +1252 grocery bag grocery bag 3 37 7 bag bag Objects suitcase 2773838 n03461288 grocery_bag.n.01 objects 39 +316 plastic bin plastic bin 3 40 7 bin otherprop Objects objects 39 +1253 pizza box pizza box 3 29 7 box box Objects objects 39 +385 cabinet doors cabinet door 3 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 door 4 +1254 legs legs 3 31 7 person person Objects person n05217688 person.n.02 misc 40 +461 car car 3 40 7 car otherprop Objects car car 2958343 n02958343 car.n.01 misc 40 +1255 shaving cream shaving cream 3 40 7 otherprop Objects n04186051 shaving_cream.n.01 objects 39 +1256 luggage stand luggage stand 3 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +599 shredder shredder 3 40 7 otherprop Objects n04210120 shredder.n.01 objects 39 +281 statue statue 3 40 7 sculpture otherprop Objects n04306847 statue.n.01 misc 40 +1257 urinal urinal 3 33 7 toilet toilet Objects toilet toilet n04515991 urinal.n.01 toilet 18 +1258 hose hose 3 40 7 otherprop Objects n03539875 hose.n.03 misc 40 +1259 bike pump bike pump 3 40 7 otherprop Objects objects 39 +319 coatrack coatrack 3 40 7 otherprop Objects n03059103 coatrack.n.01 shelving 31 +1260 bear bear 3 40 7 otherprop Objects objects 39 +28 wall lamp lamp 3 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +1261 humidifier humidifier 3 40 7 otherprop Objects objects 39 +546 toothpaste toothpaste 3 40 7 toothpaste otherprop Objects objects 39 +1262 mouthwash bottle mouthwash bottle 3 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +1263 poster cutter poster cutter 3 40 7 otherprop Objects objects 39 +1264 golf bag golf bag 3 37 7 bag bag Objects suitcase 2773838 n03445617 golf_bag.n.01 objects 39 +1265 food container food container 3 40 7 container otherprop Objects n03094503 container.n.01 objects 39 +1266 camera camera 3 40 7 otherprop Objects objects 39 +28 table lamp lamp 3 35 7 lamp lamp Objects lamp lamp 3636649 n04380533 table_lamp.n.01 lighting 28 +1267 yoga mat yoga mat 3 20 5 floor mat floor mat Floor n03727837 mat.n.01 floor 2 +1268 card card 3 40 7 otherprop Objects objects 39 +1269 mug mug 3 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 +188 shower doors shower door 3 38 7 otherstructure Objects n04208936 shower.n.01 door 4 +689 cardboard cardboard 3 40 7 otherprop Objects objects 39 +1270 rack stand rack stand 3 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +1271 boxes of paper boxes of paper 3 29 7 box box Objects n02883344 box.n.01 objects 39 +1272 flag flag 3 40 7 otherprop Objects misc 40 +354 futon futon 3 39 6 mattress otherfurniture Furniture n03408444 futon.n.01 sofa 10 +339 magazine magazine 3 40 7 magazine otherprop Objects n06595351 magazine.n.01 objects 39 +1009 exit sign exit sign 3 40 7 exit sign otherprop Objects misc 40 +1273 rolled poster rolled poster 3 40 7 otherprop Objects objects 39 +1274 wheel wheel 3 40 7 otherprop Objects objects 39 +15 pictures picture 3 11 8 picture picture Picture n03931044 picture.n.01 picture 6 +1275 blackboard eraser blackboard eraser 3 40 7 eraser otherprop Objects n03294833 eraser.n.01 objects 39 +361 organizer organizer 3 40 7 otherprop Objects n03918737 personal_digital_assistant.n.01 objects 39 +1276 doll doll 3 40 7 toy otherprop Objects n03219135 doll.n.01 objects 39 +326 book rack book rack 3 39 6 bookrack otherfurniture Furniture objects 39 +1277 laundry bag laundry bag 3 40 7 laundry basket otherprop Objects basket 2801938 n03050864 clothes_hamper.n.01 objects 39 +1278 sponge sponge 3 40 7 otherprop Objects n01906749 sponge.n.04 objects 39 +116 seating seat 3 39 6 furniture otherfurniture Furniture n04161981 seat.n.03 furniture 36 +1184 folded chairs folded chair 2 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +1279 lotion bottle lotion bottle 2 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +212 can can 2 40 7 can otherprop Objects can 2946921 n02946921 can.n.01 objects 39 +1280 lunch box lunch box 2 40 7 otherprop Objects objects 39 +1281 food display food display 2 40 7 otherprop Objects misc 40 +794 storage shelf storage shelf 2 40 7 otherprop Objects shelving 31 +1282 sliding wood door sliding wood door 2 40 7 otherprop Objects door 4 +955 pants pants 2 40 7 otherprop Objects n04489008 trouser.n.01 clothes 38 +387 wood wood 2 40 7 otherprop Objects misc 40 +69 boards board 2 38 7 board otherstructure Objects board_panel 35 +65 bottles bottle 2 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +523 washcloth washcloth 2 40 7 otherprop Objects n04554523 washcloth.n.01 towel 20 +389 workbench workbench 2 39 6 bench otherfurniture Furniture bench table 4379243 n04600486 workbench.n.01 table 5 +29 open kitchen cabinet kitchen cabinet 2 3 6 cabinet cabinet Furniture n02933112 cabinet.n.01 cabinet 7 +1283 organizer shelf organizer shelf 2 15 6 shelves shelves Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +146 frame frame 2 38 7 otherstructure Objects misc 40 +130 cups cup 2 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 +372 exercise ball exercise ball 2 40 7 ball otherprop Objects n04285146 sports_equipment.n.01 gym_equipment 33 +289 easel easel 2 39 6 stand otherfurniture Furniture n03262809 easel.n.01 furniture 36 +440 garbage bag garbage bag 2 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +321 roomba roomba 2 40 7 otherprop Objects objects 39 +976 garage door garage door 2 38 7 garage door otherstructure Objects door door 4 +1256 luggage rack luggage stand 2 39 6 stand otherfurniture Furniture n04038440 shelving 31 +1284 bike lock bike lock 2 40 7 otherprop Objects objects 39 +1285 briefcase briefcase 2 40 7 otherprop Objects n02900705 briefcase.n.01 objects 39 +357 hand towel hand towel 2 27 7 towel towel Objects n03490006 hand_towel.n.01 towel 20 +1286 bath products bath product 2 40 7 otherprop Objects objects 39 +1287 star star 2 40 7 otherprop Objects n09444783 star.n.03 misc 40 +365 map map 2 40 7 map otherprop Objects n03720163 map.n.01 misc 40 +1288 coffee bean bag coffee bean bag 2 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +81 headboard headboard 2 39 6 headboard otherfurniture Furniture n03502200 headboard.n.01 bed 11 +1289 ipad ipad 2 40 7 otherprop Objects objects 39 +1290 display rack display rack 2 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +948 traffic cone traffic cone 2 40 7 cone otherprop Objects cone objects 39 +174 toiletry toiletry 2 40 7 otherprop Objects n04447443 toiletry.n.01 objects 39 +1028 canopy canopy 2 40 7 otherprop Objects misc 40 +1291 massage chair massage chair 2 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +1292 paper organizer paper organizer 2 40 7 otherprop Objects objects 39 +1005 barricade barricade 2 40 7 otherprop Objects misc 40 +235 platform platform 2 38 7 otherstructure Objects misc 40 +1293 cap cap 2 40 7 hat otherprop Objects n03497657 hat.n.01 clothes 38 +1294 dumbbell plates dumbbell plates 2 40 7 otherprop Objects objects 39 +1295 elevator elevator 2 38 7 otherstructure Objects misc 40 +1296 cooking pan cooking pan 2 40 7 pan otherprop Objects n03880531 pan.n.01 objects 39 +1297 trash bag trash bag 2 37 7 bag bag Objects objects 39 +1298 santa santa 2 40 7 otherprop Objects misc 40 +1299 jewelry box jewelry box 2 29 7 box box Objects n02883344 box.n.01 objects 39 +1300 boat boat 2 40 7 otherprop Objects misc 40 +1301 sock sock 2 21 7 clothes clothes Objects n04254777 sock.n.01 clothes 38 +1051 kinect kinect 2 40 7 kinect otherprop Objects objects 39 +566 crib crib 2 39 6 crib otherfurniture Furniture furniture 36 +1302 plastic storage bin plastic storage bin 2 40 7 container otherprop Objects n03094503 container.n.01 objects 39 +1062 cooler cooler 2 24 6 refridgerator refridgerator Furniture n03102654 cooler.n.01 appliances 37 +1303 kitchen apron kitchen apron 2 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +1304 dishwashing soap bottle dishwashing soap bottle 2 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +1305 xbox controller xbox controller 2 40 7 otherprop Objects objects 39 +1306 banana holder banana holder 2 40 7 otherprop Objects objects 39 +298 ping pong paddle ping pong paddle 2 40 7 otherprop Objects table 5 +1307 airplane airplane 2 40 7 otherprop Objects misc 40 +1308 conditioner bottle conditioner bottle 2 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +1309 tea kettle tea kettle 2 40 7 tea kettle otherprop Objects n04397768 teakettle.n.01 objects 39 +43 bedframe bedframe 2 39 6 otherfurniture Furniture n02822579 bedstead.n.01 bed 11 +1310 wood beam wood beam 2 38 7 otherstructure Objects beam 29 +593 toilet paper package toilet paper package 2 40 7 otherprop Objects objects 39 +1311 wall mounted coat rack wall mounted coat rack 2 40 7 otherprop Objects n03059103 coatrack.n.01 shelving 31 +1312 film light film light 2 40 7 otherprop Objects lighting 28 +749 ceiling lamp ceiling lamp 1 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +623 chain chain 1 40 7 otherprop Objects chair 3 +1313 sofa sofa 1 6 9 sofa sofa Sofa sofa sofa sofa 4256520 n04256520 sofa.n.01 sofa 10 +99 closet wardrobe wardrobe 1 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 furniture 36 +265 sweater sweater 1 40 7 otherprop Objects n04370048 sweater.n.01 clothes 38 +1314 kitchen mixer kitchen mixer 1 40 7 otherprop Objects appliances 37 +99 wardrobe wardrobe 1 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 furniture 36 +1315 water softener water softener 1 40 7 otherprop Objects misc 40 +448 banister banister 1 38 7 banister otherstructure Objects n02788148 bannister.n.02 railing 30 +257 trolley trolley 1 40 7 trolley otherprop Objects n04335435 streetcar.n.01 misc 40 +1316 pantry shelf pantry shelf 1 15 6 shelves shelves Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +786 sofa bed sofa bed 1 4 1 bed bed Bed bed bed bed 2818832 n02818832 bed.n.01 bed 11 +801 loofa loofa 1 40 7 otherprop Objects objects 39 +972 shower faucet handle shower faucet handle 1 40 7 handle otherprop Objects shower 23 +1317 toy piano toy piano 1 40 7 toy otherprop Objects n03964744 plaything.n.01 objects 39 +1318 fish fish 1 40 7 otherprop Objects n02512053 fish.n.01 objects 39 +75 file cabinets file cabinet 1 3 6 cabinet cabinet Furniture cabinet 2933112 n03337140 file.n.03 cabinet 7 +657 cat litter box cat litter box 1 29 7 box box Objects objects 39 +561 electric panel electric panel 1 40 7 otherprop Objects misc 40 +93 suitcases suitcase 1 40 7 luggage otherprop Objects n02774630 baggage.n.01 objects 39 +513 curtain rod curtain rod 1 38 7 curtain rod otherstructure Objects curtain 12 +411 bunk bed bunk bed 1 39 6 bunk bed otherfurniture Furniture bed bed bed 2818832 n02920259 bunk_bed.n.01 bed 11 +1122 chandelier chandelier 1 38 7 chandelier otherstructure Objects n03005285 chandelier.n.01 lighting 28 +922 tape tape 1 40 7 tape otherprop Objects objects 39 +88 plates plate 1 40 7 otherprop Objects n03959485 plate.n.04 objects 39 +518 alarm alarm 1 40 7 alarm otherprop Objects clock 3046257 n02694662 alarm_clock.n.01 objects 39 +814 fire hose fire hose 1 40 7 otherprop Objects n03346004 fire_hose.n.01 misc 40 +1319 toy dinosaur toy dinosaur 1 40 7 toy otherprop Objects n03964744 plaything.n.01 objects 39 +1320 cone cone 1 40 7 otherprop Objects objects 39 +649 glass doors glass door 1 8 12 door door Wall door n03221720 door.n.01 door 4 +607 hatrack hatrack 1 40 7 otherprop Objects n03059103 coatrack.n.01 shelving 31 +819 subwoofer subwoofer 1 40 7 speaker otherprop Objects speaker 3691459 n04349401 subwoofer.n.01 objects 39 +1321 fire sprinkler fire sprinkler 1 40 7 otherprop Objects misc 40 +1322 trash cabinet trash cabinet 1 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +1204 pantry walls pantry wall 1 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +227 photo photo 1 40 7 photo otherprop Objects n03925226 photograph.n.01 picture 6 +817 barrier barrier 1 40 7 otherprop Objects n02796623 barrier.n.01 misc 40 +130 stacks of cups cup 1 40 7 otherprop Objects n03147509 cup.n.01 objects 39 +712 beachball beachball 1 40 7 ball otherprop Objects n02814224 beach_ball.n.01 objects 39 +1323 folded boxes folded boxes 1 40 7 otherprop Objects objects 39 +1324 contact lens solution bottle contact lens solution bottle 1 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +673 covered box covered box 1 29 7 box box Objects objects 39 +459 folder folder 1 40 7 folder otherprop Objects n03376279 folder.n.02 objects 39 +643 mail trays mail tray 1 40 7 mail tray otherprop Objects objects 39 +238 slipper slipper 1 40 7 otherprop Objects n04241394 slipper.n.01 clothes 38 +765 magazine rack magazine rack 1 39 6 stand otherfurniture Furniture n03704549 magazine_rack.n.01 shelving 31 +1008 sticker sticker 1 40 7 sticker otherprop Objects n07272545 gummed_label.n.01 objects 39 +225 lotion lotion 1 40 7 otherprop Objects n03690938 lotion.n.01 objects 39 +1083 buddha buddha 1 40 7 otherprop Objects objects 39 +813 file organizer file organizer 1 40 7 otherprop Objects objects 39 +138 paper towel rolls paper towel roll 1 40 7 paper towel otherprop Objects n03887697 paper_towel.n.01 towel 20 +1145 night lamp night lamp 1 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +796 fuse box fuse box 1 40 7 otherprop Objects misc 40 +1325 knife block knife block 1 40 7 otherprop Objects objects 39 +363 furnace furnace 1 39 6 furnace otherfurniture Furniture n03404449 furnace.n.01 +1174 cd cases cd case 1 40 7 otherprop Objects objects 39 +38 stools stool 1 40 7 stool otherprop Objects stool n04326896 stool.n.01 stool 19 +1326 hand sanitzer dispenser hand sanitzer dispenser 1 40 7 otherprop Objects n04254120 soap_dispenser.n.01 objects 39 +997 teapot teapot 1 40 7 tea pot otherprop Objects n04398044 teapot.n.01 objects 39 +1327 pen holder pen holder 1 40 7 otherprop Objects objects 39 +1328 tray rack tray rack 1 40 7 otherprop Objects objects 39 +1329 wig wig 1 40 7 otherprop Objects n04584207 wig.n.01 objects 39 +182 switch switch 1 40 7 otherprop Objects n04372370 switch.n.01 misc 40 +280 plastic containers plastic container 1 40 7 container otherprop Objects n03094503 container.n.01 objects 39 +1330 night light night light 1 40 7 otherprop Objects lighting 28 +1331 notepad notepad 1 40 7 otherprop Objects objects 39 +1332 mail bin mail bin 1 40 7 otherprop Objects misc 40 +1333 elevator button elevator button 1 40 7 otherprop Objects misc 40 +939 gaming wheel gaming wheel 1 40 7 otherprop Objects objects 39 +1334 drum set drum set 1 40 7 otherprop Objects objects 39 +480 cosmetic bag cosmetic bag 1 37 7 bag bag Objects objects 39 +907 coffee mug coffee mug 1 40 7 vessel otherprop Objects cup or mug 3797390 n03063599 coffee_mug.n.01 objects 39 +1335 closet shelf closet shelf 1 15 6 shelves shelves Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +1336 baby mobile baby mobile 1 40 7 otherprop Objects objects 39 +829 diaper bin diaper bin 1 40 7 bin otherprop Objects objects 39 +947 door wall door wall 1 1 12 wall wall Wall wall 1 +1116 stepstool stepstool 1 40 7 step stool otherprop Objects objects 39 +599 paper shredder shredder 1 40 7 otherprop Objects n04210120 shredder.n.01 objects 39 +733 dress rack dress rack 1 40 7 otherprop Objects n03238762 dress_rack.n.01 misc 40 +123 cover cover 1 40 7 blanket otherprop Objects objects 39 +506 shopping bag shopping bag 1 37 7 bag bag Objects n04204081 shopping_bag.n.01 objects 39 +569 sliding door sliding door 1 8 12 door door Wall door n04239074 sliding_door.n.01 door 4 +1337 exercise bike exercise bike 1 40 7 machine otherprop Objects n04210120 shredder.n.01 gym_equipment 33 +1338 recliner chair recliner chair 1 5 4 chair chair Chair chair chair chair 3001627 n03238762 dress_rack.n.01 chair 3 +1314 kitchenaid mixer kitchen mixer 1 40 7 otherprop Objects appliances 37 +1339 soda can soda can 1 40 7 can otherprop Objects can 2946921 n02946921 can.n.01 objects 39 +1340 stovetop stovetop 1 38 7 stove otherstructure Objects stove 4330267 n04330267 stove.n.02 appliances 37 +851 stepladder stepladder 1 39 6 ladder otherfurniture Furniture stairs n04315599 step_ladder.n.01 stairs 16 +142 tap tap 1 40 7 faucet otherprop Objects faucet 3325088 n04559451 water_faucet.n.01 objects 39 +436 cable cable 1 40 7 cables otherprop Objects objects 39 +1341 baby changing station baby changing station 1 39 6 otherfurniture Furniture furniture 36 +1342 costume costume 1 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +885 rocking chair rocking chair 1 5 4 chair chair Chair chair chair chair 3001627 n04099969 rocking_chair.n.01 chair 3 +693 binder binder 1 40 7 binder otherprop Objects objects 39 +815 media center media center 1 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +401 towel rack towel rack 1 40 7 otherprop Objects n04459773 towel_rack.n.01 misc 40 +1343 medal medal 1 40 7 otherprop Objects objects 39 +1184 stack of folded chairs folded chair 1 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +1344 telescope telescope 1 40 7 otherprop Objects n04403638 telescope.n.01 objects 39 +1345 closet doorframe closet doorframe 1 8 12 door door Wall door door 4 +160 glass glass 1 38 7 glass otherstructure Objects n03438257 glass.n.02 misc 40 +1126 baseball cap baseball cap 1 40 7 otherprop Objects cap 2954340 n02799323 baseball_cap.n.01 clothes 38 +1346 battery disposal jar battery disposal jar 1 40 7 jar otherprop Objects jar 3593526 n03593526 jar.n.01 objects 39 +332 mop mop 1 40 7 otherprop Objects n04367480 swab.n.02 objects 39 +397 tank tank 1 40 7 otherprop Objects objects 39 +643 mail tray mail tray 1 40 7 mail tray otherprop Objects objects 39 +551 centerpiece centerpiece 1 40 7 centerpiece otherprop Objects n02994419 centerpiece.n.02 objects 39 +1163 stick stick 1 40 7 stick otherprop Objects objects 39 +1347 closet floor closet floor 1 2 5 floor floor Floor n03365592 floor.n.01 floor 2 +1348 dryer sheets dryer sheets 1 40 7 otherprop Objects objects 39 +803 bycicle bycicle 1 40 7 otherprop Objects misc 40 +484 flower stand flower stand 1 39 6 stand otherfurniture Furniture furniture 36 +1349 air mattress air mattress 1 4 1 bed bed Bed bed bed bed 2818832 n02690809 air_mattress.n.01 bed 11 +1350 clip clip 1 40 7 otherprop Objects objects 39 +222 side table side table 1 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +1253 pizza boxes pizza box 1 29 7 box box Objects n02883344 box.n.01 objects 39 +1351 display display 1 39 7 otherfurniture Furniture n03211117 display.n.06 misc 40 +1352 postcard postcard 1 40 7 otherprop Objects objects 39 +828 display sign display sign 1 40 7 sign otherprop Objects misc 40 +1353 paper towel paper towel 1 40 7 paper towel otherprop Objects n03887697 paper_towel.n.01 towel 20 +612 boots boot 1 40 7 shoe otherprop Objects n04199027 shoe.n.01 clothes 38 +1354 tennis racket bag tennis racket bag 1 40 7 otherprop Objects objects 39 +1355 air hockey table air hockey table 1 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +1301 socks sock 1 21 7 clothes clothes Objects n04254777 sock.n.01 clothes 38 +1356 food bag food bag 1 37 7 bag bag Objects objects 39 +1199 clothes hangers clothes hanger 1 40 7 otherprop Objects n03057920 coat_hanger.n.01 misc 40 +1357 starbucks cup starbucks cup 1 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2-labels.combined.tsv b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2-labels.combined.tsv new file mode 100644 index 0000000000000000000000000000000000000000..cff61b132f3ebf4edd513445b76fd39db54462d2 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2-labels.combined.tsv @@ -0,0 +1,608 @@ +id raw_category category count nyu40id eigen13id nyuClass nyu40class eigen13class ModelNet40 ModelNet10 ShapeNetCore55 synsetoffset wnsynsetid wnsynsetkey mpcat40 mpcat40index +1 wall wall 8277 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +2 chair chair 4646 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +22 books book 1678 23 2 book books Books n02870526 book.n.11 objects 39 +3 floor floor 1553 2 5 floor floor Floor n03365592 floor.n.01 floor 2 +5 door door 1483 8 12 door door Wall door n03221720 door.n.01 door 4 +1163 object object 1313 40 7 otherprop Objects objects 39 +16 window window 1209 9 13 window window Window n04587648 window.n.01 window 9 +4 table table 1170 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +56 trash can trash can 1090 39 6 garbage bin otherfurniture Furniture trash_bin 2747177 n02747177 ashcan.n.01 objects 39 +13 pillow pillow 937 18 7 pillow pillow Objects pillow 3938244 n03938244 pillow.n.01 cushion 8 +15 picture picture 862 11 8 picture picture Picture n03931044 picture.n.01 picture 6 +41 ceiling ceiling 806 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 ceiling 17 +26 box box 775 29 7 box box Objects n02883344 box.n.01 objects 39 +161 doorframe doorframe 768 8 12 door door Wall door doorframe.n.01 door 4 +19 monitor monitor 765 40 7 monitor otherprop Objects monitor monitor tv or monitor 3211117 n03782190 monitor.n.04 objects 39 +7 cabinet cabinet 731 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +9 desk desk 680 14 10 desk desk Table desk desk table 4379243 n03179701 desk.n.01 table 5 +8 shelf shelf 641 15 6 shelves shelves Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +10 office chair office chair 595 5 4 chair chair Chair chair chair chair 3001627 n04373704 swivel_chair.n.01 chair 3 +31 towel towel 570 27 7 towel towel Objects n04459362 towel.n.01 towel 20 +6 couch couch 502 6 9 sofa sofa Sofa sofa sofa sofa 4256520 n04256520 sofa.n.01 sofa 10 +14 sink sink 488 34 7 sink sink Objects sink n04223580 sink.n.01 sink 15 +48 backpack backpack 479 40 7 backpack otherprop Objects n02769748 backpack.n.01 objects 39 +28 lamp lamp 419 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +11 bed bed 370 4 1 bed bed Bed bed bed bed 2818832 n02818832 bed.n.01 bed 11 +18 bookshelf bookshelf 360 10 6 bookshelf bookshelf Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +71 mirror mirror 349 19 7 mirror mirror Objects n03773035 mirror.n.01 mirror 21 +21 curtain curtain 347 16 13 curtain curtain Window curtain n03151077 curtain.n.01 curtain 12 +40 plant plant 331 40 7 plant otherprop Objects plant n00017222 plant.n.02 plant 14 +52 whiteboard whiteboard 327 30 7 whiteboard whiteboard Objects n03211616 display_panel.n.01 board_panel 35 +96 radiator radiator 322 39 6 radiator otherfurniture Furniture n04041069 radiator.n.02 misc 40 +22 book book 318 23 2 book books Books n02870526 book.n.11 objects 39 +29 kitchen cabinet kitchen cabinet 310 3 6 cabinet cabinet Furniture n02933112 cabinet.n.01 cabinet 7 +49 toilet paper toilet paper 291 40 7 toilet paper otherprop Objects n15075141 toilet_tissue.n.01 objects 39 +29 kitchen cabinets kitchen cabinet 289 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +23 armchair armchair 281 5 4 chair chair Chair chair chair chair 3001627 n02738535 armchair.n.01 chair 3 +63 shoes shoe 272 40 7 shoe otherprop Objects n04199027 shoe.n.01 clothes 38 +24 coffee table coffee table 258 7 10 coffee table table Table table table table 4379243 n03063968 coffee_table.n.01 table 5 +17 toilet toilet 256 33 7 toilet toilet Objects toilet toilet n04446276 toilet.n.01 toilet 18 +47 bag bag 252 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +32 clothes clothes 248 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +46 keyboard keyboard 246 40 7 keyboard otherprop Objects keyboard computer keyboard 3085013 n03085013 computer_keyboard.n.01 objects 39 +65 bottle bottle 226 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +97 recycling bin recycling bin 225 39 6 garbage bin otherfurniture Furniture trash_bin 2747177 n02747177 ashcan.n.01 objects 39 +34 nightstand nightstand 224 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 chest_of_drawers 13 +38 stool stool 221 40 7 stool otherprop Objects stool n04326896 stool.n.01 stool 19 +33 tv tv 219 25 11 television television TV tv or monitor 3211117 n03211117 display.n.06 tv_monitor 22 +75 file cabinet file cabinet 217 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +36 dresser dresser 213 17 6 dresser dresser Furniture dresser dresser n03015254 chest_of_drawers.n.01 chest_of_drawers 13 +64 computer tower computer tower 203 40 7 computer otherprop Objects n03082979 computer.n.01 objects 39 +32 clothing clothes 165 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +101 telephone telephone 164 40 7 telephone otherprop Objects telephone 4401088 n04401088 telephone.n.01 objects 39 +130 cup cup 157 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 +27 refrigerator refrigerator 154 24 6 refridgerator refridgerator Furniture n04070727 refrigerator.n.01 appliances 37 +44 end table end table 147 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +131 jacket jacket 146 40 7 jacket otherprop Objects n03589791 jacket.n.01 clothes 38 +55 shower curtain shower curtain 144 28 7 shower curtain shower curtain Objects curtain n04209239 shower_curtain.n.01 curtain 12 +42 bathtub bathtub 144 36 7 bathtub bathtub Objects bathtub bathtub tub 2808440 n02808440 bathtub.n.01 bathtub 25 +59 microwave microwave 141 40 7 microwave otherprop Objects microwave 3761084 n03761084 microwave.n.02 appliances 37 +159 kitchen counter kitchen counter 140 12 6 counter counter Furniture table table table 4379243 n03116530 counter.n.01 counter 26 +74 sofa chair sofa chair 129 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +82 paper towel dispenser paper towel dispenser 129 40 7 paper towel dispenser otherprop Objects objects 39 +1164 bathroom vanity bathroom vanity 126 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 table 5 +93 suitcase suitcase 118 40 7 luggage otherprop Objects n02773838 bag.n.06 objects 39 +77 laptop laptop 111 40 7 laptop otherprop Objects laptop laptop 3642806 n03642806 laptop.n.01 objects 39 +67 ottoman ottoman 111 39 6 ottoman otherfurniture Furniture stool n03380724 footstool.n.01 stool 19 +128 shower walls shower wall 109 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +50 printer printer 106 40 7 printer otherprop Objects printer 4004475 n04004475 printer.n.03 appliances 37 +35 counter counter 104 12 6 counter counter Furniture table table table 4379243 n03116530 counter.n.01 counter 26 +69 board board 100 38 7 board otherstructure Objects board_panel 35 +100 soap dispenser soap dispenser 99 40 7 otherprop Objects n04254120 soap_dispenser.n.01 objects 39 +62 stove stove 95 38 7 stove otherstructure Objects stove 4330267 n04330267 stove.n.02 appliances 37 +105 light light 93 38 7 light otherstructure Objects n03665366 light.n.02 lighting 28 +1165 closet wall closet wall 90 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +165 mini fridge mini fridge 87 24 6 refridgerator refridgerator Furniture n03273913 electric_refrigerator.n.01 appliances 37 +7 cabinets cabinet 79 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +5 doors door 76 8 12 door door Wall door n03221720 door.n.01 door 4 +76 fan fan 75 40 7 fan otherprop Objects n03320046 fan.n.01 misc 40 +230 tissue box tissue box 73 40 7 tissue box otherprop Objects n02883344 box.n.01 objects 39 +54 blanket blanket 72 40 7 blanket otherprop Objects n02849154 blanket.n.01 objects 39 +125 bathroom stall bathroom stall 71 38 7 otherstructure Objects n02873839 booth.n.02 misc 40 +72 copier copier 70 40 7 otherprop Objects n03257586 duplicator.n.01 appliances 37 +68 bench bench 66 39 6 bench otherfurniture Furniture bench bench 2828884 n02828884 bench.n.01 seating 34 +145 bar bar 66 38 7 bar otherstructure Objects n02788689 bar.n.03 misc 40 +157 soap dish soap dish 65 40 7 soap dish otherprop Objects n04254009 soap_dish.n.01 objects 39 +1166 laundry hamper laundry hamper 65 40 7 laundry basket otherprop Objects objects 39 +132 storage bin storage bin 63 40 7 storage bin otherprop Objects objects 39 +1167 bathroom stall door bathroom stall door 62 8 12 door door Wall door n03221720 door.n.01 door 4 +232 light switch light switch 61 38 7 light switch otherstructure Objects n04372370 switch.n.01 misc 40 +134 coffee maker coffee maker 61 40 7 otherprop Objects n03063338 coffee_maker.n.01 appliances 37 +51 tv stand tv stand 61 39 6 tv stand otherfurniture Furniture tv_stand n03290653 entertainment_center.n.01 furniture 36 +250 decoration decoration 60 40 7 otherprop Objects n03169390 decoration.n.01 misc 40 +1168 ceiling light ceiling light 59 38 7 light otherstructure Objects n03665366 light.n.02 lighting 28 +342 range hood range hood 59 38 7 range hood otherstructure Objects range_hood n04053677 range_hood.n.01 misc 40 +89 blackboard blackboard 58 38 7 blackboard otherstructure Objects n02846511 blackboard.n.01 board_panel 35 +103 clock clock 58 40 7 clock otherprop Objects clock 3046257 n03046257 clock.n.01 objects 39 +99 wardrobe closet wardrobe 54 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 furniture 36 +95 rail rail 53 38 7 railing otherstructure Objects n04047401 railing.n.01 railing 30 +154 bulletin board bulletin board 53 38 7 board otherstructure Objects n03211616 display_panel.n.01 board_panel 35 +140 mat mat 52 20 5 floor mat floor mat Floor n03727837 mat.n.01 floor 2 +1169 trash bin trash bin 52 39 6 garbage bin otherfurniture Furniture trash_bin 2747177 n02747177 ashcan.n.01 objects 39 +193 ledge ledge 51 38 7 otherstructure Objects n09337253 ledge.n.01 misc 40 +116 seat seat 49 39 6 furniture otherfurniture Furniture n04161981 seat.n.03 furniture 36 +202 mouse mouse 49 40 7 mouse otherprop Objects n03793489 mouse.n.04 objects 39 +73 basket basket 48 40 7 basket otherprop Objects basket 2801938 n02801938 basket.n.01 objects 39 +78 shower shower 48 38 7 otherstructure Objects n04208936 shower.n.01 shower 23 +1170 dumbbell dumbbell 48 40 7 otherprop Objects n03255030 dumbbell.n.01 objects 39 +79 paper paper 46 26 7 paper paper Objects n14974264 paper.n.01 objects 39 +80 person person 46 31 7 person person Objects person n05217688 person.n.02 misc 40 +141 windowsill windowsill 45 38 7 otherstructure Objects n04590263 windowsill.n.01 window 9 +57 closet closet 45 39 6 wardrobe otherfurniture Furniture wardrobe misc 40 +102 bucket bucket 45 40 7 bucket otherprop Objects n02909870 bucket.n.01 misc 40 +261 sign sign 44 40 7 sign otherprop Objects n04217882 signboard.n.01 objects 39 +118 speaker speaker 43 40 7 speaker otherprop Objects speaker 3691459 n03691459 loudspeaker.n.01 objects 39 +136 dishwasher dishwasher 43 38 7 dishwasher otherstructure Objects dishwasher 3207941 n03207941 dishwasher.n.01 appliances 37 +98 container container 43 40 7 container otherprop Objects n03094503 container.n.01 objects 39 +1171 stair rail stair rail 42 38 7 banister otherstructure Objects n02788148 bannister.n.02 railing 30 +170 shower curtain rod shower curtain rod 42 40 7 otherprop Objects curtain 12 +1172 tube tube 41 40 7 otherprop Objects misc 40 +1173 bathroom cabinet bathroom cabinet 39 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +79 papers paper 39 26 7 paper paper Objects n14974264 paper.n.01 objects 39 +221 storage container storage container 39 40 7 container otherprop Objects objects 39 +570 paper bag paper bag 39 37 7 bag bag Objects n04122825 sack.n.01 objects 39 +138 paper towel roll paper towel roll 39 40 7 paper towel otherprop Objects n03887697 paper_towel.n.01 towel 20 +168 ball ball 39 40 7 ball otherprop Objects objects 39 +276 closet doors closet door 38 8 12 door door Wall door n03221720 door.n.01 door 4 +106 laundry basket laundry basket 37 40 7 laundry basket otherprop Objects basket 2801938 n03050864 clothes_hamper.n.01 objects 39 +214 cart cart 37 40 7 cart otherprop Objects n03484083 handcart.n.01 shelving 31 +276 closet door closet door 35 8 12 door door Wall door n03221720 door.n.01 door 4 +323 dish rack dish rack 35 40 7 dish rack otherprop Objects n03207630 dish_rack.n.01 objects 39 +58 stairs stairs 35 38 7 stairs otherstructure Objects n04298308 stairway.n.01 stairs 16 +86 blinds blinds 35 13 13 blinds blinds Window n02851099 blind.n.03 blinds 32 +2 stack of chairs chair 35 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +399 purse purse 34 40 7 purse otherprop Objects n02774152 bag.n.04 objects 39 +121 bicycle bicycle 33 40 7 bicycle otherprop Objects bicycle 2834778 n02834778 bicycle.n.01 objects 39 +185 tray tray 32 40 7 tray otherprop Objects n04476259 tray.n.01 objects 39 +300 plunger plunger 30 40 7 otherprop Objects n03970156 plunger.n.03 objects 39 +180 paper cutter paper cutter 30 40 7 paper cutter otherprop Objects n03886940 paper_cutter.n.01 objects 39 +163 toilet paper dispenser toilet paper dispenser 29 40 7 otherprop Objects objects 39 +26 boxes box 29 29 7 box box Objects n02883344 box.n.01 objects 39 +66 bin bin 28 40 7 bin otherprop Objects n02839910 bin.n.01 objects 39 +208 toilet seat cover dispenser toilet seat cover dispenser 28 40 7 otherprop Objects objects 39 +112 guitar guitar 28 40 7 guitar otherprop Objects guitar guitar 3467517 n03467517 guitar.n.01 objects 39 +540 mailboxes mailbox 28 29 7 box box Objects mailbox 3710193 n03710193 mailbox.n.01 misc 40 +395 handicap bar handicap bar 27 38 7 bar otherstructure Objects misc 40 +166 fire extinguisher fire extinguisher 27 40 7 fire extinguisher otherprop Objects n03345837 fire_extinguisher.n.01 misc 40 +122 ladder ladder 27 39 6 ladder otherfurniture Furniture stairs n03632277 ladder.n.01 stairs 16 +120 column column 26 38 7 column otherstructure Objects n03074380 column.n.06 column 24 +107 pipe pipe 25 40 7 pipe otherprop Objects n03944672 pipe.n.02 misc 40 +283 vacuum cleaner vacuum cleaner 25 40 7 otherprop Objects n04517823 vacuum.n.04 objects 39 +88 plate plate 24 40 7 plate otherprop Objects n03959485 plate.n.04 objects 39 +90 piano piano 24 39 6 piano otherfurniture Furniture piano piano 3928116 n03928116 piano.n.01 furniture 36 +177 water cooler water cooler 24 39 6 water cooler otherfurniture Furniture n04559166 water_cooler.n.01 misc 40 +1174 cd case cd case 24 40 7 otherprop Objects objects 39 +562 bowl bowl 24 40 7 bowl otherprop Objects bowl bowl 2880940 n02880940 bowl.n.03 objects 39 +1175 closet rod closet rod 24 40 7 otherprop Objects n04100174 rod.n.01 misc 40 +1156 bathroom counter bathroom counter 24 12 6 counter counter Furniture table table table 4379243 n03116530 counter.n.01 counter 26 +84 oven oven 23 38 7 oven otherstructure Objects n03862676 oven.n.01 appliances 37 +104 stand stand 23 39 6 stand otherfurniture Furniture table table table 4379243 n04301000 stand.n.04 table 5 +229 scale scale 23 40 7 scale otherprop Objects n04141975 scale.n.07 objects 39 +70 washing machine washing machine 23 39 6 washing machine otherfurniture Furniture washing_machine 4554684 n04554684 washer.n.03 appliances 37 +325 broom broom 22 40 7 broom otherprop Objects n02906734 broom.n.01 objects 39 +169 hat hat 22 40 7 hat otherprop Objects n03497657 hat.n.01 clothes 38 +128 shower wall shower wall 22 1 12 wall wall Wall n04208936 shower.n.01 wall 1 +331 guitar case guitar case 21 40 7 guitar case otherprop Objects objects 39 +87 rack rack 21 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +488 water pitcher water pitcher 21 40 7 pitcher otherprop Objects n03950228 pitcher.n.02 objects 39 +776 laundry detergent laundry detergent 21 40 7 otherprop Objects objects 39 +370 hair dryer hair dryer 21 40 7 hair dryer otherprop Objects n03483316 hand_blower.n.01 objects 39 +191 pillar pillar 21 38 7 column otherstructure Objects n03073977 column.n.07 column 24 +748 divider divider 20 40 7 otherprop Objects wall 1 +242 power outlet power outlet 19 40 7 otherprop Objects misc 40 +45 dining table dining table 19 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +417 shower floor shower floor 19 2 5 floor floor Floor n04208936 shower.n.01 floor 2 +70 washing machines washing machine 19 39 6 washing machine otherfurniture Furniture washing_machine 4554684 n04554684 washer.n.03 appliances 37 +188 shower door shower door 19 8 12 door door Wall door n04208936 shower.n.01 door 4 +1176 coffee kettle coffee kettle 18 40 7 pot otherprop Objects n03612814 kettle.n.01 objects 39 +1177 wardrobe cabinet wardrobe 18 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 furniture 36 +1178 structure structure 18 38 7 otherstructure Objects misc 40 +18 bookshelves bookshelf 17 10 6 bookshelf bookshelf Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +110 clothes dryer clothes dryer 17 39 6 otherfurniture Furniture n03251766 dryer.n.01 appliances 37 +148 toaster toaster 17 40 7 toaster otherprop Objects n04442312 toaster.n.02 appliances 37 +63 shoe shoe 17 40 7 shoe otherprop Objects n04199027 shoe.n.01 clothes 38 +155 ironing board ironing board 16 39 6 ironing board otherfurniture Furniture n03586090 ironing_board.n.01 objects 39 +572 alarm clock alarm clock 16 40 7 alarm clock otherprop Objects clock 3046257 n02694662 alarm_clock.n.01 objects 39 +1179 shower head shower head 15 38 7 otherstructure Objects shower 23 +28 lamp base lamp 15 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +392 water bottle water bottle 15 40 7 bottle otherprop Objects bottle bottle 2876657 n04557648 water_bottle.n.01 objects 39 +1180 keyboard piano keyboard piano 15 39 6 piano otherfurniture Furniture piano piano 3928116 n03928116 piano.n.01 furniture 36 +609 projector screen projector screen 15 38 7 projector screen otherstructure Objects misc 40 +1181 case of water bottles case of water bottles 15 40 7 otherprop Objects objects 39 +195 toaster oven toaster oven 14 40 7 toaster oven otherprop Objects n04442441 toaster_oven.n.01 appliances 37 +581 music stand music stand 14 39 6 music stand otherfurniture Furniture n03801760 music_stand.n.01 furniture 36 +58 staircase stairs 14 38 7 stairs otherstructure Objects n04298308 stairway.n.01 stairs 16 +1182 coat rack coat rack 14 40 7 otherprop Objects n03059103 coatrack.n.01 shelving 3 +1183 storage organizer storage organizer 14 40 7 otherprop Objects shelving 3 +139 machine machine 14 40 7 machine otherprop Objects n03699975 machine.n.01 appliances 37 +1184 folded chair folded chair 14 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +1185 fire alarm fire alarm 14 40 7 otherprop Objects n03343737 fire_alarm.n.02 misc 40 +156 fireplace fireplace 13 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 fireplace 27 +408 vent vent 13 40 7 otherprop Objects n04526241 vent.n.01 misc 40 +213 furniture furniture 13 39 6 furniture otherfurniture Furniture n03405725 furniture.n.01 furniture 36 +1186 power strip power strip 13 40 7 otherprop Objects objects 39 +1187 calendar calendar 13 40 7 otherprop Objects objects 39 +1188 poster poster 13 11 8 picture picture Picture n03931044 picture.n.01 picture 6 +115 toilet paper holder toilet paper holder 13 40 7 toilet paper holder otherprop Objects objects 39 +1189 potted plant potted plant 12 40 7 plant otherprop Objects plant n00017222 plant.n.02 plant 14 +304 stuffed animal stuffed animal 12 40 7 stuffed animal otherprop Objects n04399382 teddy.n.01 objects 39 +1190 luggage luggage 12 40 7 luggage otherprop Objects n02774630 baggage.n.01 objects 39 +21 curtains curtain 12 16 13 curtain curtain Window curtain n03151077 curtain.n.01 curtain 12 +312 headphones headphones 12 40 7 otherprop Objects n03261776 earphone.n.01 objects 39 +233 crate crate 12 39 6 crate otherfurniture Furniture n03127925 crate.n.01 objects 39 +286 candle candle 12 40 7 candle otherprop Objects lamp n02948072 candle.n.01 objects 39 +264 projector projector 12 40 7 projector otherprop Objects n04009552 projector.n.02 objects 39 +110 clothes dryers clothes dryer 12 39 6 otherfurniture Furniture n03251766 dryer.n.01 appliances 37 +1191 mattress mattress 12 4 1 bed bed Bed bed bed bed 2818832 n02818832 bed.n.01 bed 11 +356 dustpan dustpan 12 40 7 otherprop Objects n03259009 dustpan.n.02 objects 39 +25 drawer drawer 11 39 6 drawer otherfurniture Furniture n03233905 drawer.n.01 furniture 36 +750 rod rod 11 40 7 otherprop Objects pistol 3948459 n03427202 gat.n.01 misc 40 +269 globe globe 11 40 7 globe otherprop Objects objects 39 +307 footrest footrest 11 39 6 foot rest otherfurniture Furniture stool n03380724 footstool.n.01 stool 19 +410 piano bench piano bench 11 39 6 piano bench otherfurniture Furniture bench bench 2828884 n02828884 bench.n.01 seating 34 +730 breakfast bar breakfast bar 11 38 7 bar otherstructure Objects counter 26 +216 step stool step stool 11 40 7 step stool otherprop Objects stool n04315713 step_stool.n.01 stool 19 +1192 hand rail hand rail 11 38 7 railing otherstructure Objects railing 30 +119 vending machine vending machine 11 40 7 machine otherprop Objects n04525305 vending_machine.n.01 appliances 37 +682 ceiling fan ceiling fan 11 40 7 fan otherprop Objects n03320046 fan.n.01 misc 40 +434 swiffer swiffer 11 40 7 otherprop Objects objects 39 +126 foosball table foosball table 11 39 6 foosball table otherfurniture Furniture table table table 4379243 n04379243 table.n.02 table 5 +919 jar jar 11 40 7 jar otherprop Objects jar 3593526 n03593526 jar.n.01 objects 39 +85 footstool footstool 11 39 6 ottoman otherfurniture Furniture stool n03380724 footstool.n.01 stool 19 +1193 folded table folded table 10 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +108 round table round table 10 7 10 table table Table table table table 4379243 n04114554 round_table.n.02 table 5 +135 hamper hamper 10 40 7 basket otherprop Objects basket 2801938 n03482405 hamper.n.02 objects 39 +1194 poster tube poster tube 10 40 7 otherprop Objects objects 39 +432 case case 10 40 7 case otherprop Objects objects 39 +53 carpet carpet 10 40 7 rug otherprop Objects n04118021 rug.n.01 floor 2 +1195 thermostat thermostat 10 40 7 otherprop Objects n04422875 thermostat.n.01 misc 40 +111 coat coat 10 40 7 jacket otherprop Objects n03057021 coat.n.01 clothes 38 +305 water fountain water fountain 10 38 7 water fountain otherstructure Objects n03241335 drinking_fountain.n.01 misc 40 +1125 smoke detector smoke detector 10 40 7 otherprop Objects misc 40 +13 pillows pillow 9 18 7 pillow pillow Objects pillow 3938244 n03938244 pillow.n.01 cushion 8 +1196 flip flops flip flops 9 40 7 shoe otherprop Objects n04199027 shoe.n.01 clothes 38 +1197 cloth cloth 9 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +1198 banner banner 9 40 7 otherprop Objects n02788021 banner.n.01 misc 40 +1199 clothes hanger clothes hanger 9 40 7 otherprop Objects n03057920 coat_hanger.n.01 objects 39 +1200 whiteboard eraser whiteboard eraser 9 40 7 otherprop Objects objects 39 +378 iron iron 9 40 7 otherprop Objects n03584829 iron.n.04 objects 39 +591 instrument case instrument case 9 40 7 case otherprop Objects objects 39 +49 toilet paper rolls toilet paper 9 40 7 toilet paper otherprop Objects n15075141 toilet_tissue.n.01 objects 39 +92 soap soap 9 40 7 soap otherprop Objects n04253437 soap.n.01 objects 39 +1098 block block 9 40 7 otherprop Objects misc 40 +291 wall hanging wall hanging 8 40 7 otherprop Objects n03491178 hanging.n.01 picture 6 +1063 kitchen island kitchen island 8 38 7 kitchen island otherstructure Objects n03620600 kitchen_island.n.01 counter 26 +107 pipes pipe 8 38 7 otherstructure Objects misc 40 +1135 toothbrush toothbrush 8 40 7 toothbrush otherprop Objects n04453156 toothbrush.n.01 objects 39 +189 shirt shirt 8 40 7 otherprop Objects n04197391 shirt.n.01 clothes 38 +245 cutting board cutting board 8 40 7 cutting board otherprop Objects n03025513 chopping_board.n.01 objects 39 +194 vase vase 8 40 7 vase otherprop Objects vase jar 3593526 n04522168 vase.n.01 objects 39 +1201 shower control valve shower control valve 8 38 7 otherstructure Objects n04208936 shower.n.01 shower 23 +386 exercise machine exercise machine 8 40 7 machine otherprop Objects gym_equipment 33 +1202 compost bin compost bin 8 39 6 garbage bin otherfurniture Furniture trash_bin 2747177 n02747177 ashcan.n.01 objects 39 +857 shorts shorts 8 40 7 shorts otherprop Objects clothes 38 +452 tire tire 8 40 7 otherprop Objects n04440749 tire.n.01 objects 39 +1203 teddy bear teddy bear 7 40 7 stuffed animal otherprop Objects n04399382 teddy.n.01 objects 39 +346 bathrobe bathrobe 7 40 7 otherprop Objects n02807616 bathrobe.n.01 clothes 38 +152 handrail handrail 7 38 7 railing otherstructure Objects n02788148 bannister.n.02 railing 30 +83 faucet faucet 7 40 7 faucet otherprop Objects faucet 3325088 n03325088 faucet.n.01 misc 40 +1204 pantry wall pantry wall 7 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +726 thermos thermos 7 40 7 flask otherprop Objects bottle bottle 2876657 n04422727 thermos.n.01 objects 39 +61 rug rug 7 40 7 rug otherprop Objects n04118021 rug.n.01 floor 2 +39 couch cushions cushion 7 18 7 pillow pillow Objects n03151500 cushion.n.03 cushion 8 +1117 tripod tripod 7 39 6 stand otherfurniture Furniture n04485082 tripod.n.01 objects 39 +540 mailbox mailbox 7 29 7 box box Objects mailbox 3710193 n03710193 mailbox.n.01 misc 40 +1205 tupperware tupperware 7 40 7 otherprop Objects objects 39 +415 shoe rack shoe rack 7 40 7 shoe rack otherprop Objects shelving 31 +31 towels towel 6 27 7 towel towel Objects n04459362 towel.n.01 towel 20 +1206 beer bottles beer bottle 6 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +153 treadmill treadmill 6 39 6 treadmill otherfurniture Furniture n04477387 treadmill.n.01 gym_equipment 33 +1207 salt salt 6 40 7 otherprop Objects objects 39 +129 chest chest 6 39 6 chest otherfurniture Furniture dresser dresser chest_of_drawers 13 +220 dispenser dispenser 6 40 7 otherprop Objects n03210683 dispenser.n.01 objects 39 +1208 mirror doors mirror door 6 8 12 door door Wall door n03221720 door.n.01 door 4 +231 remote remote 6 40 7 otherprop Objects remote_control 4074963 n04074963 remote_control.n.01 objects 39 +1209 folded ladder folded ladder 6 39 6 ladder otherfurniture Furniture stairs n03632277 ladder.n.01 misc 40 +39 cushion cushion 6 18 7 pillow pillow Objects n03151500 cushion.n.03 cushion 8 +1210 carton carton 6 40 7 otherprop Objects objects 39 +117 step step 6 38 7 otherstructure Objects n04314914 step.n.04 misc 40 +822 drying rack drying rack 6 39 6 drying rack otherfurniture Furniture shelving 31 +238 slippers slipper 6 40 7 shoe otherprop Objects n04241394 slipper.n.01 clothes 38 +143 pool table pool table 6 39 6 pool table otherfurniture Furniture table table table 4379243 n03982430 pool_table.n.01 table 5 +1211 soda stream soda stream 6 40 7 otherprop Objects objects 39 +228 toilet brush toilet brush 6 40 7 toilet brush otherprop Objects objects 39 +494 loft bed loft bed 6 4 1 bed bed Bed bed bed bed 2818832 n02818832 bed.n.01 bed 11 +226 cooking pot cooking pot 6 40 7 pot otherprop Objects objects 39 +91 heater heater 6 39 6 heater otherfurniture Furniture n03508101 heater.n.01 misc 40 +1072 messenger bag messenger bag 6 37 7 bag bag Objects objects 39 +435 stapler stapler 6 40 7 stapler otherprop Objects n04303497 stapler.n.01 objects 39 +1165 closet walls closet wall 5 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +345 scanner scanner 5 40 7 otherprop Objects appliances 37 +893 elliptical machine elliptical machine 5 40 7 machine otherprop Objects gym_equipment 33 +621 kettle kettle 5 40 7 pot otherprop Objects n03612814 kettle.n.01 objects 39 +1212 metronome metronome 5 40 7 otherprop Objects n03757604 metronome.n.01 objects 39 +297 dumbell dumbell 5 40 7 otherprop Objects objects 39 +1213 music book music book 5 23 2 book books Books n02870526 book.n.11 objects 39 +1214 rice cooker rice cooker 5 40 7 otherprop Objects objects 39 +1215 dart board dart board 5 38 7 board otherstructure Objects n03162940 dartboard.n.01 objects 39 +529 sewing machine sewing machine 5 40 7 sewing machine otherprop Objects n04179913 sewing_machine.n.01 objects 39 +1216 grab bar grab bar 5 38 7 railing otherstructure Objects railing 30 +1217 flowerpot flowerpot 5 40 7 vase otherprop Objects vase jar 3593526 n04522168 vase.n.01 objects 39 +1218 painting painting 5 11 8 picture picture Picture n03931044 picture.n.01 picture 6 +1219 railing railing 5 38 7 railing otherstructure Objects n04047401 railing.n.01 railing 30 +1220 stair stair 5 38 7 stairs otherstructure Objects stairs n04314914 step.n.04 stairs 16 +525 toolbox toolbox 5 39 6 chest otherfurniture Furniture n04452615 toolbox.n.01 objects 39 +204 nerf gun nerf gun 5 40 7 otherprop Objects objects 39 +693 binders binder 5 40 7 binder otherprop Objects objects 39 +179 desk lamp desk lamp 5 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +1221 quadcopter quadcopter 5 40 7 otherprop Objects objects 39 +1222 pitcher pitcher 5 40 7 pitcher otherprop Objects n03950228 pitcher.n.02 objects 39 +1223 hanging hanging 5 40 7 otherprop Objects misc 40 +1224 mail mail 5 40 7 otherprop Objects misc 40 +1225 closet ceiling closet ceiling 5 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 ceiling 17 +1226 hoverboard hoverboard 5 40 7 otherprop Objects objects 39 +1227 beanbag chair beanbag chair 5 39 6 bean bag otherfurniture Furniture n02816656 beanbag.n.01 chair 3 +571 water heater water heater 5 40 7 water heater otherprop Objects n04560113 water_heater.n.01 misc 40 +1228 spray bottle spray bottle 5 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +556 rope rope 5 40 7 rope otherprop Objects n04108268 rope.n.01 objects 39 +280 plastic container plastic container 5 40 7 container otherprop Objects objects 39 +1229 soap bottle soap bottle 5 40 7 soap otherprop Objects objects 39 +1230 ikea bag ikea bag 4 37 7 bag bag Objects 2773838 n02773838 bag.n.06 objects 39 +1231 sleeping bag sleeping bag 4 40 7 otherprop Objects n04235860 sleeping_bag.n.01 objects 39 +1232 duffel bag duffel bag 4 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +746 frying pan frying pan 4 40 7 frying pan otherprop Objects n03400231 frying_pan.n.01 objects 39 +1233 oven mitt oven mitt 4 40 7 otherprop Objects objects 39 +1234 pot pot 4 40 7 pot otherprop Objects n04235860 sleeping_bag.n.01 objects 39 +144 hand dryer hand dryer 4 40 7 otherprop Objects objects 39 +282 dollhouse dollhouse 4 39 6 doll house otherfurniture Furniture n03219483 dollhouse.n.01 objects 39 +167 shampoo bottle shampoo bottle 4 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +1235 hair brush hair brush 4 40 7 otherprop Objects n02908217 brush.n.02 objects 39 +1236 tennis racket tennis racket 4 40 7 otherprop Objects n04409806 tennis_racket.n.01 objects 39 +1237 display case display case 4 40 7 case otherprop Objects objects 39 +234 ping pong table ping pong table 4 39 6 ping pong table otherfurniture Furniture table table table 4379243 n04379243 table.n.02 table 5 +563 boiler boiler 4 40 7 otherprop Objects misc 40 +1238 bag of coffee beans bag of coffee beans 4 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +1239 bananas banana 4 40 7 otherprop Objects n00021265 food.n.01 objects 39 +1240 carseat carseat 4 40 7 otherprop Objects misc 40 +366 helmet helmet 4 40 7 otherprop Objects helmet 3513137 n03513137 helmet.n.02 clothes 38 +816 umbrella umbrella 4 40 7 umbrella otherprop Objects n04507155 umbrella.n.01 objects 39 +1241 coffee box coffee box 4 40 7 otherprop Objects objects 39 +719 envelope envelope 4 40 7 envelope otherprop Objects n03291819 envelope.n.01 objects 39 +284 wet floor sign wet floor sign 4 40 7 sign otherprop Objects misc 40 +1242 clothing rack clothing rack 4 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +247 controller controller 4 40 7 otherprop Objects n03096960 control.n.09 objects 39 +1243 bath walls bathroom wall 4 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +1244 podium podium 4 39 6 otherfurniture Furniture n03159640 dais.n.01 furniture 36 +1245 storage box storage box 4 29 7 box box Objects n02883344 box.n.01 objects 39 +1246 dolly dolly 4 40 7 otherprop Objects misc 40 +1247 shampoo shampoo 3 40 7 otherprop Objects n04183516 shampoo.n.01 objects 39 +592 paper tray paper tray 3 40 7 paper tray otherprop Objects objects 39 +385 cabinet door cabinet door 3 8 12 door door Wall door door 4 +1248 changing station changing station 3 40 7 otherprop Objects misc 40 +1249 poster printer poster printer 3 40 7 printer otherprop Objects printer 4004475 n04004475 printer.n.03 appliances 37 +133 screen screen 3 40 7 otherprop Objects n03151077 curtain.n.01 curtain 12 +301 soap bar soap bar 3 38 7 bar otherstructure Objects objects 39 +1250 crutches crutches 3 40 7 otherprop Objects n03141823 crutch.n.01 objects 39 +379 studio light studio light 3 38 7 light otherstructure Objects lighting 28 +130 stack of cups cup 3 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 +1251 toilet flush button toilet flush button 3 40 7 otherprop Objects objects 39 +450 trunk trunk 3 40 7 otherprop Objects misc 40 +1252 grocery bag grocery bag 3 37 7 bag bag Objects suitcase 2773838 n03461288 grocery_bag.n.01 objects 39 +316 plastic bin plastic bin 3 40 7 bin otherprop Objects objects 39 +1253 pizza box pizza box 3 29 7 box box Objects objects 39 +385 cabinet doors cabinet door 3 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 door 4 +1254 legs legs 3 31 7 person person Objects person n05217688 person.n.02 misc 40 +461 car car 3 40 7 car otherprop Objects car car 2958343 n02958343 car.n.01 misc 40 +1255 shaving cream shaving cream 3 40 7 otherprop Objects n04186051 shaving_cream.n.01 objects 39 +1256 luggage stand luggage stand 3 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +599 shredder shredder 3 40 7 otherprop Objects n04210120 shredder.n.01 objects 39 +281 statue statue 3 40 7 sculpture otherprop Objects n04306847 statue.n.01 misc 40 +1257 urinal urinal 3 33 7 toilet toilet Objects toilet toilet n04515991 urinal.n.01 toilet 18 +1258 hose hose 3 40 7 otherprop Objects n03539875 hose.n.03 misc 40 +1259 bike pump bike pump 3 40 7 otherprop Objects objects 39 +319 coatrack coatrack 3 40 7 otherprop Objects n03059103 coatrack.n.01 shelving 31 +1260 bear bear 3 40 7 otherprop Objects objects 39 +28 wall lamp lamp 3 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +1261 humidifier humidifier 3 40 7 otherprop Objects objects 39 +546 toothpaste toothpaste 3 40 7 toothpaste otherprop Objects objects 39 +1262 mouthwash bottle mouthwash bottle 3 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +1263 poster cutter poster cutter 3 40 7 otherprop Objects objects 39 +1264 golf bag golf bag 3 37 7 bag bag Objects suitcase 2773838 n03445617 golf_bag.n.01 objects 39 +1265 food container food container 3 40 7 container otherprop Objects n03094503 container.n.01 objects 39 +1266 camera camera 3 40 7 otherprop Objects objects 39 +28 table lamp lamp 3 35 7 lamp lamp Objects lamp lamp 3636649 n04380533 table_lamp.n.01 lighting 28 +1267 yoga mat yoga mat 3 20 5 floor mat floor mat Floor n03727837 mat.n.01 floor 2 +1268 card card 3 40 7 otherprop Objects objects 39 +1269 mug mug 3 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 +188 shower doors shower door 3 38 7 otherstructure Objects n04208936 shower.n.01 door 4 +689 cardboard cardboard 3 40 7 otherprop Objects objects 39 +1270 rack stand rack stand 3 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +1271 boxes of paper boxes of paper 3 29 7 box box Objects n02883344 box.n.01 objects 39 +1272 flag flag 3 40 7 otherprop Objects misc 40 +354 futon futon 3 39 6 mattress otherfurniture Furniture n03408444 futon.n.01 sofa 10 +339 magazine magazine 3 40 7 magazine otherprop Objects n06595351 magazine.n.01 objects 39 +1009 exit sign exit sign 3 40 7 exit sign otherprop Objects misc 40 +1273 rolled poster rolled poster 3 40 7 otherprop Objects objects 39 +1274 wheel wheel 3 40 7 otherprop Objects objects 39 +15 pictures picture 3 11 8 picture picture Picture n03931044 picture.n.01 picture 6 +1275 blackboard eraser blackboard eraser 3 40 7 eraser otherprop Objects n03294833 eraser.n.01 objects 39 +361 organizer organizer 3 40 7 otherprop Objects n03918737 personal_digital_assistant.n.01 objects 39 +1276 doll doll 3 40 7 toy otherprop Objects n03219135 doll.n.01 objects 39 +326 book rack book rack 3 39 6 bookrack otherfurniture Furniture objects 39 +1277 laundry bag laundry bag 3 40 7 laundry basket otherprop Objects basket 2801938 n03050864 clothes_hamper.n.01 objects 39 +1278 sponge sponge 3 40 7 otherprop Objects n01906749 sponge.n.04 objects 39 +116 seating seat 3 39 6 furniture otherfurniture Furniture n04161981 seat.n.03 furniture 36 +1184 folded chairs folded chair 2 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +1279 lotion bottle lotion bottle 2 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +212 can can 2 40 7 can otherprop Objects can 2946921 n02946921 can.n.01 objects 39 +1280 lunch box lunch box 2 40 7 otherprop Objects objects 39 +1281 food display food display 2 40 7 otherprop Objects misc 40 +794 storage shelf storage shelf 2 40 7 otherprop Objects shelving 31 +1282 sliding wood door sliding wood door 2 40 7 otherprop Objects door 4 +955 pants pants 2 40 7 otherprop Objects n04489008 trouser.n.01 clothes 38 +387 wood wood 2 40 7 otherprop Objects misc 40 +69 boards board 2 38 7 board otherstructure Objects board_panel 35 +65 bottles bottle 2 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +523 washcloth washcloth 2 40 7 otherprop Objects n04554523 washcloth.n.01 towel 20 +389 workbench workbench 2 39 6 bench otherfurniture Furniture bench table 4379243 n04600486 workbench.n.01 table 5 +29 open kitchen cabinet kitchen cabinet 2 3 6 cabinet cabinet Furniture n02933112 cabinet.n.01 cabinet 7 +1283 organizer shelf organizer shelf 2 15 6 shelves shelves Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +146 frame frame 2 38 7 otherstructure Objects misc 40 +130 cups cup 2 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 +372 exercise ball exercise ball 2 40 7 ball otherprop Objects n04285146 sports_equipment.n.01 gym_equipment 33 +289 easel easel 2 39 6 stand otherfurniture Furniture n03262809 easel.n.01 furniture 36 +440 garbage bag garbage bag 2 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +321 roomba roomba 2 40 7 otherprop Objects objects 39 +976 garage door garage door 2 38 7 garage door otherstructure Objects door door 4 +1256 luggage rack luggage stand 2 39 6 stand otherfurniture Furniture n04038440 shelving 31 +1284 bike lock bike lock 2 40 7 otherprop Objects objects 39 +1285 briefcase briefcase 2 40 7 otherprop Objects n02900705 briefcase.n.01 objects 39 +357 hand towel hand towel 2 27 7 towel towel Objects n03490006 hand_towel.n.01 towel 20 +1286 bath products bath product 2 40 7 otherprop Objects objects 39 +1287 star star 2 40 7 otherprop Objects n09444783 star.n.03 misc 40 +365 map map 2 40 7 map otherprop Objects n03720163 map.n.01 misc 40 +1288 coffee bean bag coffee bean bag 2 37 7 bag bag Objects suitcase 2773838 n02773838 bag.n.06 objects 39 +81 headboard headboard 2 39 6 headboard otherfurniture Furniture n03502200 headboard.n.01 bed 11 +1289 ipad ipad 2 40 7 otherprop Objects objects 39 +1290 display rack display rack 2 39 6 stand otherfurniture Furniture n04038440 rack.n.05 shelving 31 +948 traffic cone traffic cone 2 40 7 cone otherprop Objects cone objects 39 +174 toiletry toiletry 2 40 7 otherprop Objects n04447443 toiletry.n.01 objects 39 +1028 canopy canopy 2 40 7 otherprop Objects misc 40 +1291 massage chair massage chair 2 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +1292 paper organizer paper organizer 2 40 7 otherprop Objects objects 39 +1005 barricade barricade 2 40 7 otherprop Objects misc 40 +235 platform platform 2 38 7 otherstructure Objects misc 40 +1293 cap cap 2 40 7 hat otherprop Objects n03497657 hat.n.01 clothes 38 +1294 dumbbell plates dumbbell plates 2 40 7 otherprop Objects objects 39 +1295 elevator elevator 2 38 7 otherstructure Objects misc 40 +1296 cooking pan cooking pan 2 40 7 pan otherprop Objects n03880531 pan.n.01 objects 39 +1297 trash bag trash bag 2 37 7 bag bag Objects objects 39 +1298 santa santa 2 40 7 otherprop Objects misc 40 +1299 jewelry box jewelry box 2 29 7 box box Objects n02883344 box.n.01 objects 39 +1300 boat boat 2 40 7 otherprop Objects misc 40 +1301 sock sock 2 21 7 clothes clothes Objects n04254777 sock.n.01 clothes 38 +1051 kinect kinect 2 40 7 kinect otherprop Objects objects 39 +566 crib crib 2 39 6 crib otherfurniture Furniture furniture 36 +1302 plastic storage bin plastic storage bin 2 40 7 container otherprop Objects n03094503 container.n.01 objects 39 +1062 cooler cooler 2 24 6 refridgerator refridgerator Furniture n03102654 cooler.n.01 appliances 37 +1303 kitchen apron kitchen apron 2 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +1304 dishwashing soap bottle dishwashing soap bottle 2 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +1305 xbox controller xbox controller 2 40 7 otherprop Objects objects 39 +1306 banana holder banana holder 2 40 7 otherprop Objects objects 39 +298 ping pong paddle ping pong paddle 2 40 7 otherprop Objects table 5 +1307 airplane airplane 2 40 7 otherprop Objects misc 40 +1308 conditioner bottle conditioner bottle 2 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +1309 tea kettle tea kettle 2 40 7 tea kettle otherprop Objects n04397768 teakettle.n.01 objects 39 +43 bedframe bedframe 2 39 6 otherfurniture Furniture n02822579 bedstead.n.01 bed 11 +1310 wood beam wood beam 2 38 7 otherstructure Objects beam 29 +593 toilet paper package toilet paper package 2 40 7 otherprop Objects objects 39 +1311 wall mounted coat rack wall mounted coat rack 2 40 7 otherprop Objects n03059103 coatrack.n.01 shelving 31 +1312 film light film light 2 40 7 otherprop Objects lighting 28 +749 ceiling lamp ceiling lamp 1 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +623 chain chain 1 40 7 otherprop Objects chair 3 +1313 sofa sofa 1 6 9 sofa sofa Sofa sofa sofa sofa 4256520 n04256520 sofa.n.01 sofa 10 +99 closet wardrobe wardrobe 1 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 furniture 36 +265 sweater sweater 1 40 7 otherprop Objects n04370048 sweater.n.01 clothes 38 +1314 kitchen mixer kitchen mixer 1 40 7 otherprop Objects appliances 37 +99 wardrobe wardrobe 1 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 furniture 36 +1315 water softener water softener 1 40 7 otherprop Objects misc 40 +448 banister banister 1 38 7 banister otherstructure Objects n02788148 bannister.n.02 railing 30 +257 trolley trolley 1 40 7 trolley otherprop Objects n04335435 streetcar.n.01 misc 40 +1316 pantry shelf pantry shelf 1 15 6 shelves shelves Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +786 sofa bed sofa bed 1 4 1 bed bed Bed bed bed bed 2818832 n02818832 bed.n.01 bed 11 +801 loofa loofa 1 40 7 otherprop Objects objects 39 +972 shower faucet handle shower faucet handle 1 40 7 handle otherprop Objects shower 23 +1317 toy piano toy piano 1 40 7 toy otherprop Objects n03964744 plaything.n.01 objects 39 +1318 fish fish 1 40 7 otherprop Objects n02512053 fish.n.01 objects 39 +75 file cabinets file cabinet 1 3 6 cabinet cabinet Furniture cabinet 2933112 n03337140 file.n.03 cabinet 7 +657 cat litter box cat litter box 1 29 7 box box Objects objects 39 +561 electric panel electric panel 1 40 7 otherprop Objects misc 40 +93 suitcases suitcase 1 40 7 luggage otherprop Objects n02774630 baggage.n.01 objects 39 +513 curtain rod curtain rod 1 38 7 curtain rod otherstructure Objects curtain 12 +411 bunk bed bunk bed 1 39 6 bunk bed otherfurniture Furniture bed bed bed 2818832 n02920259 bunk_bed.n.01 bed 11 +1122 chandelier chandelier 1 38 7 chandelier otherstructure Objects n03005285 chandelier.n.01 lighting 28 +922 tape tape 1 40 7 tape otherprop Objects objects 39 +88 plates plate 1 40 7 otherprop Objects n03959485 plate.n.04 objects 39 +518 alarm alarm 1 40 7 alarm otherprop Objects clock 3046257 n02694662 alarm_clock.n.01 objects 39 +814 fire hose fire hose 1 40 7 otherprop Objects n03346004 fire_hose.n.01 misc 40 +1319 toy dinosaur toy dinosaur 1 40 7 toy otherprop Objects n03964744 plaything.n.01 objects 39 +1320 cone cone 1 40 7 otherprop Objects objects 39 +649 glass doors glass door 1 8 12 door door Wall door n03221720 door.n.01 door 4 +607 hatrack hatrack 1 40 7 otherprop Objects n03059103 coatrack.n.01 shelving 31 +819 subwoofer subwoofer 1 40 7 speaker otherprop Objects speaker 3691459 n04349401 subwoofer.n.01 objects 39 +1321 fire sprinkler fire sprinkler 1 40 7 otherprop Objects misc 40 +1322 trash cabinet trash cabinet 1 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +1204 pantry walls pantry wall 1 1 12 wall wall Wall n04546855 wall.n.01 wall 1 +227 photo photo 1 40 7 photo otherprop Objects n03925226 photograph.n.01 picture 6 +817 barrier barrier 1 40 7 otherprop Objects n02796623 barrier.n.01 misc 40 +130 stacks of cups cup 1 40 7 otherprop Objects n03147509 cup.n.01 objects 39 +712 beachball beachball 1 40 7 ball otherprop Objects n02814224 beach_ball.n.01 objects 39 +1323 folded boxes folded boxes 1 40 7 otherprop Objects objects 39 +1324 contact lens solution bottle contact lens solution bottle 1 40 7 bottle otherprop Objects bottle bottle 2876657 n02876657 bottle.n.01 objects 39 +673 covered box covered box 1 29 7 box box Objects objects 39 +459 folder folder 1 40 7 folder otherprop Objects n03376279 folder.n.02 objects 39 +643 mail trays mail tray 1 40 7 mail tray otherprop Objects objects 39 +238 slipper slipper 1 40 7 otherprop Objects n04241394 slipper.n.01 clothes 38 +765 magazine rack magazine rack 1 39 6 stand otherfurniture Furniture n03704549 magazine_rack.n.01 shelving 31 +1008 sticker sticker 1 40 7 sticker otherprop Objects n07272545 gummed_label.n.01 objects 39 +225 lotion lotion 1 40 7 otherprop Objects n03690938 lotion.n.01 objects 39 +1083 buddha buddha 1 40 7 otherprop Objects objects 39 +813 file organizer file organizer 1 40 7 otherprop Objects objects 39 +138 paper towel rolls paper towel roll 1 40 7 paper towel otherprop Objects n03887697 paper_towel.n.01 towel 20 +1145 night lamp night lamp 1 35 7 lamp lamp Objects lamp lamp 3636649 n03636649 lamp.n.02 lighting 28 +796 fuse box fuse box 1 40 7 otherprop Objects misc 40 +1325 knife block knife block 1 40 7 otherprop Objects objects 39 +363 furnace furnace 1 39 6 furnace otherfurniture Furniture n03404449 furnace.n.01 +1174 cd cases cd case 1 40 7 otherprop Objects objects 39 +38 stools stool 1 40 7 stool otherprop Objects stool n04326896 stool.n.01 stool 19 +1326 hand sanitzer dispenser hand sanitzer dispenser 1 40 7 otherprop Objects n04254120 soap_dispenser.n.01 objects 39 +997 teapot teapot 1 40 7 tea pot otherprop Objects n04398044 teapot.n.01 objects 39 +1327 pen holder pen holder 1 40 7 otherprop Objects objects 39 +1328 tray rack tray rack 1 40 7 otherprop Objects objects 39 +1329 wig wig 1 40 7 otherprop Objects n04584207 wig.n.01 objects 39 +182 switch switch 1 40 7 otherprop Objects n04372370 switch.n.01 misc 40 +280 plastic containers plastic container 1 40 7 container otherprop Objects n03094503 container.n.01 objects 39 +1330 night light night light 1 40 7 otherprop Objects lighting 28 +1331 notepad notepad 1 40 7 otherprop Objects objects 39 +1332 mail bin mail bin 1 40 7 otherprop Objects misc 40 +1333 elevator button elevator button 1 40 7 otherprop Objects misc 40 +939 gaming wheel gaming wheel 1 40 7 otherprop Objects objects 39 +1334 drum set drum set 1 40 7 otherprop Objects objects 39 +480 cosmetic bag cosmetic bag 1 37 7 bag bag Objects objects 39 +907 coffee mug coffee mug 1 40 7 vessel otherprop Objects cup or mug 3797390 n03063599 coffee_mug.n.01 objects 39 +1335 closet shelf closet shelf 1 15 6 shelves shelves Furniture bookshelf bookshelf 2871439 n02871439 bookshelf.n.01 shelving 31 +1336 baby mobile baby mobile 1 40 7 otherprop Objects objects 39 +829 diaper bin diaper bin 1 40 7 bin otherprop Objects objects 39 +947 door wall door wall 1 1 12 wall wall Wall wall 1 +1116 stepstool stepstool 1 40 7 step stool otherprop Objects objects 39 +599 paper shredder shredder 1 40 7 otherprop Objects n04210120 shredder.n.01 objects 39 +733 dress rack dress rack 1 40 7 otherprop Objects n03238762 dress_rack.n.01 misc 40 +123 cover cover 1 40 7 blanket otherprop Objects objects 39 +506 shopping bag shopping bag 1 37 7 bag bag Objects n04204081 shopping_bag.n.01 objects 39 +569 sliding door sliding door 1 8 12 door door Wall door n04239074 sliding_door.n.01 door 4 +1337 exercise bike exercise bike 1 40 7 machine otherprop Objects n04210120 shredder.n.01 gym_equipment 33 +1338 recliner chair recliner chair 1 5 4 chair chair Chair chair chair chair 3001627 n03238762 dress_rack.n.01 chair 3 +1314 kitchenaid mixer kitchen mixer 1 40 7 otherprop Objects appliances 37 +1339 soda can soda can 1 40 7 can otherprop Objects can 2946921 n02946921 can.n.01 objects 39 +1340 stovetop stovetop 1 38 7 stove otherstructure Objects stove 4330267 n04330267 stove.n.02 appliances 37 +851 stepladder stepladder 1 39 6 ladder otherfurniture Furniture stairs n04315599 step_ladder.n.01 stairs 16 +142 tap tap 1 40 7 faucet otherprop Objects faucet 3325088 n04559451 water_faucet.n.01 objects 39 +436 cable cable 1 40 7 cables otherprop Objects objects 39 +1341 baby changing station baby changing station 1 39 6 otherfurniture Furniture furniture 36 +1342 costume costume 1 21 7 clothes clothes Objects n02728440 apparel.n.01 clothes 38 +885 rocking chair rocking chair 1 5 4 chair chair Chair chair chair chair 3001627 n04099969 rocking_chair.n.01 chair 3 +693 binder binder 1 40 7 binder otherprop Objects objects 39 +815 media center media center 1 3 6 cabinet cabinet Furniture cabinet 2933112 n02933112 cabinet.n.01 cabinet 7 +401 towel rack towel rack 1 40 7 otherprop Objects n04459773 towel_rack.n.01 misc 40 +1343 medal medal 1 40 7 otherprop Objects objects 39 +1184 stack of folded chairs folded chair 1 5 4 chair chair Chair chair chair chair 3001627 n03001627 chair.n.01 chair 3 +1344 telescope telescope 1 40 7 otherprop Objects n04403638 telescope.n.01 objects 39 +1345 closet doorframe closet doorframe 1 8 12 door door Wall door door 4 +160 glass glass 1 38 7 glass otherstructure Objects n03438257 glass.n.02 misc 40 +1126 baseball cap baseball cap 1 40 7 otherprop Objects cap 2954340 n02799323 baseball_cap.n.01 clothes 38 +1346 battery disposal jar battery disposal jar 1 40 7 jar otherprop Objects jar 3593526 n03593526 jar.n.01 objects 39 +332 mop mop 1 40 7 otherprop Objects n04367480 swab.n.02 objects 39 +397 tank tank 1 40 7 otherprop Objects objects 39 +643 mail tray mail tray 1 40 7 mail tray otherprop Objects objects 39 +551 centerpiece centerpiece 1 40 7 centerpiece otherprop Objects n02994419 centerpiece.n.02 objects 39 +1163 object stick 1 40 7 stick otherprop Objects objects 39 +1347 closet floor closet floor 1 2 5 floor floor Floor n03365592 floor.n.01 floor 2 +1348 dryer sheets dryer sheets 1 40 7 otherprop Objects objects 39 +803 bycicle bycicle 1 40 7 otherprop Objects misc 40 +484 flower stand flower stand 1 39 6 stand otherfurniture Furniture furniture 36 +1349 air mattress air mattress 1 4 1 bed bed Bed bed bed bed 2818832 n02690809 air_mattress.n.01 bed 11 +1350 clip clip 1 40 7 otherprop Objects objects 39 +222 side table side table 1 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +1253 pizza boxes pizza box 1 29 7 box box Objects n02883344 box.n.01 objects 39 +1351 display display 1 39 7 otherfurniture Furniture n03211117 display.n.06 misc 40 +1352 postcard postcard 1 40 7 otherprop Objects objects 39 +828 display sign display sign 1 40 7 sign otherprop Objects misc 40 +1353 paper towel paper towel 1 40 7 paper towel otherprop Objects n03887697 paper_towel.n.01 towel 20 +612 boots boot 1 40 7 shoe otherprop Objects n04199027 shoe.n.01 clothes 38 +1354 tennis racket bag tennis racket bag 1 40 7 otherprop Objects objects 39 +1355 air hockey table air hockey table 1 7 10 table table Table table table table 4379243 n04379243 table.n.02 table 5 +1301 socks sock 1 21 7 clothes clothes Objects n04254777 sock.n.01 clothes 38 +1356 food bag food bag 1 37 7 bag bag Objects objects 39 +1199 clothes hangers clothes hanger 1 40 7 otherprop Objects n03057920 coat_hanger.n.01 misc 40 +1357 starbucks cup starbucks cup 1 40 7 cup otherprop Objects cup cup or mug 3797390 n03797390 mug.n.04 objects 39 \ No newline at end of file diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2_test.txt b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2_test.txt new file mode 100644 index 0000000000000000000000000000000000000000..79d15b0ee4afa889883562a722b837b78ee8ce4b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2_test.txt @@ -0,0 +1,100 @@ +scene0707_00 +scene0708_00 +scene0709_00 +scene0710_00 +scene0711_00 +scene0712_00 +scene0713_00 +scene0714_00 +scene0715_00 +scene0716_00 +scene0717_00 +scene0718_00 +scene0719_00 +scene0720_00 +scene0721_00 +scene0722_00 +scene0723_00 +scene0724_00 +scene0725_00 +scene0726_00 +scene0727_00 +scene0728_00 +scene0729_00 +scene0730_00 +scene0731_00 +scene0732_00 +scene0733_00 +scene0734_00 +scene0735_00 +scene0736_00 +scene0737_00 +scene0738_00 +scene0739_00 +scene0740_00 +scene0741_00 +scene0742_00 +scene0743_00 +scene0744_00 +scene0745_00 +scene0746_00 +scene0747_00 +scene0748_00 +scene0749_00 +scene0750_00 +scene0751_00 +scene0752_00 +scene0753_00 +scene0754_00 +scene0755_00 +scene0756_00 +scene0757_00 +scene0758_00 +scene0759_00 +scene0760_00 +scene0761_00 +scene0762_00 +scene0763_00 +scene0764_00 +scene0765_00 +scene0766_00 +scene0767_00 +scene0768_00 +scene0769_00 +scene0770_00 +scene0771_00 +scene0772_00 +scene0773_00 +scene0774_00 +scene0775_00 +scene0776_00 +scene0777_00 +scene0778_00 +scene0779_00 +scene0780_00 +scene0781_00 +scene0782_00 +scene0783_00 +scene0784_00 +scene0785_00 +scene0786_00 +scene0787_00 +scene0788_00 +scene0789_00 +scene0790_00 +scene0791_00 +scene0792_00 +scene0793_00 +scene0794_00 +scene0795_00 +scene0796_00 +scene0797_00 +scene0798_00 +scene0799_00 +scene0800_00 +scene0801_00 +scene0802_00 +scene0803_00 +scene0804_00 +scene0805_00 +scene0806_00 diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2_train.txt b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2_train.txt new file mode 100644 index 0000000000000000000000000000000000000000..ef625f120b812fea5ac507d3b7049fc7ebd2e7e4 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2_train.txt @@ -0,0 +1,1201 @@ +scene0191_00 +scene0191_01 +scene0191_02 +scene0119_00 +scene0230_00 +scene0528_00 +scene0528_01 +scene0705_00 +scene0705_01 +scene0705_02 +scene0415_00 +scene0415_01 +scene0415_02 +scene0007_00 +scene0141_00 +scene0141_01 +scene0141_02 +scene0515_00 +scene0515_01 +scene0515_02 +scene0447_00 +scene0447_01 +scene0447_02 +scene0531_00 +scene0503_00 +scene0285_00 +scene0069_00 +scene0584_00 +scene0584_01 +scene0584_02 +scene0581_00 +scene0581_01 +scene0581_02 +scene0620_00 +scene0620_01 +scene0263_00 +scene0263_01 +scene0481_00 +scene0481_01 +scene0020_00 +scene0020_01 +scene0291_00 +scene0291_01 +scene0291_02 +scene0469_00 +scene0469_01 +scene0469_02 +scene0659_00 +scene0659_01 +scene0024_00 +scene0024_01 +scene0024_02 +scene0564_00 +scene0117_00 +scene0027_00 +scene0027_01 +scene0027_02 +scene0028_00 +scene0330_00 +scene0418_00 +scene0418_01 +scene0418_02 +scene0233_00 +scene0233_01 +scene0673_00 +scene0673_01 +scene0673_02 +scene0673_03 +scene0673_04 +scene0673_05 +scene0585_00 +scene0585_01 +scene0362_00 +scene0362_01 +scene0362_02 +scene0362_03 +scene0035_00 +scene0035_01 +scene0358_00 +scene0358_01 +scene0358_02 +scene0037_00 +scene0194_00 +scene0321_00 +scene0293_00 +scene0293_01 +scene0623_00 +scene0623_01 +scene0592_00 +scene0592_01 +scene0569_00 +scene0569_01 +scene0413_00 +scene0313_00 +scene0313_01 +scene0313_02 +scene0480_00 +scene0480_01 +scene0401_00 +scene0517_00 +scene0517_01 +scene0517_02 +scene0032_00 +scene0032_01 +scene0613_00 +scene0613_01 +scene0613_02 +scene0306_00 +scene0306_01 +scene0052_00 +scene0052_01 +scene0052_02 +scene0053_00 +scene0444_00 +scene0444_01 +scene0055_00 +scene0055_01 +scene0055_02 +scene0560_00 +scene0589_00 +scene0589_01 +scene0589_02 +scene0610_00 +scene0610_01 +scene0610_02 +scene0364_00 +scene0364_01 +scene0383_00 +scene0383_01 +scene0383_02 +scene0006_00 +scene0006_01 +scene0006_02 +scene0275_00 +scene0451_00 +scene0451_01 +scene0451_02 +scene0451_03 +scene0451_04 +scene0451_05 +scene0135_00 +scene0065_00 +scene0065_01 +scene0065_02 +scene0104_00 +scene0674_00 +scene0674_01 +scene0448_00 +scene0448_01 +scene0448_02 +scene0502_00 +scene0502_01 +scene0502_02 +scene0440_00 +scene0440_01 +scene0440_02 +scene0071_00 +scene0072_00 +scene0072_01 +scene0072_02 +scene0509_00 +scene0509_01 +scene0509_02 +scene0649_00 +scene0649_01 +scene0602_00 +scene0694_00 +scene0694_01 +scene0101_00 +scene0101_01 +scene0101_02 +scene0101_03 +scene0101_04 +scene0101_05 +scene0218_00 +scene0218_01 +scene0579_00 +scene0579_01 +scene0579_02 +scene0039_00 +scene0039_01 +scene0493_00 +scene0493_01 +scene0242_00 +scene0242_01 +scene0242_02 +scene0083_00 +scene0083_01 +scene0127_00 +scene0127_01 +scene0662_00 +scene0662_01 +scene0662_02 +scene0018_00 +scene0087_00 +scene0087_01 +scene0087_02 +scene0332_00 +scene0332_01 +scene0332_02 +scene0628_00 +scene0628_01 +scene0628_02 +scene0134_00 +scene0134_01 +scene0134_02 +scene0238_00 +scene0238_01 +scene0092_00 +scene0092_01 +scene0092_02 +scene0092_03 +scene0092_04 +scene0022_00 +scene0022_01 +scene0467_00 +scene0392_00 +scene0392_01 +scene0392_02 +scene0424_00 +scene0424_01 +scene0424_02 +scene0646_00 +scene0646_01 +scene0646_02 +scene0098_00 +scene0098_01 +scene0044_00 +scene0044_01 +scene0044_02 +scene0510_00 +scene0510_01 +scene0510_02 +scene0571_00 +scene0571_01 +scene0166_00 +scene0166_01 +scene0166_02 +scene0563_00 +scene0172_00 +scene0172_01 +scene0388_00 +scene0388_01 +scene0215_00 +scene0215_01 +scene0252_00 +scene0287_00 +scene0668_00 +scene0572_00 +scene0572_01 +scene0572_02 +scene0026_00 +scene0224_00 +scene0113_00 +scene0113_01 +scene0551_00 +scene0381_00 +scene0381_01 +scene0381_02 +scene0371_00 +scene0371_01 +scene0460_00 +scene0118_00 +scene0118_01 +scene0118_02 +scene0417_00 +scene0008_00 +scene0634_00 +scene0521_00 +scene0123_00 +scene0123_01 +scene0123_02 +scene0045_00 +scene0045_01 +scene0511_00 +scene0511_01 +scene0114_00 +scene0114_01 +scene0114_02 +scene0070_00 +scene0029_00 +scene0029_01 +scene0029_02 +scene0129_00 +scene0103_00 +scene0103_01 +scene0002_00 +scene0002_01 +scene0132_00 +scene0132_01 +scene0132_02 +scene0124_00 +scene0124_01 +scene0143_00 +scene0143_01 +scene0143_02 +scene0604_00 +scene0604_01 +scene0604_02 +scene0507_00 +scene0105_00 +scene0105_01 +scene0105_02 +scene0428_00 +scene0428_01 +scene0311_00 +scene0140_00 +scene0140_01 +scene0182_00 +scene0182_01 +scene0182_02 +scene0142_00 +scene0142_01 +scene0399_00 +scene0399_01 +scene0012_00 +scene0012_01 +scene0012_02 +scene0060_00 +scene0060_01 +scene0370_00 +scene0370_01 +scene0370_02 +scene0310_00 +scene0310_01 +scene0310_02 +scene0661_00 +scene0650_00 +scene0152_00 +scene0152_01 +scene0152_02 +scene0158_00 +scene0158_01 +scene0158_02 +scene0482_00 +scene0482_01 +scene0600_00 +scene0600_01 +scene0600_02 +scene0393_00 +scene0393_01 +scene0393_02 +scene0562_00 +scene0174_00 +scene0174_01 +scene0157_00 +scene0157_01 +scene0161_00 +scene0161_01 +scene0161_02 +scene0159_00 +scene0254_00 +scene0254_01 +scene0115_00 +scene0115_01 +scene0115_02 +scene0162_00 +scene0163_00 +scene0163_01 +scene0523_00 +scene0523_01 +scene0523_02 +scene0459_00 +scene0459_01 +scene0175_00 +scene0085_00 +scene0085_01 +scene0279_00 +scene0279_01 +scene0279_02 +scene0201_00 +scene0201_01 +scene0201_02 +scene0283_00 +scene0456_00 +scene0456_01 +scene0429_00 +scene0043_00 +scene0043_01 +scene0419_00 +scene0419_01 +scene0419_02 +scene0368_00 +scene0368_01 +scene0348_00 +scene0348_01 +scene0348_02 +scene0442_00 +scene0178_00 +scene0380_00 +scene0380_01 +scene0380_02 +scene0165_00 +scene0165_01 +scene0165_02 +scene0181_00 +scene0181_01 +scene0181_02 +scene0181_03 +scene0333_00 +scene0614_00 +scene0614_01 +scene0614_02 +scene0404_00 +scene0404_01 +scene0404_02 +scene0185_00 +scene0126_00 +scene0126_01 +scene0126_02 +scene0519_00 +scene0236_00 +scene0236_01 +scene0189_00 +scene0075_00 +scene0267_00 +scene0192_00 +scene0192_01 +scene0192_02 +scene0281_00 +scene0420_00 +scene0420_01 +scene0420_02 +scene0195_00 +scene0195_01 +scene0195_02 +scene0597_00 +scene0597_01 +scene0597_02 +scene0041_00 +scene0041_01 +scene0111_00 +scene0111_01 +scene0111_02 +scene0666_00 +scene0666_01 +scene0666_02 +scene0200_00 +scene0200_01 +scene0200_02 +scene0536_00 +scene0536_01 +scene0536_02 +scene0390_00 +scene0280_00 +scene0280_01 +scene0280_02 +scene0344_00 +scene0344_01 +scene0205_00 +scene0205_01 +scene0205_02 +scene0484_00 +scene0484_01 +scene0009_00 +scene0009_01 +scene0009_02 +scene0302_00 +scene0302_01 +scene0209_00 +scene0209_01 +scene0209_02 +scene0210_00 +scene0210_01 +scene0395_00 +scene0395_01 +scene0395_02 +scene0683_00 +scene0601_00 +scene0601_01 +scene0214_00 +scene0214_01 +scene0214_02 +scene0477_00 +scene0477_01 +scene0439_00 +scene0439_01 +scene0468_00 +scene0468_01 +scene0468_02 +scene0546_00 +scene0466_00 +scene0466_01 +scene0220_00 +scene0220_01 +scene0220_02 +scene0122_00 +scene0122_01 +scene0130_00 +scene0110_00 +scene0110_01 +scene0110_02 +scene0327_00 +scene0156_00 +scene0266_00 +scene0266_01 +scene0001_00 +scene0001_01 +scene0228_00 +scene0199_00 +scene0219_00 +scene0464_00 +scene0232_00 +scene0232_01 +scene0232_02 +scene0299_00 +scene0299_01 +scene0530_00 +scene0363_00 +scene0453_00 +scene0453_01 +scene0570_00 +scene0570_01 +scene0570_02 +scene0183_00 +scene0239_00 +scene0239_01 +scene0239_02 +scene0373_00 +scene0373_01 +scene0241_00 +scene0241_01 +scene0241_02 +scene0188_00 +scene0622_00 +scene0622_01 +scene0244_00 +scene0244_01 +scene0691_00 +scene0691_01 +scene0206_00 +scene0206_01 +scene0206_02 +scene0247_00 +scene0247_01 +scene0061_00 +scene0061_01 +scene0082_00 +scene0250_00 +scene0250_01 +scene0250_02 +scene0501_00 +scene0501_01 +scene0501_02 +scene0320_00 +scene0320_01 +scene0320_02 +scene0320_03 +scene0631_00 +scene0631_01 +scene0631_02 +scene0255_00 +scene0255_01 +scene0255_02 +scene0047_00 +scene0265_00 +scene0265_01 +scene0265_02 +scene0004_00 +scene0336_00 +scene0336_01 +scene0058_00 +scene0058_01 +scene0260_00 +scene0260_01 +scene0260_02 +scene0243_00 +scene0603_00 +scene0603_01 +scene0093_00 +scene0093_01 +scene0093_02 +scene0109_00 +scene0109_01 +scene0434_00 +scene0434_01 +scene0434_02 +scene0290_00 +scene0627_00 +scene0627_01 +scene0470_00 +scene0470_01 +scene0137_00 +scene0137_01 +scene0137_02 +scene0270_00 +scene0270_01 +scene0270_02 +scene0271_00 +scene0271_01 +scene0504_00 +scene0274_00 +scene0274_01 +scene0274_02 +scene0036_00 +scene0036_01 +scene0276_00 +scene0276_01 +scene0272_00 +scene0272_01 +scene0499_00 +scene0698_00 +scene0698_01 +scene0051_00 +scene0051_01 +scene0051_02 +scene0051_03 +scene0108_00 +scene0245_00 +scene0369_00 +scene0369_01 +scene0369_02 +scene0284_00 +scene0289_00 +scene0289_01 +scene0286_00 +scene0286_01 +scene0286_02 +scene0286_03 +scene0031_00 +scene0031_01 +scene0031_02 +scene0545_00 +scene0545_01 +scene0545_02 +scene0557_00 +scene0557_01 +scene0557_02 +scene0533_00 +scene0533_01 +scene0116_00 +scene0116_01 +scene0116_02 +scene0611_00 +scene0611_01 +scene0688_00 +scene0294_00 +scene0294_01 +scene0294_02 +scene0295_00 +scene0295_01 +scene0296_00 +scene0296_01 +scene0596_00 +scene0596_01 +scene0596_02 +scene0532_00 +scene0532_01 +scene0637_00 +scene0638_00 +scene0121_00 +scene0121_01 +scene0121_02 +scene0040_00 +scene0040_01 +scene0197_00 +scene0197_01 +scene0197_02 +scene0410_00 +scene0410_01 +scene0305_00 +scene0305_01 +scene0615_00 +scene0615_01 +scene0703_00 +scene0703_01 +scene0555_00 +scene0297_00 +scene0297_01 +scene0297_02 +scene0582_00 +scene0582_01 +scene0582_02 +scene0023_00 +scene0094_00 +scene0013_00 +scene0013_01 +scene0013_02 +scene0136_00 +scene0136_01 +scene0136_02 +scene0407_00 +scene0407_01 +scene0062_00 +scene0062_01 +scene0062_02 +scene0386_00 +scene0318_00 +scene0554_00 +scene0554_01 +scene0497_00 +scene0213_00 +scene0258_00 +scene0323_00 +scene0323_01 +scene0324_00 +scene0324_01 +scene0016_00 +scene0016_01 +scene0016_02 +scene0681_00 +scene0398_00 +scene0398_01 +scene0227_00 +scene0090_00 +scene0066_00 +scene0262_00 +scene0262_01 +scene0155_00 +scene0155_01 +scene0155_02 +scene0352_00 +scene0352_01 +scene0352_02 +scene0038_00 +scene0038_01 +scene0038_02 +scene0335_00 +scene0335_01 +scene0335_02 +scene0261_00 +scene0261_01 +scene0261_02 +scene0261_03 +scene0640_00 +scene0640_01 +scene0640_02 +scene0080_00 +scene0080_01 +scene0080_02 +scene0403_00 +scene0403_01 +scene0282_00 +scene0282_01 +scene0282_02 +scene0682_00 +scene0173_00 +scene0173_01 +scene0173_02 +scene0522_00 +scene0687_00 +scene0345_00 +scene0345_01 +scene0612_00 +scene0612_01 +scene0411_00 +scene0411_01 +scene0411_02 +scene0625_00 +scene0625_01 +scene0211_00 +scene0211_01 +scene0211_02 +scene0211_03 +scene0676_00 +scene0676_01 +scene0179_00 +scene0498_00 +scene0498_01 +scene0498_02 +scene0547_00 +scene0547_01 +scene0547_02 +scene0269_00 +scene0269_01 +scene0269_02 +scene0366_00 +scene0680_00 +scene0680_01 +scene0588_00 +scene0588_01 +scene0588_02 +scene0588_03 +scene0346_00 +scene0346_01 +scene0359_00 +scene0359_01 +scene0014_00 +scene0120_00 +scene0120_01 +scene0212_00 +scene0212_01 +scene0212_02 +scene0176_00 +scene0049_00 +scene0259_00 +scene0259_01 +scene0586_00 +scene0586_01 +scene0586_02 +scene0309_00 +scene0309_01 +scene0125_00 +scene0455_00 +scene0177_00 +scene0177_01 +scene0177_02 +scene0326_00 +scene0372_00 +scene0171_00 +scene0171_01 +scene0374_00 +scene0654_00 +scene0654_01 +scene0445_00 +scene0445_01 +scene0475_00 +scene0475_01 +scene0475_02 +scene0349_00 +scene0349_01 +scene0234_00 +scene0669_00 +scene0669_01 +scene0375_00 +scene0375_01 +scene0375_02 +scene0387_00 +scene0387_01 +scene0387_02 +scene0312_00 +scene0312_01 +scene0312_02 +scene0384_00 +scene0385_00 +scene0385_01 +scene0385_02 +scene0000_00 +scene0000_01 +scene0000_02 +scene0376_00 +scene0376_01 +scene0376_02 +scene0301_00 +scene0301_01 +scene0301_02 +scene0322_00 +scene0542_00 +scene0079_00 +scene0079_01 +scene0099_00 +scene0099_01 +scene0476_00 +scene0476_01 +scene0476_02 +scene0394_00 +scene0394_01 +scene0147_00 +scene0147_01 +scene0067_00 +scene0067_01 +scene0067_02 +scene0397_00 +scene0397_01 +scene0337_00 +scene0337_01 +scene0337_02 +scene0431_00 +scene0223_00 +scene0223_01 +scene0223_02 +scene0010_00 +scene0010_01 +scene0402_00 +scene0268_00 +scene0268_01 +scene0268_02 +scene0679_00 +scene0679_01 +scene0405_00 +scene0128_00 +scene0408_00 +scene0408_01 +scene0190_00 +scene0107_00 +scene0076_00 +scene0167_00 +scene0361_00 +scene0361_01 +scene0361_02 +scene0216_00 +scene0202_00 +scene0303_00 +scene0303_01 +scene0303_02 +scene0446_00 +scene0446_01 +scene0089_00 +scene0089_01 +scene0089_02 +scene0360_00 +scene0150_00 +scene0150_01 +scene0150_02 +scene0421_00 +scene0421_01 +scene0421_02 +scene0454_00 +scene0626_00 +scene0626_01 +scene0626_02 +scene0186_00 +scene0186_01 +scene0538_00 +scene0479_00 +scene0479_01 +scene0479_02 +scene0656_00 +scene0656_01 +scene0656_02 +scene0656_03 +scene0525_00 +scene0525_01 +scene0525_02 +scene0308_00 +scene0396_00 +scene0396_01 +scene0396_02 +scene0624_00 +scene0292_00 +scene0292_01 +scene0632_00 +scene0253_00 +scene0021_00 +scene0325_00 +scene0325_01 +scene0437_00 +scene0437_01 +scene0438_00 +scene0590_00 +scene0590_01 +scene0400_00 +scene0400_01 +scene0541_00 +scene0541_01 +scene0541_02 +scene0677_00 +scene0677_01 +scene0677_02 +scene0443_00 +scene0315_00 +scene0288_00 +scene0288_01 +scene0288_02 +scene0422_00 +scene0672_00 +scene0672_01 +scene0184_00 +scene0449_00 +scene0449_01 +scene0449_02 +scene0048_00 +scene0048_01 +scene0138_00 +scene0452_00 +scene0452_01 +scene0452_02 +scene0667_00 +scene0667_01 +scene0667_02 +scene0463_00 +scene0463_01 +scene0078_00 +scene0078_01 +scene0078_02 +scene0636_00 +scene0457_00 +scene0457_01 +scene0457_02 +scene0465_00 +scene0465_01 +scene0577_00 +scene0151_00 +scene0151_01 +scene0339_00 +scene0573_00 +scene0573_01 +scene0154_00 +scene0096_00 +scene0096_01 +scene0096_02 +scene0235_00 +scene0168_00 +scene0168_01 +scene0168_02 +scene0594_00 +scene0587_00 +scene0587_01 +scene0587_02 +scene0587_03 +scene0229_00 +scene0229_01 +scene0229_02 +scene0512_00 +scene0106_00 +scene0106_01 +scene0106_02 +scene0472_00 +scene0472_01 +scene0472_02 +scene0489_00 +scene0489_01 +scene0489_02 +scene0425_00 +scene0425_01 +scene0641_00 +scene0526_00 +scene0526_01 +scene0317_00 +scene0317_01 +scene0544_00 +scene0017_00 +scene0017_01 +scene0017_02 +scene0042_00 +scene0042_01 +scene0042_02 +scene0576_00 +scene0576_01 +scene0576_02 +scene0347_00 +scene0347_01 +scene0347_02 +scene0436_00 +scene0226_00 +scene0226_01 +scene0485_00 +scene0486_00 +scene0487_00 +scene0487_01 +scene0619_00 +scene0097_00 +scene0367_00 +scene0367_01 +scene0491_00 +scene0492_00 +scene0492_01 +scene0005_00 +scene0005_01 +scene0543_00 +scene0543_01 +scene0543_02 +scene0657_00 +scene0341_00 +scene0341_01 +scene0534_00 +scene0534_01 +scene0319_00 +scene0273_00 +scene0273_01 +scene0225_00 +scene0198_00 +scene0003_00 +scene0003_01 +scene0003_02 +scene0409_00 +scene0409_01 +scene0331_00 +scene0331_01 +scene0505_00 +scene0505_01 +scene0505_02 +scene0505_03 +scene0505_04 +scene0506_00 +scene0057_00 +scene0057_01 +scene0074_00 +scene0074_01 +scene0074_02 +scene0091_00 +scene0112_00 +scene0112_01 +scene0112_02 +scene0240_00 +scene0102_00 +scene0102_01 +scene0513_00 +scene0514_00 +scene0514_01 +scene0537_00 +scene0516_00 +scene0516_01 +scene0495_00 +scene0617_00 +scene0133_00 +scene0520_00 +scene0520_01 +scene0635_00 +scene0635_01 +scene0054_00 +scene0473_00 +scene0473_01 +scene0524_00 +scene0524_01 +scene0379_00 +scene0471_00 +scene0471_01 +scene0471_02 +scene0566_00 +scene0248_00 +scene0248_01 +scene0248_02 +scene0529_00 +scene0529_01 +scene0529_02 +scene0391_00 +scene0264_00 +scene0264_01 +scene0264_02 +scene0675_00 +scene0675_01 +scene0350_00 +scene0350_01 +scene0350_02 +scene0450_00 +scene0068_00 +scene0068_01 +scene0237_00 +scene0237_01 +scene0365_00 +scene0365_01 +scene0365_02 +scene0605_00 +scene0605_01 +scene0539_00 +scene0539_01 +scene0539_02 +scene0540_00 +scene0540_01 +scene0540_02 +scene0170_00 +scene0170_01 +scene0170_02 +scene0433_00 +scene0340_00 +scene0340_01 +scene0340_02 +scene0160_00 +scene0160_01 +scene0160_02 +scene0160_03 +scene0160_04 +scene0059_00 +scene0059_01 +scene0059_02 +scene0056_00 +scene0056_01 +scene0478_00 +scene0478_01 +scene0548_00 +scene0548_01 +scene0548_02 +scene0204_00 +scene0204_01 +scene0204_02 +scene0033_00 +scene0145_00 +scene0483_00 +scene0508_00 +scene0508_01 +scene0508_02 +scene0180_00 +scene0148_00 +scene0556_00 +scene0556_01 +scene0416_00 +scene0416_01 +scene0416_02 +scene0416_03 +scene0416_04 +scene0073_00 +scene0073_01 +scene0073_02 +scene0073_03 +scene0034_00 +scene0034_01 +scene0034_02 +scene0639_00 +scene0561_00 +scene0561_01 +scene0298_00 +scene0692_00 +scene0692_01 +scene0692_02 +scene0692_03 +scene0692_04 +scene0642_00 +scene0642_01 +scene0642_02 +scene0642_03 +scene0630_00 +scene0630_01 +scene0630_02 +scene0630_03 +scene0630_04 +scene0630_05 +scene0630_06 +scene0706_00 +scene0567_00 +scene0567_01 diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2_val.txt b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2_val.txt new file mode 100644 index 0000000000000000000000000000000000000000..b9e7d9205321e8ca047a527466f4b7100c9c9d2c --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/meta_data/scannetv2_val.txt @@ -0,0 +1,312 @@ +scene0568_00 +scene0568_01 +scene0568_02 +scene0304_00 +scene0488_00 +scene0488_01 +scene0412_00 +scene0412_01 +scene0217_00 +scene0019_00 +scene0019_01 +scene0414_00 +scene0575_00 +scene0575_01 +scene0575_02 +scene0426_00 +scene0426_01 +scene0426_02 +scene0426_03 +scene0549_00 +scene0549_01 +scene0578_00 +scene0578_01 +scene0578_02 +scene0665_00 +scene0665_01 +scene0050_00 +scene0050_01 +scene0050_02 +scene0257_00 +scene0025_00 +scene0025_01 +scene0025_02 +scene0583_00 +scene0583_01 +scene0583_02 +scene0701_00 +scene0701_01 +scene0701_02 +scene0580_00 +scene0580_01 +scene0565_00 +scene0169_00 +scene0169_01 +scene0655_00 +scene0655_01 +scene0655_02 +scene0063_00 +scene0221_00 +scene0221_01 +scene0591_00 +scene0591_01 +scene0591_02 +scene0678_00 +scene0678_01 +scene0678_02 +scene0462_00 +scene0427_00 +scene0595_00 +scene0193_00 +scene0193_01 +scene0164_00 +scene0164_01 +scene0164_02 +scene0164_03 +scene0598_00 +scene0598_01 +scene0598_02 +scene0599_00 +scene0599_01 +scene0599_02 +scene0328_00 +scene0300_00 +scene0300_01 +scene0354_00 +scene0458_00 +scene0458_01 +scene0423_00 +scene0423_01 +scene0423_02 +scene0307_00 +scene0307_01 +scene0307_02 +scene0606_00 +scene0606_01 +scene0606_02 +scene0432_00 +scene0432_01 +scene0608_00 +scene0608_01 +scene0608_02 +scene0651_00 +scene0651_01 +scene0651_02 +scene0430_00 +scene0430_01 +scene0689_00 +scene0357_00 +scene0357_01 +scene0574_00 +scene0574_01 +scene0574_02 +scene0329_00 +scene0329_01 +scene0329_02 +scene0153_00 +scene0153_01 +scene0616_00 +scene0616_01 +scene0671_00 +scene0671_01 +scene0618_00 +scene0382_00 +scene0382_01 +scene0490_00 +scene0621_00 +scene0607_00 +scene0607_01 +scene0149_00 +scene0695_00 +scene0695_01 +scene0695_02 +scene0695_03 +scene0389_00 +scene0377_00 +scene0377_01 +scene0377_02 +scene0342_00 +scene0139_00 +scene0629_00 +scene0629_01 +scene0629_02 +scene0496_00 +scene0633_00 +scene0633_01 +scene0518_00 +scene0652_00 +scene0406_00 +scene0406_01 +scene0406_02 +scene0144_00 +scene0144_01 +scene0494_00 +scene0278_00 +scene0278_01 +scene0316_00 +scene0609_00 +scene0609_01 +scene0609_02 +scene0609_03 +scene0084_00 +scene0084_01 +scene0084_02 +scene0696_00 +scene0696_01 +scene0696_02 +scene0351_00 +scene0351_01 +scene0643_00 +scene0644_00 +scene0645_00 +scene0645_01 +scene0645_02 +scene0081_00 +scene0081_01 +scene0081_02 +scene0647_00 +scene0647_01 +scene0535_00 +scene0353_00 +scene0353_01 +scene0353_02 +scene0559_00 +scene0559_01 +scene0559_02 +scene0593_00 +scene0593_01 +scene0246_00 +scene0653_00 +scene0653_01 +scene0064_00 +scene0064_01 +scene0356_00 +scene0356_01 +scene0356_02 +scene0030_00 +scene0030_01 +scene0030_02 +scene0222_00 +scene0222_01 +scene0338_00 +scene0338_01 +scene0338_02 +scene0378_00 +scene0378_01 +scene0378_02 +scene0660_00 +scene0553_00 +scene0553_01 +scene0553_02 +scene0527_00 +scene0663_00 +scene0663_01 +scene0663_02 +scene0664_00 +scene0664_01 +scene0664_02 +scene0334_00 +scene0334_01 +scene0334_02 +scene0046_00 +scene0046_01 +scene0046_02 +scene0203_00 +scene0203_01 +scene0203_02 +scene0088_00 +scene0088_01 +scene0088_02 +scene0088_03 +scene0086_00 +scene0086_01 +scene0086_02 +scene0670_00 +scene0670_01 +scene0256_00 +scene0256_01 +scene0256_02 +scene0249_00 +scene0441_00 +scene0658_00 +scene0704_00 +scene0704_01 +scene0187_00 +scene0187_01 +scene0131_00 +scene0131_01 +scene0131_02 +scene0207_00 +scene0207_01 +scene0207_02 +scene0461_00 +scene0011_00 +scene0011_01 +scene0343_00 +scene0251_00 +scene0077_00 +scene0077_01 +scene0684_00 +scene0684_01 +scene0550_00 +scene0686_00 +scene0686_01 +scene0686_02 +scene0208_00 +scene0500_00 +scene0500_01 +scene0552_00 +scene0552_01 +scene0648_00 +scene0648_01 +scene0435_00 +scene0435_01 +scene0435_02 +scene0435_03 +scene0690_00 +scene0690_01 +scene0693_00 +scene0693_01 +scene0693_02 +scene0700_00 +scene0700_01 +scene0700_02 +scene0699_00 +scene0231_00 +scene0231_01 +scene0231_02 +scene0697_00 +scene0697_01 +scene0697_02 +scene0697_03 +scene0474_00 +scene0474_01 +scene0474_02 +scene0474_03 +scene0474_04 +scene0474_05 +scene0355_00 +scene0355_01 +scene0146_00 +scene0146_01 +scene0146_02 +scene0196_00 +scene0702_00 +scene0702_01 +scene0702_02 +scene0314_00 +scene0277_00 +scene0277_01 +scene0277_02 +scene0095_00 +scene0095_01 +scene0015_00 +scene0100_00 +scene0100_01 +scene0100_02 +scene0558_00 +scene0558_01 +scene0558_02 +scene0685_00 +scene0685_01 +scene0685_02 diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/preprocess_scannet.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/preprocess_scannet.py new file mode 100644 index 0000000000000000000000000000000000000000..549a4261080b0dfb47f31ed390f821446d35322e --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/preprocess_scannet.py @@ -0,0 +1,253 @@ +""" +Preprocessing Script for ScanNet 20/200 + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import warnings + +warnings.filterwarnings("ignore", category=DeprecationWarning) + +import os +import argparse +import glob +import json +import plyfile +import numpy as np +import pandas as pd +import multiprocessing as mp +from concurrent.futures import ProcessPoolExecutor +from itertools import repeat +from pathlib import Path + +# Load external constants +from meta_data.scannet200_constants import VALID_CLASS_IDS_200, VALID_CLASS_IDS_20 + +CLOUD_FILE_PFIX = "_vh_clean_2" +SEGMENTS_FILE_PFIX = ".0.010000.segs.json" +AGGREGATIONS_FILE_PFIX = ".aggregation.json" +CLASS_IDS200 = VALID_CLASS_IDS_200 +CLASS_IDS20 = VALID_CLASS_IDS_20 +IGNORE_INDEX = -1 + + +def read_plymesh(filepath): + """Read ply file and return it as numpy array. Returns None if emtpy.""" + with open(filepath, "rb") as f: + plydata = plyfile.PlyData.read(f) + if plydata.elements: + vertices = pd.DataFrame(plydata["vertex"].data).values + faces = np.stack(plydata["face"].data["vertex_indices"], axis=0) + return vertices, faces + + +# Map the raw category id to the point cloud +def point_indices_from_group(seg_indices, group, labels_pd): + group_segments = np.array(group["segments"]) + label = group["label"] + + # Map the category name to id + label_id20 = labels_pd[labels_pd["raw_category"] == label]["nyu40id"] + label_id20 = int(label_id20.iloc[0]) if len(label_id20) > 0 else 0 + label_id200 = labels_pd[labels_pd["raw_category"] == label]["id"] + label_id200 = int(label_id200.iloc[0]) if len(label_id200) > 0 else 0 + + # Only store for the valid categories + if label_id20 in CLASS_IDS20: + label_id20 = CLASS_IDS20.index(label_id20) + else: + label_id20 = IGNORE_INDEX + + if label_id200 in CLASS_IDS200: + label_id200 = CLASS_IDS200.index(label_id200) + else: + label_id200 = IGNORE_INDEX + + # get points, where segment indices (points labelled with segment ids) are in the group segment list + point_idx = np.where(np.isin(seg_indices, group_segments))[0] + return point_idx, label_id20, label_id200 + + +def face_normal(vertex, face): + v01 = vertex[face[:, 1]] - vertex[face[:, 0]] + v02 = vertex[face[:, 2]] - vertex[face[:, 0]] + vec = np.cross(v01, v02) + length = np.sqrt(np.sum(vec**2, axis=1, keepdims=True)) + 1.0e-8 + nf = vec / length + area = length * 0.5 + return nf, area + + +def vertex_normal(vertex, face): + nf, area = face_normal(vertex, face) + nf = nf * area + + nv = np.zeros_like(vertex) + for i in range(face.shape[0]): + nv[face[i]] += nf[i] + + length = np.sqrt(np.sum(nv**2, axis=1, keepdims=True)) + 1.0e-8 + nv = nv / length + return nv + + +def handle_process( + scene_path, output_path, labels_pd, train_scenes, val_scenes, parse_normals=True +): + scene_id = os.path.basename(scene_path) + mesh_path = os.path.join(scene_path, f"{scene_id}{CLOUD_FILE_PFIX}.ply") + segments_file = os.path.join( + scene_path, f"{scene_id}{CLOUD_FILE_PFIX}{SEGMENTS_FILE_PFIX}" + ) + aggregations_file = os.path.join(scene_path, f"{scene_id}{AGGREGATIONS_FILE_PFIX}") + info_file = os.path.join(scene_path, f"{scene_id}.txt") + + if scene_id in train_scenes: + output_path = os.path.join(output_path, "train", f"{scene_id}") + split_name = "train" + elif scene_id in val_scenes: + output_path = os.path.join(output_path, "val", f"{scene_id}") + split_name = "val" + else: + output_path = os.path.join(output_path, "test", f"{scene_id}") + split_name = "test" + + print(f"Processing: {scene_id} in {split_name}") + + vertices, faces = read_plymesh(mesh_path) + coords = vertices[:, :3] + colors = vertices[:, 3:6] + save_dict = dict( + coord=coords.astype(np.float32), + color=colors.astype(np.uint8), + ) + + # # Rotating the mesh to axis aligned + # info_dict = {} + # with open(info_file) as f: + # for line in f: + # (key, val) = line.split(" = ") + # info_dict[key] = np.fromstring(val, sep=' ') + # + # if 'axisAlignment' not in info_dict: + # rot_matrix = np.identity(4) + # else: + # rot_matrix = info_dict['axisAlignment'].reshape(4, 4) + # r_coords = coords.transpose() + # r_coords = np.append(r_coords, np.ones((1, r_coords.shape[1])), axis=0) + # r_coords = np.dot(rot_matrix, r_coords) + # coords = r_coords + + # Parse Normals + if parse_normals: + save_dict["normal"] = vertex_normal(coords, faces).astype(np.float32) + + # Load segments file + if split_name != "test": + with open(segments_file) as f: + segments = json.load(f) + seg_indices = np.array(segments["segIndices"]) + + # Load Aggregations file + with open(aggregations_file) as f: + aggregation = json.load(f) + seg_groups = np.array(aggregation["segGroups"]) + + # Generate new labels + semantic_gt20 = np.ones((vertices.shape[0]), dtype=np.int16) * IGNORE_INDEX + semantic_gt200 = np.ones((vertices.shape[0]), dtype=np.int16) * IGNORE_INDEX + instance_ids = np.ones((vertices.shape[0]), dtype=np.int16) * IGNORE_INDEX + for group in seg_groups: + point_idx, label_id20, label_id200 = point_indices_from_group( + seg_indices, group, labels_pd + ) + + semantic_gt20[point_idx] = label_id20 + semantic_gt200[point_idx] = label_id200 + instance_ids[point_idx] = group["id"] + + semantic_gt20 = semantic_gt20.astype(int) + semantic_gt200 = semantic_gt200.astype(int) + instance_ids = instance_ids.astype(int) + + save_dict["segment20"] = semantic_gt20 + save_dict["segment200"] = semantic_gt200 + save_dict["instance"] = instance_ids + + # Concatenate with original cloud + processed_vertices = np.hstack((semantic_gt200, instance_ids)) + + if np.any(np.isnan(processed_vertices)) or not np.all( + np.isfinite(processed_vertices) + ): + raise ValueError(f"Find NaN in Scene: {scene_id}") + + # Save processed data + os.makedirs(output_path, exist_ok=True) + for key in save_dict.keys(): + np.save(os.path.join(output_path, f"{key}.npy"), save_dict[key]) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset_root", + required=True, + help="Path to the ScanNet dataset containing scene folders", + ) + parser.add_argument( + "--output_root", + required=True, + help="Output path where train/val folders will be located", + ) + parser.add_argument( + "--parse_normals", default=True, type=bool, help="Whether parse point normals" + ) + parser.add_argument( + "--num_workers", + default=mp.cpu_count(), + type=int, + help="Num workers for preprocessing.", + ) + config = parser.parse_args() + meta_root = Path(os.path.dirname(__file__)) / "meta_data" + + # Load label map + labels_pd = pd.read_csv( + meta_root / "scannetv2-labels.combined.tsv", + sep="\t", + header=0, + ) + + # Load train/val splits + with open(meta_root / "scannetv2_train.txt") as train_file: + train_scenes = train_file.read().splitlines() + with open(meta_root / "scannetv2_val.txt") as val_file: + val_scenes = val_file.read().splitlines() + + # Create output directories + train_output_dir = os.path.join(config.output_root, "train") + os.makedirs(train_output_dir, exist_ok=True) + val_output_dir = os.path.join(config.output_root, "val") + os.makedirs(val_output_dir, exist_ok=True) + test_output_dir = os.path.join(config.output_root, "test") + os.makedirs(test_output_dir, exist_ok=True) + + # Load scene paths + scene_paths = sorted(glob.glob(config.dataset_root + "/scans*/scene*")) + + # Preprocess data. + print("Processing scenes...") + pool = ProcessPoolExecutor(max_workers=config.num_workers) + _ = list( + pool.map( + handle_process, + scene_paths, + repeat(config.output_root), + repeat(labels_pd), + repeat(train_scenes), + repeat(val_scenes), + repeat(config.parse_normals), + ) + ) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/SensorData.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/SensorData.py new file mode 100644 index 0000000000000000000000000000000000000000..d90c8770e812f782e4735cc7095c100cd6258bf6 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/SensorData.py @@ -0,0 +1,183 @@ +import os, struct +import numpy as np +import zlib +import imageio +import cv2 + +COMPRESSION_TYPE_COLOR = {-1: "unknown", 0: "raw", 1: "png", 2: "jpeg"} +COMPRESSION_TYPE_DEPTH = { + -1: "unknown", + 0: "raw_ushort", + 1: "zlib_ushort", + 2: "occi_ushort", +} + + +class RGBDFrame: + def load(self, file_handle): + self.camera_to_world = np.asarray( + struct.unpack("f" * 16, file_handle.read(16 * 4)), dtype=np.float32 + ).reshape(4, 4) + self.timestamp_color = struct.unpack("Q", file_handle.read(8))[0] + self.timestamp_depth = struct.unpack("Q", file_handle.read(8))[0] + self.color_size_bytes = struct.unpack("Q", file_handle.read(8))[0] + self.depth_size_bytes = struct.unpack("Q", file_handle.read(8))[0] + self.color_data = b"".join( + struct.unpack( + "c" * self.color_size_bytes, file_handle.read(self.color_size_bytes) + ) + ) + self.depth_data = b"".join( + struct.unpack( + "c" * self.depth_size_bytes, file_handle.read(self.depth_size_bytes) + ) + ) + + def decompress_depth(self, compression_type): + if compression_type == "zlib_ushort": + return self.decompress_depth_zlib() + else: + raise + + def decompress_depth_zlib(self): + return zlib.decompress(self.depth_data) + + def decompress_color(self, compression_type): + if compression_type == "jpeg": + return self.decompress_color_jpeg() + else: + raise + + def decompress_color_jpeg(self): + return imageio.imread(self.color_data) + + +class SensorData: + def __init__(self, filename): + self.version = 4 + self.load(filename) + + def load(self, filename): + with open(filename, "rb") as f: + version = struct.unpack("I", f.read(4))[0] + assert self.version == version + strlen = struct.unpack("Q", f.read(8))[0] + self.sensor_name = b"".join(struct.unpack("c" * strlen, f.read(strlen))) + self.intrinsic_color = np.asarray( + struct.unpack("f" * 16, f.read(16 * 4)), dtype=np.float32 + ).reshape(4, 4) + self.extrinsic_color = np.asarray( + struct.unpack("f" * 16, f.read(16 * 4)), dtype=np.float32 + ).reshape(4, 4) + self.intrinsic_depth = np.asarray( + struct.unpack("f" * 16, f.read(16 * 4)), dtype=np.float32 + ).reshape(4, 4) + self.extrinsic_depth = np.asarray( + struct.unpack("f" * 16, f.read(16 * 4)), dtype=np.float32 + ).reshape(4, 4) + self.color_compression_type = COMPRESSION_TYPE_COLOR[ + struct.unpack("i", f.read(4))[0] + ] + self.depth_compression_type = COMPRESSION_TYPE_DEPTH[ + struct.unpack("i", f.read(4))[0] + ] + self.color_width = struct.unpack("I", f.read(4))[0] + self.color_height = struct.unpack("I", f.read(4))[0] + self.depth_width = struct.unpack("I", f.read(4))[0] + self.depth_height = struct.unpack("I", f.read(4))[0] + self.depth_shift = struct.unpack("f", f.read(4))[0] + num_frames = struct.unpack("Q", f.read(8))[0] + self.frames = [] + for i in range(num_frames): + frame = RGBDFrame() + frame.load(f) + self.frames.append(frame) + + def export_depth_images(self, output_path, image_size=None, frame_skip=1): + if not os.path.exists(output_path): + os.makedirs(output_path) + print( + "exporting", len(self.frames) // frame_skip, " depth frames to", output_path + ) + for f in range(0, len(self.frames), frame_skip): + if os.path.exists((os.path.join(output_path, str(f) + ".png"))): + continue + if f % 100 == 0: + print( + "exporting", + f, + "th depth frames to", + os.path.join(output_path, str(f) + ".png"), + ) + + depth_data = self.frames[f].decompress_depth(self.depth_compression_type) + depth = np.fromstring(depth_data, dtype=np.uint16).reshape( + self.depth_height, self.depth_width + ) + if image_size is not None: + depth = cv2.resize( + depth, + (image_size[1], image_size[0]), + interpolation=cv2.INTER_NEAREST, + ) + imageio.imwrite(os.path.join(output_path, str(f) + ".png"), depth) + + def export_color_images(self, output_path, image_size=None, frame_skip=1): + if not os.path.exists(output_path): + os.makedirs(output_path) + print( + "exporting", len(self.frames) // frame_skip, "color frames to", output_path + ) + for f in range(0, len(self.frames), frame_skip): + if os.path.exists((os.path.join(output_path, str(f) + ".png"))): + continue + if f % 100 == 0: + print( + "exporting", + f, + "th color frames to", + os.path.join(output_path, str(f) + ".png"), + ) + color = self.frames[f].decompress_color(self.color_compression_type) + if image_size is not None: + color = cv2.resize( + color, + (image_size[1], image_size[0]), + interpolation=cv2.INTER_NEAREST, + ) + # imageio.imwrite(os.path.join(output_path, str(f) + '.jpg'), color) + imageio.imwrite(os.path.join(output_path, str(f) + ".png"), color) + + def save_mat_to_file(self, matrix, filename): + with open(filename, "w") as f: + for line in matrix: + np.savetxt(f, line[np.newaxis], fmt="%f") + + def export_poses(self, output_path, frame_skip=1): + if not os.path.exists(output_path): + os.makedirs(output_path) + print( + "exporting", len(self.frames) // frame_skip, "camera poses to", output_path + ) + for f in range(0, len(self.frames), frame_skip): + self.save_mat_to_file( + self.frames[f].camera_to_world, + os.path.join(output_path, str(f) + ".txt"), + ) + + def export_intrinsics(self, output_path): + if not os.path.exists(output_path): + os.makedirs(output_path) + print("exporting camera intrinsics to", output_path) + self.save_mat_to_file( + self.intrinsic_color, os.path.join(output_path, "intrinsic_color.txt") + ) + self.save_mat_to_file( + self.extrinsic_color, os.path.join(output_path, "extrinsic_color.txt") + ) + self.save_mat_to_file( + self.intrinsic_depth, os.path.join(output_path, "intrinsic_depth.txt") + ) + self.save_mat_to_file( + self.extrinsic_depth, os.path.join(output_path, "extrinsic_depth.txt") + ) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/compute_full_overlapping.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/compute_full_overlapping.py new file mode 100644 index 0000000000000000000000000000000000000000..a6b407eebad280f2817805d15ec43b9f7f6afbf4 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/compute_full_overlapping.py @@ -0,0 +1,91 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import copy +import torch +import numpy as np +import math +import glob, os +import argparse +import open3d as o3d + + +def make_open3d_point_cloud(xyz, color=None, voxel_size=None): + if np.isnan(xyz).any(): + return None + + xyz = xyz[:, :3] + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(xyz) + if color is not None: + pcd.colors = o3d.utility.Vector3dVector(color) + if voxel_size is not None: + pcd = pcd.voxel_down_sample(voxel_size) + + return pcd + + +def compute_overlap_ratio(pcd0, pcd1, voxel_size): + pcd0_down = pcd0.voxel_down_sample(voxel_size) + pcd1_down = pcd1.voxel_down_sample(voxel_size) + matching01 = get_matching_indices(pcd0_down, pcd1_down, voxel_size * 1.5, 1) + matching10 = get_matching_indices(pcd1_down, pcd0_down, voxel_size * 1.5, 1) + overlap0 = float(len(matching01)) / float(len(pcd0_down.points)) + overlap1 = float(len(matching10)) / float(len(pcd1_down.points)) + return max(overlap0, overlap1) + + +def get_matching_indices(source, pcd_tree, search_voxel_size, K=None): + match_inds = [] + for i, point in enumerate(source.points): + [_, idx, _] = pcd_tree.search_radius_vector_3d(point, search_voxel_size) + if K is not None: + idx = idx[:K] + for j in idx: + match_inds.append((i, j)) + return match_inds + + +def compute_full_overlapping(data_root, scene_id, voxel_size=0.05): + _points = [ + ( + pcd_name, + make_open3d_point_cloud( + torch.load(pcd_name)["coord"], voxel_size=voxel_size + ), + ) + for pcd_name in glob.glob(os.path.join(data_root, scene_id, "pcd", "*.pth")) + ] + points = [(pcd_name, pcd) for (pcd_name, pcd) in _points if pcd is not None] + print( + "load {} point clouds ({} invalid has been filtered), computing matching/overlapping".format( + len(points), len(_points) - len(points) + ) + ) + + matching_matrix = np.zeros((len(points), len(points))) + for i, (pcd0_name, pcd0) in enumerate(points): + print("matching to...{}".format(pcd0_name)) + pcd0_tree = o3d.geometry.KDTreeFlann(copy.deepcopy(pcd0)) + for j, (pcd1_name, pcd1) in enumerate(points): + if i == j: + continue + matching_matrix[i, j] = float( + len(get_matching_indices(pcd1, pcd0_tree, 1.5 * voxel_size, 1)) + ) / float(len(pcd1.points)) + + # write to file + with open(os.path.join(data_root, scene_id, "pcd", "overlap.txt"), "w") as f: + for i, (pcd0_name, pcd0) in enumerate(points): + for j, (pcd1_name, pcd1) in enumerate(points): + if i < j: + overlap = max(matching_matrix[i, j], matching_matrix[j, i]) + f.write( + "{} {} {}\n".format( + pcd0_name.replace(data_root, ""), + pcd1_name.replace(data_root, ""), + overlap, + ) + ) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/generage_list.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/generage_list.py new file mode 100644 index 0000000000000000000000000000000000000000..a8943ba040cc24fc8d3130bd8784052cb57ce6c9 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/generage_list.py @@ -0,0 +1,33 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import argparse +import glob, os, sys + +from SensorData import SensorData + +# params +parser = argparse.ArgumentParser() +# data paths +parser.add_argument("--target_dir", required=True, help="path to the target dir") + +opt = parser.parse_args() +print(opt) + + +def main(): + overlaps = glob.glob(os.path.join(opt.target_dir, "*/pcd/overlap.txt")) + with open(os.path.join(opt.target_dir, "overlap30.txt"), "w") as f: + for fo in overlaps: + for line in open(fo): + pcd0, pcd1, op = line.strip().split() + if float(op) >= 0.3: + print("{} {} {}".format(pcd0, pcd1, op), file=f) + print("done") + + +if __name__ == "__main__": + main() diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/plyfile.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/plyfile.py new file mode 100644 index 0000000000000000000000000000000000000000..17400c4bd28764829d248e90dc141182fa1d8f03 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/plyfile.py @@ -0,0 +1,894 @@ +# Copyright 2014 Darsh Ranjan +# +# This file is part of python-plyfile. +# +# python-plyfile is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# python-plyfile is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with python-plyfile. If not, see +# . + +from itertools import islice as _islice + +import numpy as _np +from sys import byteorder as _byteorder + + +try: + _range = xrange +except NameError: + _range = range + + +# Many-many relation +_data_type_relation = [ + ("int8", "i1"), + ("char", "i1"), + ("uint8", "u1"), + ("uchar", "b1"), + ("uchar", "u1"), + ("int16", "i2"), + ("short", "i2"), + ("uint16", "u2"), + ("ushort", "u2"), + ("int32", "i4"), + ("int", "i4"), + ("uint32", "u4"), + ("uint", "u4"), + ("float32", "f4"), + ("float", "f4"), + ("float64", "f8"), + ("double", "f8"), +] + +_data_types = dict(_data_type_relation) +_data_type_reverse = dict((b, a) for (a, b) in _data_type_relation) + +_types_list = [] +_types_set = set() +for _a, _b in _data_type_relation: + if _a not in _types_set: + _types_list.append(_a) + _types_set.add(_a) + if _b not in _types_set: + _types_list.append(_b) + _types_set.add(_b) + + +_byte_order_map = {"ascii": "=", "binary_little_endian": "<", "binary_big_endian": ">"} + +_byte_order_reverse = {"<": "binary_little_endian", ">": "binary_big_endian"} + +_native_byte_order = {"little": "<", "big": ">"}[_byteorder] + + +def _lookup_type(type_str): + if type_str not in _data_type_reverse: + try: + type_str = _data_types[type_str] + except KeyError: + raise ValueError("field type %r not in %r" % (type_str, _types_list)) + + return _data_type_reverse[type_str] + + +def _split_line(line, n): + fields = line.split(None, n) + if len(fields) == n: + fields.append("") + + assert len(fields) == n + 1 + + return fields + + +def make2d(array, cols=None, dtype=None): + """ + Make a 2D array from an array of arrays. The `cols' and `dtype' + arguments can be omitted if the array is not empty. + + """ + if (cols is None or dtype is None) and not len(array): + raise RuntimeError("cols and dtype must be specified for empty " "array") + + if cols is None: + cols = len(array[0]) + + if dtype is None: + dtype = array[0].dtype + + return _np.fromiter(array, [("_", dtype, (cols,))], count=len(array))["_"] + + +class PlyParseError(Exception): + """ + Raised when a PLY file cannot be parsed. + + The attributes `element', `row', `property', and `message' give + additional information. + + """ + + def __init__(self, message, element=None, row=None, prop=None): + self.message = message + self.element = element + self.row = row + self.prop = prop + + s = "" + if self.element: + s += "element %r: " % self.element.name + if self.row is not None: + s += "row %d: " % self.row + if self.prop: + s += "property %r: " % self.prop.name + s += self.message + + Exception.__init__(self, s) + + def __repr__(self): + return ( + "PlyParseError(%r, element=%r, row=%r, prop=%r)" % self.message, + self.element, + self.row, + self.prop, + ) + + +class PlyData(object): + """ + PLY file header and data. + + A PlyData instance is created in one of two ways: by the static + method PlyData.read (to read a PLY file), or directly from __init__ + given a sequence of elements (which can then be written to a PLY + file). + + """ + + def __init__( + self, elements=[], text=False, byte_order="=", comments=[], obj_info=[] + ): + """ + elements: sequence of PlyElement instances. + + text: whether the resulting PLY file will be text (True) or + binary (False). + + byte_order: '<' for little-endian, '>' for big-endian, or '=' + for native. This is only relevant if `text' is False. + + comments: sequence of strings that will be placed in the header + between the 'ply' and 'format ...' lines. + + obj_info: like comments, but will be placed in the header with + "obj_info ..." instead of "comment ...". + + """ + if byte_order == "=" and not text: + byte_order = _native_byte_order + + self.byte_order = byte_order + self.text = text + + self.comments = list(comments) + self.obj_info = list(obj_info) + self.elements = elements + + def _get_elements(self): + return self._elements + + def _set_elements(self, elements): + self._elements = tuple(elements) + self._index() + + elements = property(_get_elements, _set_elements) + + def _get_byte_order(self): + return self._byte_order + + def _set_byte_order(self, byte_order): + if byte_order not in ["<", ">", "="]: + raise ValueError("byte order must be '<', '>', or '='") + + self._byte_order = byte_order + + byte_order = property(_get_byte_order, _set_byte_order) + + def _index(self): + self._element_lookup = dict((elt.name, elt) for elt in self._elements) + if len(self._element_lookup) != len(self._elements): + raise ValueError("two elements with same name") + + @staticmethod + def _parse_header(stream): + """ + Parse a PLY header from a readable file-like stream. + + """ + lines = [] + comments = {"comment": [], "obj_info": []} + while True: + line = stream.readline().decode("ascii").strip() + fields = _split_line(line, 1) + + if fields[0] == "end_header": + break + + elif fields[0] in comments.keys(): + lines.append(fields) + else: + lines.append(line.split()) + + a = 0 + if lines[a] != ["ply"]: + raise PlyParseError("expected 'ply'") + + a += 1 + while lines[a][0] in comments.keys(): + comments[lines[a][0]].append(lines[a][1]) + a += 1 + + if lines[a][0] != "format": + raise PlyParseError("expected 'format'") + + if lines[a][2] != "1.0": + raise PlyParseError("expected version '1.0'") + + if len(lines[a]) != 3: + raise PlyParseError("too many fields after 'format'") + + fmt = lines[a][1] + + if fmt not in _byte_order_map: + raise PlyParseError("don't understand format %r" % fmt) + + byte_order = _byte_order_map[fmt] + text = fmt == "ascii" + + a += 1 + while a < len(lines) and lines[a][0] in comments.keys(): + comments[lines[a][0]].append(lines[a][1]) + a += 1 + + return PlyData( + PlyElement._parse_multi(lines[a:]), + text, + byte_order, + comments["comment"], + comments["obj_info"], + ) + + @staticmethod + def read(stream): + """ + Read PLY data from a readable file-like object or filename. + + """ + (must_close, stream) = _open_stream(stream, "read") + try: + data = PlyData._parse_header(stream) + for elt in data: + elt._read(stream, data.text, data.byte_order) + finally: + if must_close: + stream.close() + + return data + + def write(self, stream): + """ + Write PLY data to a writeable file-like object or filename. + + """ + (must_close, stream) = _open_stream(stream, "write") + try: + stream.write(self.header.encode("ascii")) + stream.write(b"\r\n") + for elt in self: + elt._write(stream, self.text, self.byte_order) + finally: + if must_close: + stream.close() + + @property + def header(self): + """ + Provide PLY-formatted metadata for the instance. + + """ + lines = ["ply"] + + if self.text: + lines.append("format ascii 1.0") + else: + lines.append("format " + _byte_order_reverse[self.byte_order] + " 1.0") + + # Some information is lost here, since all comments are placed + # between the 'format' line and the first element. + for c in self.comments: + lines.append("comment " + c) + + for c in self.obj_info: + lines.append("obj_info " + c) + + lines.extend(elt.header for elt in self.elements) + lines.append("end_header") + return "\r\n".join(lines) + + def __iter__(self): + return iter(self.elements) + + def __len__(self): + return len(self.elements) + + def __contains__(self, name): + return name in self._element_lookup + + def __getitem__(self, name): + return self._element_lookup[name] + + def __str__(self): + return self.header + + def __repr__(self): + return "PlyData(%r, text=%r, byte_order=%r, " "comments=%r, obj_info=%r)" % ( + self.elements, + self.text, + self.byte_order, + self.comments, + self.obj_info, + ) + + +def _open_stream(stream, read_or_write): + if hasattr(stream, read_or_write): + return (False, stream) + try: + return (True, open(stream, read_or_write[0] + "b")) + except TypeError: + raise RuntimeError("expected open file or filename") + + +class PlyElement(object): + """ + PLY file element. + + A client of this library doesn't normally need to instantiate this + directly, so the following is only for the sake of documenting the + internals. + + Creating a PlyElement instance is generally done in one of two ways: + as a byproduct of PlyData.read (when reading a PLY file) and by + PlyElement.describe (before writing a PLY file). + + """ + + def __init__(self, name, properties, count, comments=[]): + """ + This is not part of the public interface. The preferred methods + of obtaining PlyElement instances are PlyData.read (to read from + a file) and PlyElement.describe (to construct from a numpy + array). + + """ + self._name = str(name) + self._check_name() + self._count = count + + self._properties = tuple(properties) + self._index() + + self.comments = list(comments) + + self._have_list = any(isinstance(p, PlyListProperty) for p in self.properties) + + @property + def count(self): + return self._count + + def _get_data(self): + return self._data + + def _set_data(self, data): + self._data = data + self._count = len(data) + self._check_sanity() + + data = property(_get_data, _set_data) + + def _check_sanity(self): + for prop in self.properties: + if prop.name not in self._data.dtype.fields: + raise ValueError("dangling property %r" % prop.name) + + def _get_properties(self): + return self._properties + + def _set_properties(self, properties): + self._properties = tuple(properties) + self._check_sanity() + self._index() + + properties = property(_get_properties, _set_properties) + + def _index(self): + self._property_lookup = dict((prop.name, prop) for prop in self._properties) + if len(self._property_lookup) != len(self._properties): + raise ValueError("two properties with same name") + + def ply_property(self, name): + return self._property_lookup[name] + + @property + def name(self): + return self._name + + def _check_name(self): + if any(c.isspace() for c in self._name): + msg = "element name %r contains spaces" % self._name + raise ValueError(msg) + + def dtype(self, byte_order="="): + """ + Return the numpy dtype of the in-memory representation of the + data. (If there are no list properties, and the PLY format is + binary, then this also accurately describes the on-disk + representation of the element.) + + """ + return [(prop.name, prop.dtype(byte_order)) for prop in self.properties] + + @staticmethod + def _parse_multi(header_lines): + """ + Parse a list of PLY element definitions. + + """ + elements = [] + while header_lines: + (elt, header_lines) = PlyElement._parse_one(header_lines) + elements.append(elt) + + return elements + + @staticmethod + def _parse_one(lines): + """ + Consume one element definition. The unconsumed input is + returned along with a PlyElement instance. + + """ + a = 0 + line = lines[a] + + if line[0] != "element": + raise PlyParseError("expected 'element'") + if len(line) > 3: + raise PlyParseError("too many fields after 'element'") + if len(line) < 3: + raise PlyParseError("too few fields after 'element'") + + (name, count) = (line[1], int(line[2])) + + comments = [] + properties = [] + while True: + a += 1 + if a >= len(lines): + break + + if lines[a][0] == "comment": + comments.append(lines[a][1]) + elif lines[a][0] == "property": + properties.append(PlyProperty._parse_one(lines[a])) + else: + break + + return (PlyElement(name, properties, count, comments), lines[a:]) + + @staticmethod + def describe(data, name, len_types={}, val_types={}, comments=[]): + """ + Construct a PlyElement from an array's metadata. + + len_types and val_types can be given as mappings from list + property names to type strings (like 'u1', 'f4', etc., or + 'int8', 'float32', etc.). These can be used to define the length + and value types of list properties. List property lengths + always default to type 'u1' (8-bit unsigned integer), and value + types default to 'i4' (32-bit integer). + + """ + if not isinstance(data, _np.ndarray): + raise TypeError("only numpy arrays are supported") + + if len(data.shape) != 1: + raise ValueError("only one-dimensional arrays are " "supported") + + count = len(data) + + properties = [] + descr = data.dtype.descr + + for t in descr: + if not isinstance(t[1], str): + raise ValueError("nested records not supported") + + if not t[0]: + raise ValueError("field with empty name") + + if len(t) != 2 or t[1][1] == "O": + # non-scalar field, which corresponds to a list + # property in PLY. + + if t[1][1] == "O": + if len(t) != 2: + raise ValueError("non-scalar object fields not " "supported") + + len_str = _data_type_reverse[len_types.get(t[0], "u1")] + if t[1][1] == "O": + val_type = val_types.get(t[0], "i4") + val_str = _lookup_type(val_type) + else: + val_str = _lookup_type(t[1][1:]) + + prop = PlyListProperty(t[0], len_str, val_str) + else: + val_str = _lookup_type(t[1][1:]) + prop = PlyProperty(t[0], val_str) + + properties.append(prop) + + elt = PlyElement(name, properties, count, comments) + elt.data = data + + return elt + + def _read(self, stream, text, byte_order): + """ + Read the actual data from a PLY file. + + """ + if text: + self._read_txt(stream) + else: + if self._have_list: + # There are list properties, so a simple load is + # impossible. + self._read_bin(stream, byte_order) + else: + # There are no list properties, so loading the data is + # much more straightforward. + self._data = _np.fromfile(stream, self.dtype(byte_order), self.count) + + if len(self._data) < self.count: + k = len(self._data) + del self._data + raise PlyParseError("early end-of-file", self, k) + + self._check_sanity() + + def _write(self, stream, text, byte_order): + """ + Write the data to a PLY file. + + """ + if text: + self._write_txt(stream) + else: + if self._have_list: + # There are list properties, so serialization is + # slightly complicated. + self._write_bin(stream, byte_order) + else: + # no list properties, so serialization is + # straightforward. + self.data.astype(self.dtype(byte_order), copy=False).tofile(stream) + + def _read_txt(self, stream): + """ + Load a PLY element from an ASCII-format PLY file. The element + may contain list properties. + + """ + self._data = _np.empty(self.count, dtype=self.dtype()) + + k = 0 + for line in _islice(iter(stream.readline, b""), self.count): + fields = iter(line.strip().split()) + for prop in self.properties: + try: + self._data[prop.name][k] = prop._from_fields(fields) + except StopIteration: + raise PlyParseError("early end-of-line", self, k, prop) + except ValueError: + raise PlyParseError("malformed input", self, k, prop) + try: + next(fields) + except StopIteration: + pass + else: + raise PlyParseError("expected end-of-line", self, k) + k += 1 + + if k < self.count: + del self._data + raise PlyParseError("early end-of-file", self, k) + + def _write_txt(self, stream): + """ + Save a PLY element to an ASCII-format PLY file. The element may + contain list properties. + + """ + for rec in self.data: + fields = [] + for prop in self.properties: + fields.extend(prop._to_fields(rec[prop.name])) + + _np.savetxt(stream, [fields], "%.18g", newline="\r\n") + + def _read_bin(self, stream, byte_order): + """ + Load a PLY element from a binary PLY file. The element may + contain list properties. + + """ + self._data = _np.empty(self.count, dtype=self.dtype(byte_order)) + + for k in _range(self.count): + for prop in self.properties: + try: + self._data[prop.name][k] = prop._read_bin(stream, byte_order) + except StopIteration: + raise PlyParseError("early end-of-file", self, k, prop) + + def _write_bin(self, stream, byte_order): + """ + Save a PLY element to a binary PLY file. The element may + contain list properties. + + """ + for rec in self.data: + for prop in self.properties: + prop._write_bin(rec[prop.name], stream, byte_order) + + @property + def header(self): + """ + Format this element's metadata as it would appear in a PLY + header. + + """ + lines = ["element %s %d" % (self.name, self.count)] + + # Some information is lost here, since all comments are placed + # between the 'element' line and the first property definition. + for c in self.comments: + lines.append("comment " + c) + + lines.extend(list(map(str, self.properties))) + + return "\r\n".join(lines) + + def __getitem__(self, key): + return self.data[key] + + def __setitem__(self, key, value): + self.data[key] = value + + def __str__(self): + return self.header + + def __repr__(self): + return "PlyElement(%r, %r, count=%d, comments=%r)" % ( + self.name, + self.properties, + self.count, + self.comments, + ) + + +class PlyProperty(object): + """ + PLY property description. This class is pure metadata; the data + itself is contained in PlyElement instances. + + """ + + def __init__(self, name, val_dtype): + self._name = str(name) + self._check_name() + self.val_dtype = val_dtype + + def _get_val_dtype(self): + return self._val_dtype + + def _set_val_dtype(self, val_dtype): + self._val_dtype = _data_types[_lookup_type(val_dtype)] + + val_dtype = property(_get_val_dtype, _set_val_dtype) + + @property + def name(self): + return self._name + + def _check_name(self): + if any(c.isspace() for c in self._name): + msg = "Error: property name %r contains spaces" % self._name + raise RuntimeError(msg) + + @staticmethod + def _parse_one(line): + assert line[0] == "property" + + if line[1] == "list": + if len(line) > 5: + raise PlyParseError("too many fields after " "'property list'") + if len(line) < 5: + raise PlyParseError("too few fields after " "'property list'") + + return PlyListProperty(line[4], line[2], line[3]) + + else: + if len(line) > 3: + raise PlyParseError("too many fields after " "'property'") + if len(line) < 3: + raise PlyParseError("too few fields after " "'property'") + + return PlyProperty(line[2], line[1]) + + def dtype(self, byte_order="="): + """ + Return the numpy dtype description for this property (as a tuple + of strings). + + """ + return byte_order + self.val_dtype + + def _from_fields(self, fields): + """ + Parse from generator. Raise StopIteration if the property could + not be read. + + """ + return _np.dtype(self.dtype()).type(next(fields)) + + def _to_fields(self, data): + """ + Return generator over one item. + + """ + yield _np.dtype(self.dtype()).type(data) + + def _read_bin(self, stream, byte_order): + """ + Read data from a binary stream. Raise StopIteration if the + property could not be read. + + """ + try: + return _np.fromfile(stream, self.dtype(byte_order), 1)[0] + except IndexError: + raise StopIteration + + def _write_bin(self, data, stream, byte_order): + """ + Write data to a binary stream. + + """ + _np.dtype(self.dtype(byte_order)).type(data).tofile(stream) + + def __str__(self): + val_str = _data_type_reverse[self.val_dtype] + return "property %s %s" % (val_str, self.name) + + def __repr__(self): + return "PlyProperty(%r, %r)" % (self.name, _lookup_type(self.val_dtype)) + + +class PlyListProperty(PlyProperty): + """ + PLY list property description. + + """ + + def __init__(self, name, len_dtype, val_dtype): + PlyProperty.__init__(self, name, val_dtype) + + self.len_dtype = len_dtype + + def _get_len_dtype(self): + return self._len_dtype + + def _set_len_dtype(self, len_dtype): + self._len_dtype = _data_types[_lookup_type(len_dtype)] + + len_dtype = property(_get_len_dtype, _set_len_dtype) + + def dtype(self, byte_order="="): + """ + List properties always have a numpy dtype of "object". + + """ + return "|O" + + def list_dtype(self, byte_order="="): + """ + Return the pair (len_dtype, val_dtype) (both numpy-friendly + strings). + + """ + return (byte_order + self.len_dtype, byte_order + self.val_dtype) + + def _from_fields(self, fields): + (len_t, val_t) = self.list_dtype() + + n = int(_np.dtype(len_t).type(next(fields))) + + data = _np.loadtxt(list(_islice(fields, n)), val_t, ndmin=1) + if len(data) < n: + raise StopIteration + + return data + + def _to_fields(self, data): + """ + Return generator over the (numerical) PLY representation of the + list data (length followed by actual data). + + """ + (len_t, val_t) = self.list_dtype() + + data = _np.asarray(data, dtype=val_t).ravel() + + yield _np.dtype(len_t).type(data.size) + for x in data: + yield x + + def _read_bin(self, stream, byte_order): + (len_t, val_t) = self.list_dtype(byte_order) + + try: + n = _np.fromfile(stream, len_t, 1)[0] + except IndexError: + raise StopIteration + + data = _np.fromfile(stream, val_t, n) + if len(data) < n: + raise StopIteration + + return data + + def _write_bin(self, data, stream, byte_order): + """ + Write data to a binary stream. + + """ + (len_t, val_t) = self.list_dtype(byte_order) + + data = _np.asarray(data, dtype=val_t).ravel() + + _np.array(data.size, dtype=len_t).tofile(stream) + data.tofile(stream) + + def __str__(self): + len_str = _data_type_reverse[self.len_dtype] + val_str = _data_type_reverse[self.val_dtype] + return "property list %s %s %s" % (len_str, val_str, self.name) + + def __repr__(self): + return "PlyListProperty(%r, %r, %r)" % ( + self.name, + _lookup_type(self.len_dtype), + _lookup_type(self.val_dtype), + ) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/point_cloud_extractor.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/point_cloud_extractor.py new file mode 100644 index 0000000000000000000000000000000000000000..1cbff78d9453bac7efe8359448dabcd6edb60452 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/point_cloud_extractor.py @@ -0,0 +1,98 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import glob, os +import numpy as np +import cv2 +import torch + + +def extractor(input_path, output_path): + if not os.path.exists(output_path): + os.mkdir(output_path) + + # Load Depth Camera Intrinsic + depth_intrinsic = np.loadtxt(input_path + "/intrinsic/intrinsic_depth.txt") + print("Depth intrinsic: ") + print(depth_intrinsic) + + # Compute Camrea Distance (just for demo, so you can choose the camera distance in frame sampling) + poses = sorted( + glob.glob(input_path + "/pose/*.txt"), + key=lambda a: int(os.path.basename(a).split(".")[0]), + ) + depths = sorted( + glob.glob(input_path + "/depth/*.png"), + key=lambda a: int(os.path.basename(a).split(".")[0]), + ) + colors = sorted( + glob.glob(input_path + "/color/*.png"), + key=lambda a: int(os.path.basename(a).split(".")[0]), + ) + + # # Get Aligned Point Clouds. + for ind, (pose, depth, color) in enumerate(zip(poses, depths, colors)): + name = os.path.basename(pose).split(".")[0] + + if os.path.exists(output_path + "/{}.npz".format(name)): + continue + + try: + print("=" * 50, ": {}".format(pose)) + depth_img = cv2.imread(depth, -1) # read 16bit grayscale image + mask = depth_img != 0 + color_image = cv2.imread(color) + color_image = cv2.resize(color_image, (640, 480)) + color_image = np.reshape(color_image[mask], [-1, 3]) + colors = np.zeros_like(color_image) + colors[:, 0] = color_image[:, 2] + colors[:, 1] = color_image[:, 1] + colors[:, 2] = color_image[:, 0] + + pose = np.loadtxt(poses[ind]) + print("Camera pose: ") + print(pose) + + depth_shift = 1000.0 + x, y = np.meshgrid( + np.linspace(0, depth_img.shape[1] - 1, depth_img.shape[1]), + np.linspace(0, depth_img.shape[0] - 1, depth_img.shape[0]), + ) + uv_depth = np.zeros((depth_img.shape[0], depth_img.shape[1], 3)) + uv_depth[:, :, 0] = x + uv_depth[:, :, 1] = y + uv_depth[:, :, 2] = depth_img / depth_shift + uv_depth = np.reshape(uv_depth, [-1, 3]) + uv_depth = uv_depth[np.where(uv_depth[:, 2] != 0), :].squeeze() + + intrinsic_inv = np.linalg.inv(depth_intrinsic) + fx = depth_intrinsic[0, 0] + fy = depth_intrinsic[1, 1] + cx = depth_intrinsic[0, 2] + cy = depth_intrinsic[1, 2] + bx = depth_intrinsic[0, 3] + by = depth_intrinsic[1, 3] + point_list = [] + n = uv_depth.shape[0] + points = np.ones((n, 4)) + X = (uv_depth[:, 0] - cx) * uv_depth[:, 2] / fx + bx + Y = (uv_depth[:, 1] - cy) * uv_depth[:, 2] / fy + by + points[:, 0] = X + points[:, 1] = Y + points[:, 2] = uv_depth[:, 2] + points_world = np.dot(points, np.transpose(pose)) + print(points_world.shape) + + pcd = dict(coord=points_world[:, :3], color=colors) + # pcd_save = np.zeros((points_world.shape[0], 7)) + # pcd_save[:, :3] = points_world[:, :3] + # pcd_save[:, 3:6] = colors + + # print('Saving npz file...') + # np.savez(output_path + '/{}.npz'.format(name), pcd=pcd_save) + torch.save(pcd, output_path + "/{}.pth".format(name)) + except: + continue diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/preprocess.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/preprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..818c369fa60b841736012950895c6dbaebdbef86 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/preprocess.py @@ -0,0 +1,51 @@ +import os +import argparse +import glob +import multiprocessing as mp +from concurrent.futures import ProcessPoolExecutor +from itertools import repeat +from reader import reader +from point_cloud_extractor import extractor +from compute_full_overlapping import compute_full_overlapping + + +frame_skip = 25 + + +def parse_sens(sens_dir, output_dir): + scene_id = os.path.basename(os.path.dirname(sens_dir)) + print(f"Parsing sens data{sens_dir}") + reader( + sens_dir, + os.path.join(output_dir, scene_id), + frame_skip, + export_color_images=True, + export_depth_images=True, + export_poses=True, + export_intrinsics=True, + ) + extractor( + os.path.join(output_dir, scene_id), os.path.join(output_dir, scene_id, "pcd") + ) + compute_full_overlapping(output_dir, scene_id) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset_root", + required=True, + help="Path to the ScanNet dataset containing scene folders", + ) + parser.add_argument( + "--output_root", + required=True, + help="Output path where train/val folders will be located", + ) + opt = parser.parse_args() + sens_list = sorted(glob.glob(os.path.join(opt.dataset_root, "scans/scene*/*.sens"))) + # Preprocess data. + pool = ProcessPoolExecutor(max_workers=mp.cpu_count()) + # pool = ProcessPoolExecutor(max_workers=1) + print("Processing scenes...") + _ = list(pool.map(parse_sens, sens_list, repeat(opt.output_root))) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/reader.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/reader.py new file mode 100644 index 0000000000000000000000000000000000000000..d21aa0ce88006f34775edc9d9aaf4e750d523197 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannet/scannet_pair/reader.py @@ -0,0 +1,33 @@ +import argparse +import os, sys + +from SensorData import SensorData + + +def reader( + filename, + output_path, + frame_skip, + export_color_images=False, + export_depth_images=False, + export_poses=False, + export_intrinsics=False, +): + if not os.path.exists(output_path): + os.makedirs(output_path) + + # load the data + print("loading %s..." % filename) + sd = SensorData(filename) + if export_depth_images: + sd.export_depth_images( + os.path.join(output_path, "depth"), frame_skip=frame_skip + ) + if export_color_images: + sd.export_color_images( + os.path.join(output_path, "color"), frame_skip=frame_skip + ) + if export_poses: + sd.export_poses(os.path.join(output_path, "pose"), frame_skip=frame_skip) + if export_intrinsics: + sd.export_intrinsics(os.path.join(output_path, "intrinsic")) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannetpp/preprocess_scannetpp.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannetpp/preprocess_scannetpp.py new file mode 100644 index 0000000000000000000000000000000000000000..ad820029016ff16c53ab0ad872ede9449add6a7b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/scannetpp/preprocess_scannetpp.py @@ -0,0 +1,252 @@ +""" +Preprocessing Script for ScanNet++ +modified from official preprocess code. + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import argparse +import json +import numpy as np +import pandas as pd +import open3d as o3d +import multiprocessing as mp +from collections import OrderedDict +from concurrent.futures import ProcessPoolExecutor +from itertools import repeat +from pathlib import Path + + +def parse_scene( + name, + split, + dataset_root, + output_root, + label_mapping, + class2idx, + ignore_index=-1, +): + print(f"Parsing scene {name} in {split} split") + dataset_root = Path(dataset_root) + output_root = Path(output_root) + scene_path = dataset_root / "data" / name / "scans" + mesh_path = scene_path / "mesh_aligned_0.05.ply" + segs_path = scene_path / "segments.json" + anno_path = scene_path / "segments_anno.json" + + # load mesh vertices and colors + mesh = o3d.io.read_triangle_mesh(str(mesh_path)) + + # extract mesh information + mesh.compute_vertex_normals(normalized=True) + coord = np.array(mesh.vertices).astype(np.float32) + color = (np.array(mesh.vertex_colors) * 255).astype(np.uint8) + normal = np.array(mesh.vertex_normals).astype(np.float32) + + save_path = output_root / split / name + save_path.mkdir(parents=True, exist_ok=True) + np.save(save_path / "coord.npy", coord) + np.save(save_path / "color.npy", color) + np.save(save_path / "normal.npy", normal) + + if split == "test": + return + + # get label on vertices + # load segments = vertices per segment ID + with open(segs_path) as f: + segments = json.load(f) + # load anno = (instance, groups of segments) + with open(anno_path) as f: + anno = json.load(f) + seg_indices = np.array(segments["segIndices"], dtype=np.uint32) + num_vertices = len(seg_indices) + assert num_vertices == len(coord) + semantic_gt = np.ones((num_vertices, 3), dtype=np.int16) * ignore_index + instance_gt = np.ones((num_vertices, 3), dtype=np.int16) * ignore_index + + # number of labels are used per vertex. initially 0 + # increment each time a new label is added + instance_size = np.ones((num_vertices, 3), dtype=np.int16) * np.inf + + # keep track of the size of the instance (#vertices) assigned to each vertex + # later, keep the label of the smallest instance for major label of vertices + # store inf initially so that we can pick the smallest instance + labels_used = np.zeros(num_vertices, dtype=np.int16) + + for idx, instance in enumerate(anno["segGroups"]): + label = instance["label"] + instance["label_orig"] = label + # remap label + instance["label"] = label_mapping.get(label, None) + instance["label_index"] = class2idx.get(label, ignore_index) + + if instance["label_index"] == ignore_index: + continue + # get all the vertices with segment index in this instance + # and max number of labels not yet applied + mask = np.isin(seg_indices, instance["segments"]) & (labels_used < 3) + size = mask.sum() + if size == 0: + continue + + # get the position to add the label - 0, 1, 2 + label_position = labels_used[mask] + semantic_gt[mask, label_position] = instance["label_index"] + # store all valid instance (include ignored instance) + instance_gt[mask, label_position] = instance["objectId"] + instance_size[mask, label_position] = size + labels_used[mask] += 1 + + # major label is the label of smallest instance for each vertex + # use major label for single class segmentation + # shift major label to the first column + mask = labels_used > 1 + if mask.sum() > 0: + major_label_position = np.argmin(instance_size[mask], axis=1) + + major_semantic_label = semantic_gt[mask, major_label_position] + semantic_gt[mask, major_label_position] = semantic_gt[:, 0][mask] + semantic_gt[:, 0][mask] = major_semantic_label + + major_instance_label = instance_gt[mask, major_label_position] + instance_gt[mask, major_label_position] = instance_gt[:, 0][mask] + instance_gt[:, 0][mask] = major_instance_label + + np.save(save_path / "segment.npy", semantic_gt) + np.save(save_path / "instance.npy", instance_gt) + + +def filter_map_classes(mapping, count_thresh, count_type, mapping_type): + mapping = mapping[mapping[count_type] >= count_thresh] + if mapping_type == "semantic": + map_key = "semantic_map_to" + elif mapping_type == "instance": + map_key = "instance_map_to" + else: + raise NotImplementedError + # create a dict with classes to be mapped + # classes that don't have mapping are entered as x->x + # otherwise x->y + map_dict = OrderedDict() + + for i in range(mapping.shape[0]): + row = mapping.iloc[i] + class_name = row["class"] + map_target = row[map_key] + + # map to None or some other label -> don't add this class to the label list + try: + if len(map_target) > 0: + # map to None -> don't use this class + if map_target == "None": + pass + else: + # map to something else -> use this class + map_dict[class_name] = map_target + except TypeError: + # nan values -> no mapping, keep label as is + if class_name not in map_dict: + map_dict[class_name] = class_name + + return map_dict + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset_root", + required=True, + help="Path to the ScanNet++ dataset containing data/metadata/splits.", + ) + parser.add_argument( + "--output_root", + required=True, + help="Output path where train/val/test folders will be located.", + ) + parser.add_argument( + "--ignore_index", + default=-1, + type=int, + help="Default ignore index.", + ) + parser.add_argument( + "--num_workers", + default=mp.cpu_count(), + type=int, + help="Num workers for preprocessing.", + ) + config = parser.parse_args() + + print("Loading meta data...") + config.dataset_root = Path(config.dataset_root) + config.output_root = Path(config.output_root) + + train_list = np.loadtxt( + config.dataset_root / "splits" / "nvs_sem_train.txt", + dtype=str, + ) + print("Num samples in training split:", len(train_list)) + + val_list = np.loadtxt( + config.dataset_root / "splits" / "nvs_sem_val.txt", + dtype=str, + ) + print("Num samples in validation split:", len(val_list)) + + test_list = np.loadtxt( + config.dataset_root / "splits" / "sem_test.txt", + dtype=str, + ) + print("Num samples in testing split:", len(test_list)) + + data_list = np.concatenate([train_list, val_list, test_list]) + split_list = np.concatenate( + [ + np.full_like(train_list, "train"), + np.full_like(val_list, "val"), + np.full_like(test_list, "test"), + ] + ) + + # Parsing label information and mapping + segment_class_names = np.loadtxt( + config.dataset_root / "metadata" / "semantic_benchmark" / "top100.txt", + dtype=str, + delimiter=".", # dummy delimiter to replace " " + ) + print("Num classes in segment class list:", len(segment_class_names)) + + instance_class_names = np.loadtxt( + config.dataset_root / "metadata" / "semantic_benchmark" / "top100_instance.txt", + dtype=str, + delimiter=".", # dummy delimiter to replace " " + ) + print("Num classes in instance class list:", len(instance_class_names)) + + label_mapping = pd.read_csv( + config.dataset_root / "metadata" / "semantic_benchmark" / "map_benchmark.csv" + ) + label_mapping = filter_map_classes( + label_mapping, count_thresh=0, count_type="count", mapping_type="semantic" + ) + class2idx = { + class_name: idx for (idx, class_name) in enumerate(segment_class_names) + } + + print("Processing scenes...") + pool = ProcessPoolExecutor(max_workers=config.num_workers) + _ = list( + pool.map( + parse_scene, + data_list, + split_list, + repeat(config.dataset_root), + repeat(config.output_root), + repeat(label_mapping), + repeat(class2idx), + repeat(config.ignore_index), + ) + ) + pool.shutdown() diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/structured3d/preprocess_structured3d.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/structured3d/preprocess_structured3d.py new file mode 100644 index 0000000000000000000000000000000000000000..6924dc9abf3a3c253ee80dd3b3d85454e3df35d2 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/structured3d/preprocess_structured3d.py @@ -0,0 +1,420 @@ +""" +Preprocessing Script for Structured3D + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import argparse +import io +import os +import PIL +from PIL import Image +import cv2 +import zipfile +import numpy as np +import multiprocessing as mp +from concurrent.futures import ProcessPoolExecutor +from itertools import repeat + + +VALID_CLASS_IDS_25 = ( + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 11, + 14, + 15, + 16, + 17, + 18, + 19, + 22, + 24, + 25, + 32, + 34, + 35, + 38, + 39, + 40, +) +CLASS_LABELS_25 = ( + "wall", + "floor", + "cabinet", + "bed", + "chair", + "sofa", + "table", + "door", + "window", + "picture", + "desk", + "shelves", + "curtain", + "dresser", + "pillow", + "mirror", + "ceiling", + "refrigerator", + "television", + "nightstand", + "sink", + "lamp", + "otherstructure", + "otherfurniture", + "otherprop", +) + + +def normal_from_cross_product(points_2d: np.ndarray) -> np.ndarray: + xyz_points_pad = np.pad(points_2d, ((0, 1), (0, 1), (0, 0)), mode="symmetric") + xyz_points_ver = (xyz_points_pad[:, :-1, :] - xyz_points_pad[:, 1:, :])[:-1, :, :] + xyz_points_hor = (xyz_points_pad[:-1, :, :] - xyz_points_pad[1:, :, :])[:, :-1, :] + xyz_normal = np.cross(xyz_points_hor, xyz_points_ver) + xyz_dist = np.linalg.norm(xyz_normal, axis=-1, keepdims=True) + xyz_normal = np.divide( + xyz_normal, xyz_dist, out=np.zeros_like(xyz_normal), where=xyz_dist != 0 + ) + return xyz_normal + + +class Structured3DReader: + def __init__(self, files): + super().__init__() + if isinstance(files, str): + files = [files] + self.readers = [zipfile.ZipFile(f, "r") for f in files] + self.names_mapper = dict() + for idx, reader in enumerate(self.readers): + for name in reader.namelist(): + self.names_mapper[name] = idx + + def filelist(self): + return list(self.names_mapper.keys()) + + def listdir(self, dir_name): + dir_name = dir_name.lstrip(os.path.sep).rstrip(os.path.sep) + file_list = list( + np.unique( + [ + f.replace(dir_name + os.path.sep, "", 1).split(os.path.sep)[0] + for f in self.filelist() + if f.startswith(dir_name + os.path.sep) + ] + ) + ) + if "" in file_list: + file_list.remove("") + return file_list + + def read(self, file_name): + split = self.names_mapper[file_name] + return self.readers[split].read(file_name) + + def read_camera(self, camera_path): + z2y_top_m = np.array([[0, 1, 0], [0, 0, 1], [1, 0, 0]], dtype=np.float32) + cam_extr = np.fromstring(self.read(camera_path), dtype=np.float32, sep=" ") + cam_t = np.matmul(z2y_top_m, cam_extr[:3] / 1000) + if cam_extr.shape[0] > 3: + cam_front, cam_up = cam_extr[3:6], cam_extr[6:9] + cam_n = np.cross(cam_front, cam_up) + cam_r = np.stack((cam_front, cam_up, cam_n), axis=1).astype(np.float32) + cam_r = np.matmul(z2y_top_m, cam_r) + cam_f = cam_extr[9:11] + else: + cam_r = np.eye(3, dtype=np.float32) + cam_f = None + return cam_r, cam_t, cam_f + + def read_depth(self, depth_path): + depth = cv2.imdecode( + np.frombuffer(self.read(depth_path), np.uint8), cv2.IMREAD_UNCHANGED + )[..., np.newaxis] + depth[depth == 0] = 65535 + return depth + + def read_color(self, color_path): + color = cv2.imdecode( + np.frombuffer(self.read(color_path), np.uint8), cv2.IMREAD_UNCHANGED + )[..., :3][..., ::-1] + return color + + def read_segment(self, segment_path): + segment = np.array(PIL.Image.open(io.BytesIO(self.read(segment_path))))[ + ..., np.newaxis + ] + return segment + + +def parse_scene( + scene, + dataset_root, + output_root, + ignore_index=-1, + grid_size=None, + fuse_prsp=True, + fuse_pano=True, + vis=False, +): + assert fuse_prsp or fuse_pano + reader = Structured3DReader( + [ + os.path.join(dataset_root, f) + for f in os.listdir(dataset_root) + if f.endswith(".zip") + ] + ) + scene_id = int(os.path.basename(scene).split("_")[-1]) + if scene_id < 3000: + split = "train" + elif 3000 <= scene_id < 3250: + split = "val" + else: + split = "test" + + print(f"Processing: {scene} in {split}") + rooms = reader.listdir(os.path.join("Structured3D", scene, "2D_rendering")) + for room in rooms: + room_path = os.path.join("Structured3D", scene, "2D_rendering", room) + coord_list = list() + color_list = list() + normal_list = list() + segment_list = list() + if fuse_prsp: + prsp_path = os.path.join(room_path, "perspective", "full") + frames = reader.listdir(prsp_path) + + for frame in frames: + try: + cam_r, cam_t, cam_f = reader.read_camera( + os.path.join(prsp_path, frame, "camera_pose.txt") + ) + depth = reader.read_depth( + os.path.join(prsp_path, frame, "depth.png") + ) + color = reader.read_color( + os.path.join(prsp_path, frame, "rgb_rawlight.png") + ) + segment = reader.read_segment( + os.path.join(prsp_path, frame, "semantic.png") + ) + except: + print( + f"Skipping {scene}_room{room}_frame{frame} perspective view due to loading error" + ) + else: + fx, fy = cam_f + height, width = depth.shape[0], depth.shape[1] + pixel = np.transpose(np.indices((width, height)), (2, 1, 0)) + pixel = pixel.reshape((-1, 2)) + pixel = np.hstack((pixel, np.ones((pixel.shape[0], 1)))) + k = np.diag([1.0, 1.0, 1.0]) + + k[0, 2] = width / 2 + k[1, 2] = height / 2 + + k[0, 0] = k[0, 2] / np.tan(fx) + k[1, 1] = k[1, 2] / np.tan(fy) + coord = ( + depth.reshape((-1, 1)) * (np.linalg.inv(k) @ pixel.T).T + ).reshape(height, width, 3) + coord = coord @ np.array([[0, 0, 1], [0, -1, 0], [1, 0, 0]]) + normal = normal_from_cross_product(coord) + + # Filtering invalid points + view_dist = np.maximum( + np.linalg.norm(coord, axis=-1, keepdims=True), float(10e-5) + ) + cosine_dist = np.sum( + (coord * normal / view_dist), axis=-1, keepdims=True + ) + cosine_dist = np.abs(cosine_dist) + mask = ((cosine_dist > 0.15) & (depth < 65535) & (segment > 0))[ + ..., 0 + ].reshape(-1) + + coord = np.matmul(coord / 1000, cam_r.T) + cam_t + normal = normal_from_cross_product(coord) + + if sum(mask) > 0: + coord_list.append(coord.reshape(-1, 3)[mask]) + color_list.append(color.reshape(-1, 3)[mask]) + normal_list.append(normal.reshape(-1, 3)[mask]) + segment_list.append(segment.reshape(-1, 1)[mask]) + else: + print( + f"Skipping {scene}_room{room}_frame{frame} perspective view due to all points are filtered out" + ) + + if fuse_pano: + pano_path = os.path.join(room_path, "panorama") + try: + _, cam_t, _ = reader.read_camera( + os.path.join(pano_path, "camera_xyz.txt") + ) + depth = reader.read_depth(os.path.join(pano_path, "full", "depth.png")) + color = reader.read_color( + os.path.join(pano_path, "full", "rgb_rawlight.png") + ) + segment = reader.read_segment( + os.path.join(pano_path, "full", "semantic.png") + ) + except: + print(f"Skipping {scene}_room{room} panorama view due to loading error") + else: + p_h, p_w = depth.shape[:2] + p_a = np.arange(p_w, dtype=np.float32) / p_w * 2 * np.pi - np.pi + p_b = np.arange(p_h, dtype=np.float32) / p_h * np.pi * -1 + np.pi / 2 + p_a = np.tile(p_a[None], [p_h, 1])[..., np.newaxis] + p_b = np.tile(p_b[:, None], [1, p_w])[..., np.newaxis] + p_a_sin, p_a_cos, p_b_sin, p_b_cos = ( + np.sin(p_a), + np.cos(p_a), + np.sin(p_b), + np.cos(p_b), + ) + x = depth * p_a_cos * p_b_cos + y = depth * p_b_sin + z = depth * p_a_sin * p_b_cos + coord = np.concatenate([x, y, z], axis=-1) / 1000 + normal = normal_from_cross_product(coord) + + # Filtering invalid points + view_dist = np.maximum( + np.linalg.norm(coord, axis=-1, keepdims=True), float(10e-5) + ) + cosine_dist = np.sum( + (coord * normal / view_dist), axis=-1, keepdims=True + ) + cosine_dist = np.abs(cosine_dist) + mask = ((cosine_dist > 0.15) & (depth < 65535) & (segment > 0))[ + ..., 0 + ].reshape(-1) + coord = coord + cam_t + + if sum(mask) > 0: + coord_list.append(coord.reshape(-1, 3)[mask]) + color_list.append(color.reshape(-1, 3)[mask]) + normal_list.append(normal.reshape(-1, 3)[mask]) + segment_list.append(segment.reshape(-1, 1)[mask]) + else: + print( + f"Skipping {scene}_room{room} panorama view due to all points are filtered out" + ) + + if len(coord_list) > 0: + coord = np.concatenate(coord_list, axis=0) + coord = coord @ np.array([[1, 0, 0], [0, 0, 1], [0, 1, 0]]) + color = np.concatenate(color_list, axis=0) + normal = np.concatenate(normal_list, axis=0) + normal = normal @ np.array([[1, 0, 0], [0, 0, 1], [0, 1, 0]]) + segment = np.concatenate(segment_list, axis=0) + segment25 = np.ones_like(segment, dtype=np.int64) * ignore_index + for idx, value in enumerate(VALID_CLASS_IDS_25): + mask = np.all(segment == value, axis=-1) + segment25[mask] = idx + + data_dict = dict( + coord=coord.astype(np.float32), + color=color.astype(np.uint8), + normal=normal.astype(np.float32), + segment=segment25.astype(np.int16), + ) + # Grid sampling data + if grid_size is not None: + grid_coord = np.floor(coord / grid_size).astype(int) + _, idx = np.unique(grid_coord, axis=0, return_index=True) + coord = coord[idx] + for key in data_dict.keys(): + data_dict[key] = data_dict[key][idx] + + # Save data + save_path = os.path.join( + output_root, split, os.path.basename(scene), f"room_{room}" + ) + os.makedirs(save_path, exist_ok=True) + for key in data_dict.keys(): + np.save(os.path.join(save_path, f"{key}.npy"), data_dict[key]) + + if vis: + from pointcept.utils.visualization import save_point_cloud + + os.makedirs("./vis", exist_ok=True) + save_point_cloud( + coord, color / 255, f"./vis/{scene}_room{room}_color.ply" + ) + save_point_cloud( + coord, (normal + 1) / 2, f"./vis/{scene}_room{room}_normal.ply" + ) + else: + print(f"Skipping {scene}_room{room} due to no valid points") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset_root", + required=True, + help="Path to the ScanNet dataset containing scene folders.", + ) + parser.add_argument( + "--output_root", + required=True, + help="Output path where train/val folders will be located.", + ) + parser.add_argument( + "--num_workers", + default=mp.cpu_count(), + type=int, + help="Num workers for preprocessing.", + ) + parser.add_argument( + "--grid_size", default=None, type=float, help="Grid size for grid sampling." + ) + parser.add_argument("--ignore_index", default=-1, type=float, help="Ignore index.") + parser.add_argument( + "--fuse_prsp", action="store_true", help="Whether fuse perspective view." + ) + parser.add_argument( + "--fuse_pano", action="store_true", help="Whether fuse panorama view." + ) + config = parser.parse_args() + + reader = Structured3DReader( + [ + os.path.join(config.dataset_root, f) + for f in os.listdir(config.dataset_root) + if f.endswith(".zip") + ] + ) + + scenes_list = reader.listdir("Structured3D") + scenes_list = sorted(scenes_list) + os.makedirs(os.path.join(config.output_root, "train"), exist_ok=True) + os.makedirs(os.path.join(config.output_root, "val"), exist_ok=True) + os.makedirs(os.path.join(config.output_root, "test"), exist_ok=True) + + # Preprocess data. + print("Processing scenes...") + pool = ProcessPoolExecutor(max_workers=config.num_workers) + _ = list( + pool.map( + parse_scene, + scenes_list, + repeat(config.dataset_root), + repeat(config.output_root), + repeat(config.ignore_index), + repeat(config.grid_size), + repeat(config.fuse_prsp), + repeat(config.fuse_pano), + ) + ) + pool.shutdown() diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/waymo/3d_semseg_test_set_frames.txt b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/waymo/3d_semseg_test_set_frames.txt new file mode 100644 index 0000000000000000000000000000000000000000..e25dc615331aaff261c25bfc3e6b930fccf880ac --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/waymo/3d_semseg_test_set_frames.txt @@ -0,0 +1,2982 @@ +2830680430134047327_1720_000_1740_000,1558034229922468 +2830680430134047327_1720_000_1740_000,1558034232422787 +2830680430134047327_1720_000_1740_000,1558034222922333 +2830680430134047327_1720_000_1740_000,1558034223422411 +2830680430134047327_1720_000_1740_000,1558034232922788 +2830680430134047327_1720_000_1740_000,1558034234422988 +2830680430134047327_1720_000_1740_000,1558034220422269 +2830680430134047327_1720_000_1740_000,1558034224422390 +2830680430134047327_1720_000_1740_000,1558034230422275 +2830680430134047327_1720_000_1740_000,1558034221922367 +2830680430134047327_1720_000_1740_000,1558034231422462 +2830680430134047327_1720_000_1740_000,1558034233422761 +2830680430134047327_1720_000_1740_000,1558034220922343 +2830680430134047327_1720_000_1740_000,1558034223922363 +2830680430134047327_1720_000_1740_000,1558034221422343 +2830680430134047327_1720_000_1740_000,1558034222422363 +2830680430134047327_1720_000_1740_000,1558034233922841 +2830680430134047327_1720_000_1740_000,1558034231922698 +2830680430134047327_1720_000_1740_000,1558034219922361 +2830680430134047327_1720_000_1740_000,1558034230922233 +14586026017427828517_700_000_720_000,1557363460737615 +14586026017427828517_700_000_720_000,1557363461737535 +14586026017427828517_700_000_720_000,1557363470737492 +14586026017427828517_700_000_720_000,1557363458737857 +14586026017427828517_700_000_720_000,1557363471237472 +14586026017427828517_700_000_720_000,1557363459237861 +14586026017427828517_700_000_720_000,1557363460237653 +14586026017427828517_700_000_720_000,1557363467737151 +14586026017427828517_700_000_720_000,1557363469237474 +14586026017427828517_700_000_720_000,1557363468237103 +14586026017427828517_700_000_720_000,1557363461237576 +14586026017427828517_700_000_720_000,1557363457237421 +14586026017427828517_700_000_720_000,1557363467237306 +14586026017427828517_700_000_720_000,1557363468737287 +14586026017427828517_700_000_720_000,1557363459737792 +14586026017427828517_700_000_720_000,1557363469737664 +14586026017427828517_700_000_720_000,1557363458237773 +14586026017427828517_700_000_720_000,1557363471737431 +14586026017427828517_700_000_720_000,1557363470237558 +14586026017427828517_700_000_720_000,1557363457737578 +6079272500228273268_2480_000_2500_000,1557546171272882 +6079272500228273268_2480_000_2500_000,1557546172772631 +6079272500228273268_2480_000_2500_000,1557546159772245 +6079272500228273268_2480_000_2500_000,1557546169773016 +6079272500228273268_2480_000_2500_000,1557546162272567 +6079272500228273268_2480_000_2500_000,1557546170272998 +6079272500228273268_2480_000_2500_000,1557546161272265 +6079272500228273268_2480_000_2500_000,1557546173772741 +6079272500228273268_2480_000_2500_000,1557546170772958 +6079272500228273268_2480_000_2500_000,1557546163272476 +6079272500228273268_2480_000_2500_000,1557546161772481 +6079272500228273268_2480_000_2500_000,1557546171772762 +6079272500228273268_2480_000_2500_000,1557546163772228 +6079272500228273268_2480_000_2500_000,1557546160772102 +6079272500228273268_2480_000_2500_000,1557546162772554 +6079272500228273268_2480_000_2500_000,1557546172272635 +6079272500228273268_2480_000_2500_000,1557546159272357 +6079272500228273268_2480_000_2500_000,1557546173272639 +6079272500228273268_2480_000_2500_000,1557546169272728 +6079272500228273268_2480_000_2500_000,1557546160272024 +1936395688683397781_2580_000_2600_000,1557546260797397 +1936395688683397781_2580_000_2600_000,1557546263297538 +1936395688683397781_2580_000_2600_000,1557546273797478 +1936395688683397781_2580_000_2600_000,1557546259797381 +1936395688683397781_2580_000_2600_000,1557546261797507 +1936395688683397781_2580_000_2600_000,1557546269797705 +1936395688683397781_2580_000_2600_000,1557546271297274 +1936395688683397781_2580_000_2600_000,1557546260297363 +1936395688683397781_2580_000_2600_000,1557546273297464 +1936395688683397781_2580_000_2600_000,1557546272297368 +1936395688683397781_2580_000_2600_000,1557546261297445 +1936395688683397781_2580_000_2600_000,1557546263797578 +1936395688683397781_2580_000_2600_000,1557546270297559 +1936395688683397781_2580_000_2600_000,1557546269297756 +1936395688683397781_2580_000_2600_000,1557546270797376 +1936395688683397781_2580_000_2600_000,1557546262797541 +1936395688683397781_2580_000_2600_000,1557546259297393 +1936395688683397781_2580_000_2600_000,1557546272797434 +1936395688683397781_2580_000_2600_000,1557546262297511 +1936395688683397781_2580_000_2600_000,1557546271797317 +12537711031998520792_3080_000_3100_000,1559178584137607 +12537711031998520792_3080_000_3100_000,1559178596138055 +12537711031998520792_3080_000_3100_000,1559178598137718 +12537711031998520792_3080_000_3100_000,1559178588137379 +12537711031998520792_3080_000_3100_000,1559178588637547 +12537711031998520792_3080_000_3100_000,1559178597637619 +12537711031998520792_3080_000_3100_000,1559178587137003 +12537711031998520792_3080_000_3100_000,1559178594138804 +12537711031998520792_3080_000_3100_000,1559178584637645 +12537711031998520792_3080_000_3100_000,1559178587637113 +12537711031998520792_3080_000_3100_000,1559178598637642 +12537711031998520792_3080_000_3100_000,1559178595638405 +12537711031998520792_3080_000_3100_000,1559178594638876 +12537711031998520792_3080_000_3100_000,1559178585137535 +12537711031998520792_3080_000_3100_000,1559178586637145 +12537711031998520792_3080_000_3100_000,1559178595138665 +12537711031998520792_3080_000_3100_000,1559178585637309 +12537711031998520792_3080_000_3100_000,1559178586137201 +12537711031998520792_3080_000_3100_000,1559178597137539 +12537711031998520792_3080_000_3100_000,1559178596637690 +614453665074997770_1060_000_1080_000,1557449600160586 +614453665074997770_1060_000_1080_000,1557449609661257 +614453665074997770_1060_000_1080_000,1557449599161330 +614453665074997770_1060_000_1080_000,1557449608161401 +614453665074997770_1060_000_1080_000,1557449607661221 +614453665074997770_1060_000_1080_000,1557449598662064 +614453665074997770_1060_000_1080_000,1557449600660450 +614453665074997770_1060_000_1080_000,1557449596170831 +614453665074997770_1060_000_1080_000,1557449610161229 +614453665074997770_1060_000_1080_000,1557449607161133 +614453665074997770_1060_000_1080_000,1557449610661288 +614453665074997770_1060_000_1080_000,1557449597165803 +614453665074997770_1060_000_1080_000,1557449606161418 +614453665074997770_1060_000_1080_000,1557449599660888 +614453665074997770_1060_000_1080_000,1557449597664327 +614453665074997770_1060_000_1080_000,1557449609161403 +614453665074997770_1060_000_1080_000,1557449606661312 +614453665074997770_1060_000_1080_000,1557449596667769 +614453665074997770_1060_000_1080_000,1557449598163004 +614453665074997770_1060_000_1080_000,1557449608661487 +10488772413132920574_680_000_700_000,1557276689370724 +10488772413132920574_680_000_700_000,1557276681870683 +10488772413132920574_680_000_700_000,1557276687370626 +10488772413132920574_680_000_700_000,1557276691870689 +10488772413132920574_680_000_700_000,1557276677372053 +10488772413132920574_680_000_700_000,1557276688370712 +10488772413132920574_680_000_700_000,1557276689870718 +10488772413132920574_680_000_700_000,1557276687870742 +10488772413132920574_680_000_700_000,1557276690370698 +10488772413132920574_680_000_700_000,1557276690870678 +10488772413132920574_680_000_700_000,1557276677871875 +10488772413132920574_680_000_700_000,1557276679370967 +10488772413132920574_680_000_700_000,1557276678871230 +10488772413132920574_680_000_700_000,1557276688870677 +10488772413132920574_680_000_700_000,1557276691370685 +10488772413132920574_680_000_700_000,1557276680370783 +10488772413132920574_680_000_700_000,1557276678371569 +10488772413132920574_680_000_700_000,1557276679870868 +10488772413132920574_680_000_700_000,1557276680870747 +10488772413132920574_680_000_700_000,1557276681370633 +17174012103392027911_3500_000_3520_000,1570909461425651 +17174012103392027911_3500_000_3520_000,1570909452924866 +17174012103392027911_3500_000_3520_000,1570909453924848 +17174012103392027911_3500_000_3520_000,1570909451425527 +17174012103392027911_3500_000_3520_000,1570909450426192 +17174012103392027911_3500_000_3520_000,1570909460425382 +17174012103392027911_3500_000_3520_000,1570909449427011 +17174012103392027911_3500_000_3520_000,1570909449926640 +17174012103392027911_3500_000_3520_000,1570909459925389 +17174012103392027911_3500_000_3520_000,1570909459425389 +17174012103392027911_3500_000_3520_000,1570909461925903 +17174012103392027911_3500_000_3520_000,1570909462926099 +17174012103392027911_3500_000_3520_000,1570909452425024 +17174012103392027911_3500_000_3520_000,1570909450925798 +17174012103392027911_3500_000_3520_000,1570909463926171 +17174012103392027911_3500_000_3520_000,1570909451925298 +17174012103392027911_3500_000_3520_000,1570909463426072 +17174012103392027911_3500_000_3520_000,1570909462426098 +17174012103392027911_3500_000_3520_000,1570909453424839 +17174012103392027911_3500_000_3520_000,1570909460925464 +16062780403777359835_2580_000_2600_000,1570323463399590 +16062780403777359835_2580_000_2600_000,1570323461899536 +16062780403777359835_2580_000_2600_000,1570323456400014 +16062780403777359835_2580_000_2600_000,1570323465899811 +16062780403777359835_2580_000_2600_000,1570323452400102 +16062780403777359835_2580_000_2600_000,1570323454400277 +16062780403777359835_2580_000_2600_000,1570323454900086 +16062780403777359835_2580_000_2600_000,1570323464399655 +16062780403777359835_2580_000_2600_000,1570323452900168 +16062780403777359835_2580_000_2600_000,1570323453400256 +16062780403777359835_2580_000_2600_000,1570323462899433 +16062780403777359835_2580_000_2600_000,1570323451900017 +16062780403777359835_2580_000_2600_000,1570323466399934 +16062780403777359835_2580_000_2600_000,1570323464899679 +16062780403777359835_2580_000_2600_000,1570323455399863 +16062780403777359835_2580_000_2600_000,1570323453900317 +16062780403777359835_2580_000_2600_000,1570323462399389 +16062780403777359835_2580_000_2600_000,1570323455899851 +16062780403777359835_2580_000_2600_000,1570323465399682 +16062780403777359835_2580_000_2600_000,1570323463899688 +1376304843325714018_3420_000_3440_000,1557855926972381 +1376304843325714018_3420_000_3440_000,1557855912972456 +1376304843325714018_3420_000_3440_000,1557855925972176 +1376304843325714018_3420_000_3440_000,1557855927472462 +1376304843325714018_3420_000_3440_000,1557855917472088 +1376304843325714018_3420_000_3440_000,1557855914472215 +1376304843325714018_3420_000_3440_000,1557855923972406 +1376304843325714018_3420_000_3440_000,1557855914972144 +1376304843325714018_3420_000_3440_000,1557855915972108 +1376304843325714018_3420_000_3440_000,1557855924472251 +1376304843325714018_3420_000_3440_000,1557855926472255 +1376304843325714018_3420_000_3440_000,1557855913472347 +1376304843325714018_3420_000_3440_000,1557855923472548 +1376304843325714018_3420_000_3440_000,1557855915472102 +1376304843325714018_3420_000_3440_000,1557855922972694 +1376304843325714018_3420_000_3440_000,1557855924972252 +1376304843325714018_3420_000_3440_000,1557855916472106 +1376304843325714018_3420_000_3440_000,1557855925472198 +1376304843325714018_3420_000_3440_000,1557855913972269 +1376304843325714018_3420_000_3440_000,1557855916972142 +5648007586817904385_3220_000_3240_000,1569901291300092 +5648007586817904385_3220_000_3240_000,1569901302299589 +5648007586817904385_3220_000_3240_000,1569901290300004 +5648007586817904385_3220_000_3240_000,1569901302799659 +5648007586817904385_3220_000_3240_000,1569901301799512 +5648007586817904385_3220_000_3240_000,1569901290800085 +5648007586817904385_3220_000_3240_000,1569901293800265 +5648007586817904385_3220_000_3240_000,1569901292800206 +5648007586817904385_3220_000_3240_000,1569901300799428 +5648007586817904385_3220_000_3240_000,1569901293300205 +5648007586817904385_3220_000_3240_000,1569901294300108 +5648007586817904385_3220_000_3240_000,1569901301299422 +5648007586817904385_3220_000_3240_000,1569901303299757 +5648007586817904385_3220_000_3240_000,1569901304799913 +5648007586817904385_3220_000_3240_000,1569901303799751 +5648007586817904385_3220_000_3240_000,1569901291800157 +5648007586817904385_3220_000_3240_000,1569901304299911 +5648007586817904385_3220_000_3240_000,1569901292300126 +5648007586817904385_3220_000_3240_000,1569901300299426 +14470988792985854683_760_000_780_000,1567607330924872 +14470988792985854683_760_000_780_000,1567607341924939 +14470988792985854683_760_000_780_000,1567607339424824 +14470988792985854683_760_000_780_000,1567607339924818 +14470988792985854683_760_000_780_000,1567607332924840 +14470988792985854683_760_000_780_000,1567607329924795 +14470988792985854683_760_000_780_000,1567607332424911 +14470988792985854683_760_000_780_000,1567607340924888 +14470988792985854683_760_000_780_000,1567607330424797 +14470988792985854683_760_000_780_000,1567607342924753 +14470988792985854683_760_000_780_000,1567607340424823 +14470988792985854683_760_000_780_000,1567607342424852 +14470988792985854683_760_000_780_000,1567607331424817 +14470988792985854683_760_000_780_000,1567607329424827 +14470988792985854683_760_000_780_000,1567607338924788 +14470988792985854683_760_000_780_000,1567607338424807 +14470988792985854683_760_000_780_000,1567607331924837 +14470988792985854683_760_000_780_000,1567607341424947 +14470988792985854683_760_000_780_000,1567607328424803 +14470988792985854683_760_000_780_000,1567607328924826 +16951245307634830999_1400_000_1420_000,1568599891824038 +16951245307634830999_1400_000_1420_000,1568599901824641 +16951245307634830999_1400_000_1420_000,1568599904324788 +16951245307634830999_1400_000_1420_000,1568599903324933 +16951245307634830999_1400_000_1420_000,1568599902824777 +16951245307634830999_1400_000_1420_000,1568599904824728 +16951245307634830999_1400_000_1420_000,1568599895824670 +16951245307634830999_1400_000_1420_000,1568599891324145 +16951245307634830999_1400_000_1420_000,1568599894824764 +16951245307634830999_1400_000_1420_000,1568599893824574 +16951245307634830999_1400_000_1420_000,1568599894324795 +16951245307634830999_1400_000_1420_000,1568599905824790 +16951245307634830999_1400_000_1420_000,1568599903824893 +16951245307634830999_1400_000_1420_000,1568599902324620 +16951245307634830999_1400_000_1420_000,1568599905324768 +16951245307634830999_1400_000_1420_000,1568599893324265 +16951245307634830999_1400_000_1420_000,1568599892824082 +16951245307634830999_1400_000_1420_000,1568599895324702 +16951245307634830999_1400_000_1420_000,1568599892323941 +17835886859721116155_1860_000_1880_000,1558151678787439 +17835886859721116155_1860_000_1880_000,1558151676287513 +17835886859721116155_1860_000_1880_000,1558151670787584 +17835886859721116155_1860_000_1880_000,1558151680287177 +17835886859721116155_1860_000_1880_000,1558151670287284 +17835886859721116155_1860_000_1880_000,1558151679287439 +17835886859721116155_1860_000_1880_000,1558151679787332 +17835886859721116155_1860_000_1880_000,1558151680787183 +17835886859721116155_1860_000_1880_000,1558151668786681 +17835886859721116155_1860_000_1880_000,1558151678287466 +17835886859721116155_1860_000_1880_000,1558151667787272 +17835886859721116155_1860_000_1880_000,1558151677287479 +17835886859721116155_1860_000_1880_000,1558151669286533 +17835886859721116155_1860_000_1880_000,1558151669786756 +17835886859721116155_1860_000_1880_000,1558151676787561 +17835886859721116155_1860_000_1880_000,1558151668286995 +17835886859721116155_1860_000_1880_000,1558151666786923 +17835886859721116155_1860_000_1880_000,1558151677787410 +17835886859721116155_1860_000_1880_000,1558151667287257 +17835886859721116155_1860_000_1880_000,1558151666286432 +9145030426583202228_1060_000_1080_000,1557424274778866 +9145030426583202228_1060_000_1080_000,1557424275778987 +9145030426583202228_1060_000_1080_000,1557424266779291 +9145030426583202228_1060_000_1080_000,1557424278279219 +9145030426583202228_1060_000_1080_000,1557424276779170 +9145030426583202228_1060_000_1080_000,1557424279279575 +9145030426583202228_1060_000_1080_000,1557424268279175 +9145030426583202228_1060_000_1080_000,1557424277779106 +9145030426583202228_1060_000_1080_000,1557424266279249 +9145030426583202228_1060_000_1080_000,1557424269279152 +9145030426583202228_1060_000_1080_000,1557424268779150 +9145030426583202228_1060_000_1080_000,1557424277279133 +9145030426583202228_1060_000_1080_000,1557424275278791 +9145030426583202228_1060_000_1080_000,1557424265779130 +9145030426583202228_1060_000_1080_000,1557424264779014 +9145030426583202228_1060_000_1080_000,1557424265279048 +9145030426583202228_1060_000_1080_000,1557424267279322 +9145030426583202228_1060_000_1080_000,1557424276279143 +9145030426583202228_1060_000_1080_000,1557424278779413 +9145030426583202228_1060_000_1080_000,1557424267779293 +13781857304705519152_2740_000_2760_000,1558018015472758 +13781857304705519152_2740_000_2760_000,1558018006972289 +13781857304705519152_2740_000_2760_000,1558018007472306 +13781857304705519152_2740_000_2760_000,1558018014472458 +13781857304705519152_2740_000_2760_000,1558018017472179 +13781857304705519152_2740_000_2760_000,1558018014972710 +13781857304705519152_2740_000_2760_000,1558018008472276 +13781857304705519152_2740_000_2760_000,1558018006472299 +13781857304705519152_2740_000_2760_000,1558018004472242 +13781857304705519152_2740_000_2760_000,1558018017972558 +13781857304705519152_2740_000_2760_000,1558018004972259 +13781857304705519152_2740_000_2760_000,1558018007972307 +13781857304705519152_2740_000_2760_000,1558018013972483 +13781857304705519152_2740_000_2760_000,1558018005972338 +13781857304705519152_2740_000_2760_000,1558018016972032 +13781857304705519152_2740_000_2760_000,1558018015972514 +13781857304705519152_2740_000_2760_000,1558018005472310 +13781857304705519152_2740_000_2760_000,1558018003972238 +13781857304705519152_2740_000_2760_000,1558018018472666 +13781857304705519152_2740_000_2760_000,1558018016472185 +5154724129640787887_4840_000_4860_000,1557342396562648 +5154724129640787887_4840_000_4860_000,1557342399062810 +5154724129640787887_4840_000_4860_000,1557342395062806 +5154724129640787887_4840_000_4860_000,1557342405062520 +5154724129640787887_4840_000_4860_000,1557342399562770 +5154724129640787887_4840_000_4860_000,1557342395562652 +5154724129640787887_4840_000_4860_000,1557342406562476 +5154724129640787887_4840_000_4860_000,1557342408562474 +5154724129640787887_4840_000_4860_000,1557342406062444 +5154724129640787887_4840_000_4860_000,1557342397562592 +5154724129640787887_4840_000_4860_000,1557342407562646 +5154724129640787887_4840_000_4860_000,1557342396062602 +5154724129640787887_4840_000_4860_000,1557342409562395 +5154724129640787887_4840_000_4860_000,1557342397062617 +5154724129640787887_4840_000_4860_000,1557342409062401 +5154724129640787887_4840_000_4860_000,1557342398062702 +5154724129640787887_4840_000_4860_000,1557342407062596 +5154724129640787887_4840_000_4860_000,1557342405562490 +5154724129640787887_4840_000_4860_000,1557342408062539 +5154724129640787887_4840_000_4860_000,1557342398562701 +12892154548237137398_2820_000_2840_000,1558018087522764 +12892154548237137398_2820_000_2840_000,1558018098022390 +12892154548237137398_2820_000_2840_000,1558018088022638 +12892154548237137398_2820_000_2840_000,1558018095522691 +12892154548237137398_2820_000_2840_000,1558018087022717 +12892154548237137398_2820_000_2840_000,1558018086022213 +12892154548237137398_2820_000_2840_000,1558018086522385 +12892154548237137398_2820_000_2840_000,1558018085522203 +12892154548237137398_2820_000_2840_000,1558018094522190 +12892154548237137398_2820_000_2840_000,1558018084022848 +12892154548237137398_2820_000_2840_000,1558018085022352 +12892154548237137398_2820_000_2840_000,1558018088522537 +12892154548237137398_2820_000_2840_000,1558018084522834 +12892154548237137398_2820_000_2840_000,1558018097022451 +12892154548237137398_2820_000_2840_000,1558018097522376 +12892154548237137398_2820_000_2840_000,1558018098522395 +12892154548237137398_2820_000_2840_000,1558018096022561 +12892154548237137398_2820_000_2840_000,1558018096522494 +12892154548237137398_2820_000_2840_000,1558018094021934 +12892154548237137398_2820_000_2840_000,1558018095022568 +17262030607996041518_540_000_560_000,1558150357737631 +17262030607996041518_540_000_560_000,1558150360737468 +17262030607996041518_540_000_560_000,1558150358737355 +17262030607996041518_540_000_560_000,1558150346737340 +17262030607996041518_540_000_560_000,1558150350737099 +17262030607996041518_540_000_560_000,1558150347237353 +17262030607996041518_540_000_560_000,1558150349237231 +17262030607996041518_540_000_560_000,1558150348237167 +17262030607996041518_540_000_560_000,1558150359237305 +17262030607996041518_540_000_560_000,1558150348737035 +17262030607996041518_540_000_560_000,1558150359737335 +17262030607996041518_540_000_560_000,1558150347737351 +17262030607996041518_540_000_560_000,1558150350237481 +17262030607996041518_540_000_560_000,1558150356237309 +17262030607996041518_540_000_560_000,1558150349737529 +17262030607996041518_540_000_560_000,1558150356737414 +17262030607996041518_540_000_560_000,1558150346237488 +17262030607996041518_540_000_560_000,1558150358237512 +17262030607996041518_540_000_560_000,1558150360237386 +17262030607996041518_540_000_560_000,1558150357237609 +1735154401471216485_440_000_460_000,1566351679575063 +1735154401471216485_440_000_460_000,1566351680574951 +1735154401471216485_440_000_460_000,1566351667075023 +1735154401471216485_440_000_460_000,1566351668074924 +1735154401471216485_440_000_460_000,1566351681074884 +1735154401471216485_440_000_460_000,1566351679075007 +1735154401471216485_440_000_460_000,1566351671574819 +1735154401471216485_440_000_460_000,1566351670575041 +1735154401471216485_440_000_460_000,1566351681574847 +1735154401471216485_440_000_460_000,1566351678574927 +1735154401471216485_440_000_460_000,1566351667575012 +1735154401471216485_440_000_460_000,1566351668574986 +1735154401471216485_440_000_460_000,1566351678074851 +1735154401471216485_440_000_460_000,1566351670075165 +1735154401471216485_440_000_460_000,1566351671074932 +1735154401471216485_440_000_460_000,1566351680075032 +1735154401471216485_440_000_460_000,1566351677075266 +1735154401471216485_440_000_460_000,1566351669075103 +1735154401471216485_440_000_460_000,1566351669575114 +1735154401471216485_440_000_460_000,1566351677575057 +16721473705085324478_2580_000_2600_000,1559143954073968 +16721473705085324478_2580_000_2600_000,1559143946067629 +16721473705085324478_2580_000_2600_000,1559143948570004 +16721473705085324478_2580_000_2600_000,1559143957574188 +16721473705085324478_2580_000_2600_000,1559143945567167 +16721473705085324478_2580_000_2600_000,1559143945066818 +16721473705085324478_2580_000_2600_000,1559143947068541 +16721473705085324478_2580_000_2600_000,1559143956574149 +16721473705085324478_2580_000_2600_000,1559143958574172 +16721473705085324478_2580_000_2600_000,1559143955573951 +16721473705085324478_2580_000_2600_000,1559143957074228 +16721473705085324478_2580_000_2600_000,1559143947568997 +16721473705085324478_2580_000_2600_000,1559143944066354 +16721473705085324478_2580_000_2600_000,1559143954573995 +16721473705085324478_2580_000_2600_000,1559143946568047 +16721473705085324478_2580_000_2600_000,1559143956074028 +16721473705085324478_2580_000_2600_000,1559143948069446 +16721473705085324478_2580_000_2600_000,1559143944566550 +16721473705085324478_2580_000_2600_000,1559143955073909 +16721473705085324478_2580_000_2600_000,1559143958074171 +5046614299208670619_1760_000_1780_000,1557859931896818 +5046614299208670619_1760_000_1780_000,1557859942397098 +5046614299208670619_1760_000_1780_000,1557859941397484 +5046614299208670619_1760_000_1780_000,1557859941897278 +5046614299208670619_1760_000_1780_000,1557859939397451 +5046614299208670619_1760_000_1780_000,1557859929394856 +5046614299208670619_1760_000_1780_000,1557859938397438 +5046614299208670619_1760_000_1780_000,1557859931396740 +5046614299208670619_1760_000_1780_000,1557859940397424 +5046614299208670619_1760_000_1780_000,1557859930896546 +5046614299208670619_1760_000_1780_000,1557859939897429 +5046614299208670619_1760_000_1780_000,1557859929895843 +5046614299208670619_1760_000_1780_000,1557859928893310 +5046614299208670619_1760_000_1780_000,1557859938897438 +5046614299208670619_1760_000_1780_000,1557859940897554 +5046614299208670619_1760_000_1780_000,1557859942897115 +5046614299208670619_1760_000_1780_000,1557859932897120 +5046614299208670619_1760_000_1780_000,1557859930396386 +5046614299208670619_1760_000_1780_000,1557859928391171 +5046614299208670619_1760_000_1780_000,1557859932396854 +6259508587655502768_780_000_800_000,1557843985062519 +6259508587655502768_780_000_800_000,1557843985562521 +6259508587655502768_780_000_800_000,1557843976562766 +6259508587655502768_780_000_800_000,1557843976062713 +6259508587655502768_780_000_800_000,1557843978562284 +6259508587655502768_780_000_800_000,1557843989062285 +6259508587655502768_780_000_800_000,1557843979062370 +6259508587655502768_780_000_800_000,1557843988562341 +6259508587655502768_780_000_800_000,1557843977562120 +6259508587655502768_780_000_800_000,1557843975562542 +6259508587655502768_780_000_800_000,1557843977062493 +6259508587655502768_780_000_800_000,1557843978062117 +6259508587655502768_780_000_800_000,1557843986562332 +6259508587655502768_780_000_800_000,1557843975062365 +6259508587655502768_780_000_800_000,1557843988062465 +6259508587655502768_780_000_800_000,1557843986062494 +6259508587655502768_780_000_800_000,1557843987062399 +6259508587655502768_780_000_800_000,1557843979562469 +6259508587655502768_780_000_800_000,1557843987562501 +6259508587655502768_780_000_800_000,1557843989562412 +11436803605426256250_1720_000_1740_000,1558151527787782 +11436803605426256250_1720_000_1740_000,1558151526787865 +11436803605426256250_1720_000_1740_000,1558151528287716 +11436803605426256250_1720_000_1740_000,1558151530287466 +11436803605426256250_1720_000_1740_000,1558151537786930 +11436803605426256250_1720_000_1740_000,1558151528787637 +11436803605426256250_1720_000_1740_000,1558151538786570 +11436803605426256250_1720_000_1740_000,1558151540786822 +11436803605426256250_1720_000_1740_000,1558151530787441 +11436803605426256250_1720_000_1740_000,1558151527287885 +11436803605426256250_1720_000_1740_000,1558151539786751 +11436803605426256250_1720_000_1740_000,1558151529787489 +11436803605426256250_1720_000_1740_000,1558151539286648 +11436803605426256250_1720_000_1740_000,1558151526287909 +11436803605426256250_1720_000_1740_000,1558151536786870 +11436803605426256250_1720_000_1740_000,1558151536287214 +11436803605426256250_1720_000_1740_000,1558151529287531 +11436803605426256250_1720_000_1740_000,1558151540286973 +11436803605426256250_1720_000_1740_000,1558151538286751 +11436803605426256250_1720_000_1740_000,1558151537286653 +15410814825574326536_2620_000_2640_000,1557860798372836 +15410814825574326536_2620_000_2640_000,1557860800872838 +15410814825574326536_2620_000_2640_000,1557860790372597 +15410814825574326536_2620_000_2640_000,1557860791372832 +15410814825574326536_2620_000_2640_000,1557860799872854 +15410814825574326536_2620_000_2640_000,1557860789372743 +15410814825574326536_2620_000_2640_000,1557860791872904 +15410814825574326536_2620_000_2640_000,1557860798872877 +15410814825574326536_2620_000_2640_000,1557860788372735 +15410814825574326536_2620_000_2640_000,1557860801372803 +15410814825574326536_2620_000_2640_000,1557860802372685 +15410814825574326536_2620_000_2640_000,1557860801872720 +15410814825574326536_2620_000_2640_000,1557860802872671 +15410814825574326536_2620_000_2640_000,1557860792372830 +15410814825574326536_2620_000_2640_000,1557860790872704 +15410814825574326536_2620_000_2640_000,1557860799372902 +15410814825574326536_2620_000_2640_000,1557860792872709 +15410814825574326536_2620_000_2640_000,1557860788872750 +15410814825574326536_2620_000_2640_000,1557860800372906 +15410814825574326536_2620_000_2640_000,1557860789872662 +13585389505831587326_2560_000_2580_000,1557241472137342 +13585389505831587326_2560_000_2580_000,1557241476637531 +13585389505831587326_2560_000_2580_000,1557241484636865 +13585389505831587326_2560_000_2580_000,1557241473137454 +13585389505831587326_2560_000_2580_000,1557241476137536 +13585389505831587326_2560_000_2580_000,1557241472637386 +13585389505831587326_2560_000_2580_000,1557241485637136 +13585389505831587326_2560_000_2580_000,1557241484136968 +13585389505831587326_2560_000_2580_000,1557241485137091 +13585389505831587326_2560_000_2580_000,1557241473637451 +13585389505831587326_2560_000_2580_000,1557241482137115 +13585389505831587326_2560_000_2580_000,1557241475637469 +13585389505831587326_2560_000_2580_000,1557241483636983 +13585389505831587326_2560_000_2580_000,1557241474637506 +13585389505831587326_2560_000_2580_000,1557241483136950 +13585389505831587326_2560_000_2580_000,1557241486137285 +13585389505831587326_2560_000_2580_000,1557241474137501 +13585389505831587326_2560_000_2580_000,1557241486637439 +13585389505831587326_2560_000_2580_000,1557241475137435 +13585389505831587326_2560_000_2580_000,1557241482636985 +15739335479094705947_1420_000_1440_000,1557240344647374 +15739335479094705947_1420_000_1440_000,1557240333147825 +15739335479094705947_1420_000_1440_000,1557240332647832 +15739335479094705947_1420_000_1440_000,1557240336647687 +15739335479094705947_1420_000_1440_000,1557240345147370 +15739335479094705947_1420_000_1440_000,1557240334147846 +15739335479094705947_1420_000_1440_000,1557240335648112 +15739335479094705947_1420_000_1440_000,1557240345647376 +15739335479094705947_1420_000_1440_000,1557240332147799 +15739335479094705947_1420_000_1440_000,1557240344147429 +15739335479094705947_1420_000_1440_000,1557240342147432 +15739335479094705947_1420_000_1440_000,1557240343647467 +15739335479094705947_1420_000_1440_000,1557240346647461 +15739335479094705947_1420_000_1440_000,1557240343147461 +15739335479094705947_1420_000_1440_000,1557240333647840 +15739335479094705947_1420_000_1440_000,1557240335147955 +15739335479094705947_1420_000_1440_000,1557240342647438 +15739335479094705947_1420_000_1440_000,1557240334647920 +15739335479094705947_1420_000_1440_000,1557240346147451 +15739335479094705947_1420_000_1440_000,1557240336147836 +16743182245734335352_1260_000_1280_000,1557888790949495 +16743182245734335352_1260_000_1280_000,1557888787449383 +16743182245734335352_1260_000_1280_000,1557888788948833 +16743182245734335352_1260_000_1280_000,1557888786949263 +16743182245734335352_1260_000_1280_000,1557888776449903 +16743182245734335352_1260_000_1280_000,1557888780449779 +16743182245734335352_1260_000_1280_000,1557888786448960 +16743182245734335352_1260_000_1280_000,1557888777950853 +16743182245734335352_1260_000_1280_000,1557888789448778 +16743182245734335352_1260_000_1280_000,1557888790449312 +16743182245734335352_1260_000_1280_000,1557888779950298 +16743182245734335352_1260_000_1280_000,1557888778451116 +16743182245734335352_1260_000_1280_000,1557888788449105 +16743182245734335352_1260_000_1280_000,1557888779450837 +16743182245734335352_1260_000_1280_000,1557888776950096 +16743182245734335352_1260_000_1280_000,1557888789949015 +16743182245734335352_1260_000_1280_000,1557888787949303 +16743182245734335352_1260_000_1280_000,1557888778951257 +16743182245734335352_1260_000_1280_000,1557888780949350 +16743182245734335352_1260_000_1280_000,1557888777450467 +4037952268810331899_2420_000_2440_000,1567028476924185 +4037952268810331899_2420_000_2440_000,1567028464925058 +4037952268810331899_2420_000_2440_000,1567028466425018 +4037952268810331899_2420_000_2440_000,1567028477924371 +4037952268810331899_2420_000_2440_000,1567028475423773 +4037952268810331899_2420_000_2440_000,1567028475923773 +4037952268810331899_2420_000_2440_000,1567028478424492 +4037952268810331899_2420_000_2440_000,1567028468424910 +4037952268810331899_2420_000_2440_000,1567028466924954 +4037952268810331899_2420_000_2440_000,1567028477424335 +4037952268810331899_2420_000_2440_000,1567028465925047 +4037952268810331899_2420_000_2440_000,1567028476424000 +4037952268810331899_2420_000_2440_000,1567028474424271 +4037952268810331899_2420_000_2440_000,1567028467924880 +4037952268810331899_2420_000_2440_000,1567028478924633 +4037952268810331899_2420_000_2440_000,1567028467424848 +4037952268810331899_2420_000_2440_000,1567028465425099 +4037952268810331899_2420_000_2440_000,1567028464424994 +4037952268810331899_2420_000_2440_000,1567028468924846 +4037952268810331899_2420_000_2440_000,1567028474924011 +17052666463197337241_4560_000_4580_000,1558019835965165 +17052666463197337241_4560_000_4580_000,1558019834964122 +17052666463197337241_4560_000_4580_000,1558019826962706 +17052666463197337241_4560_000_4580_000,1558019837466540 +17052666463197337241_4560_000_4580_000,1558019823962469 +17052666463197337241_4560_000_4580_000,1558019826462862 +17052666463197337241_4560_000_4580_000,1558019834463718 +17052666463197337241_4560_000_4580_000,1558019827962424 +17052666463197337241_4560_000_4580_000,1558019836465729 +17052666463197337241_4560_000_4580_000,1558019827462613 +17052666463197337241_4560_000_4580_000,1558019833963377 +17052666463197337241_4560_000_4580_000,1558019824462615 +17052666463197337241_4560_000_4580_000,1558019836966268 +17052666463197337241_4560_000_4580_000,1558019835464590 +17052666463197337241_4560_000_4580_000,1558019828462295 +17052666463197337241_4560_000_4580_000,1558019825962899 +17052666463197337241_4560_000_4580_000,1558019824962730 +17052666463197337241_4560_000_4580_000,1558019837966298 +17052666463197337241_4560_000_4580_000,1558019825462832 +17052666463197337241_4560_000_4580_000,1558019838465664 +8197312656120253218_3120_000_3140_000,1569346275474782 +8197312656120253218_3120_000_3140_000,1569346279974791 +8197312656120253218_3120_000_3140_000,1569346268974889 +8197312656120253218_3120_000_3140_000,1569346266474964 +8197312656120253218_3120_000_3140_000,1569346267974935 +8197312656120253218_3120_000_3140_000,1569346269974854 +8197312656120253218_3120_000_3140_000,1569346268474908 +8197312656120253218_3120_000_3140_000,1569346266975023 +8197312656120253218_3120_000_3140_000,1569346265475116 +8197312656120253218_3120_000_3140_000,1569346267475024 +8197312656120253218_3120_000_3140_000,1569346276974820 +8197312656120253218_3120_000_3140_000,1569346275974860 +8197312656120253218_3120_000_3140_000,1569346276474878 +8197312656120253218_3120_000_3140_000,1569346279474792 +8197312656120253218_3120_000_3140_000,1569346269474905 +8197312656120253218_3120_000_3140_000,1569346278974783 +8197312656120253218_3120_000_3140_000,1569346265975042 +8197312656120253218_3120_000_3140_000,1569346277974754 +8197312656120253218_3120_000_3140_000,1569346278474771 +8197312656120253218_3120_000_3140_000,1569346277474745 +7844300897851889216_500_000_520_000,1569180269849584 +7844300897851889216_500_000_520_000,1569180283349326 +7844300897851889216_500_000_520_000,1569180270349514 +7844300897851889216_500_000_520_000,1569180281349367 +7844300897851889216_500_000_520_000,1569180273349112 +7844300897851889216_500_000_520_000,1569180280349315 +7844300897851889216_500_000_520_000,1569180280849273 +7844300897851889216_500_000_520_000,1569180283849207 +7844300897851889216_500_000_520_000,1569180272849305 +7844300897851889216_500_000_520_000,1569180270849484 +7844300897851889216_500_000_520_000,1569180282849497 +7844300897851889216_500_000_520_000,1569180271349596 +7844300897851889216_500_000_520_000,1569180271849879 +7844300897851889216_500_000_520_000,1569180284349457 +7844300897851889216_500_000_520_000,1569180282349589 +7844300897851889216_500_000_520_000,1569180281849491 +7844300897851889216_500_000_520_000,1569180272349632 +7844300897851889216_500_000_520_000,1569180274349414 +7844300897851889216_500_000_520_000,1569180279849307 +7844300897851889216_500_000_520_000,1569180273849225 +14918167237855418464_1420_000_1440_000,1557265451487590 +14918167237855418464_1420_000_1440_000,1557265453487513 +14918167237855418464_1420_000_1440_000,1557265440987220 +14918167237855418464_1420_000_1440_000,1557265452987516 +14918167237855418464_1420_000_1440_000,1557265441487272 +14918167237855418464_1420_000_1440_000,1557265449987389 +14918167237855418464_1420_000_1440_000,1557265450487458 +14918167237855418464_1420_000_1440_000,1557265450987504 +14918167237855418464_1420_000_1440_000,1557265440487216 +14918167237855418464_1420_000_1440_000,1557265452487693 +14918167237855418464_1420_000_1440_000,1557265443487465 +14918167237855418464_1420_000_1440_000,1557265451987681 +14918167237855418464_1420_000_1440_000,1557265453987788 +14918167237855418464_1420_000_1440_000,1557265449487404 +14918167237855418464_1420_000_1440_000,1557265442487348 +14918167237855418464_1420_000_1440_000,1557265439487550 +14918167237855418464_1420_000_1440_000,1557265441987298 +14918167237855418464_1420_000_1440_000,1557265439987371 +14918167237855418464_1420_000_1440_000,1557265443987430 +14918167237855418464_1420_000_1440_000,1557265442987426 +1765211916310163252_4400_000_4420_000,1557548091247400 +1765211916310163252_4400_000_4420_000,1557548092247422 +1765211916310163252_4400_000_4420_000,1557548082747340 +1765211916310163252_4400_000_4420_000,1557548080247436 +1765211916310163252_4400_000_4420_000,1557548081747442 +1765211916310163252_4400_000_4420_000,1557548079747433 +1765211916310163252_4400_000_4420_000,1557548093747379 +1765211916310163252_4400_000_4420_000,1557548079247435 +1765211916310163252_4400_000_4420_000,1557548089247264 +1765211916310163252_4400_000_4420_000,1557548092747360 +1765211916310163252_4400_000_4420_000,1557548093247395 +1765211916310163252_4400_000_4420_000,1557548090747296 +1765211916310163252_4400_000_4420_000,1557548083747413 +1765211916310163252_4400_000_4420_000,1557548091747409 +1765211916310163252_4400_000_4420_000,1557548080747512 +1765211916310163252_4400_000_4420_000,1557548090247209 +1765211916310163252_4400_000_4420_000,1557548089747220 +1765211916310163252_4400_000_4420_000,1557548082247344 +1765211916310163252_4400_000_4420_000,1557548081247513 +1765211916310163252_4400_000_4420_000,1557548083247412 +365416647046203224_1080_000_1100_000,1557424297779078 +365416647046203224_1080_000_1100_000,1557424298279187 +365416647046203224_1080_000_1100_000,1557424284779145 +365416647046203224_1080_000_1100_000,1557424299279496 +365416647046203224_1080_000_1100_000,1557424285779375 +365416647046203224_1080_000_1100_000,1557424286279493 +365416647046203224_1080_000_1100_000,1557424288279208 +365416647046203224_1080_000_1100_000,1557424289279220 +365416647046203224_1080_000_1100_000,1557424286779477 +365416647046203224_1080_000_1100_000,1557424294779296 +365416647046203224_1080_000_1100_000,1557424297279126 +365416647046203224_1080_000_1100_000,1557424288779176 +365416647046203224_1080_000_1100_000,1557424287779352 +365416647046203224_1080_000_1100_000,1557424296779274 +365416647046203224_1080_000_1100_000,1557424298779408 +365416647046203224_1080_000_1100_000,1557424295779354 +365416647046203224_1080_000_1100_000,1557424295279343 +365416647046203224_1080_000_1100_000,1557424287279453 +365416647046203224_1080_000_1100_000,1557424285279259 +365416647046203224_1080_000_1100_000,1557424296279315 +3122599254941105215_2980_000_3000_000,1557267013486064 +3122599254941105215_2980_000_3000_000,1557266999471976 +3122599254941105215_2980_000_3000_000,1557267003971991 +3122599254941105215_2980_000_3000_000,1557267002972068 +3122599254941105215_2980_000_3000_000,1557267011978743 +3122599254941105215_2980_000_3000_000,1557267010473667 +3122599254941105215_2980_000_3000_000,1557267001472099 +3122599254941105215_2980_000_3000_000,1557267009973013 +3122599254941105215_2980_000_3000_000,1557267001972106 +3122599254941105215_2980_000_3000_000,1557267009472852 +3122599254941105215_2980_000_3000_000,1557267013987647 +3122599254941105215_2980_000_3000_000,1557267000972170 +3122599254941105215_2980_000_3000_000,1557267011476593 +3122599254941105215_2980_000_3000_000,1557267012983667 +3122599254941105215_2980_000_3000_000,1557266999972086 +3122599254941105215_2980_000_3000_000,1557267012481088 +3122599254941105215_2980_000_3000_000,1557267010974840 +3122599254941105215_2980_000_3000_000,1557267000472146 +3122599254941105215_2980_000_3000_000,1557267002472069 +3122599254941105215_2980_000_3000_000,1557267003472050 +11672844176539348333_4440_000_4460_000,1557548130247264 +11672844176539348333_4440_000_4460_000,1557548119247298 +11672844176539348333_4440_000_4460_000,1557548120747400 +11672844176539348333_4440_000_4460_000,1557548129247403 +11672844176539348333_4440_000_4460_000,1557548121747436 +11672844176539348333_4440_000_4460_000,1557548131747575 +11672844176539348333_4440_000_4460_000,1557548122747361 +11672844176539348333_4440_000_4460_000,1557548132247553 +11672844176539348333_4440_000_4460_000,1557548129747331 +11672844176539348333_4440_000_4460_000,1557548119747262 +11672844176539348333_4440_000_4460_000,1557548121247414 +11672844176539348333_4440_000_4460_000,1557548133747542 +11672844176539348333_4440_000_4460_000,1557548131247534 +11672844176539348333_4440_000_4460_000,1557548122247407 +11672844176539348333_4440_000_4460_000,1557548120247254 +11672844176539348333_4440_000_4460_000,1557548132747504 +11672844176539348333_4440_000_4460_000,1557548123247374 +11672844176539348333_4440_000_4460_000,1557548133247537 +11672844176539348333_4440_000_4460_000,1557548130747376 +11672844176539348333_4440_000_4460_000,1557548123747487 +17212025549630306883_2500_000_2520_000,1558035014396914 +17212025549630306883_2500_000_2520_000,1558035010397071 +17212025549630306883_2500_000_2520_000,1558035000879571 +17212025549630306883_2500_000_2520_000,1558035010897075 +17212025549630306883_2500_000_2520_000,1558035003389800 +17212025549630306883_2500_000_2520_000,1558034999877494 +17212025549630306883_2500_000_2520_000,1558035001883076 +17212025549630306883_2500_000_2520_000,1558035013896904 +17212025549630306883_2500_000_2520_000,1558035002385104 +17212025549630306883_2500_000_2520_000,1558035013397429 +17212025549630306883_2500_000_2520_000,1558035012398066 +17212025549630306883_2500_000_2520_000,1558035009897309 +17212025549630306883_2500_000_2520_000,1558035011397333 +17212025549630306883_2500_000_2520_000,1558035003892217 +17212025549630306883_2500_000_2520_000,1558035002887308 +17212025549630306883_2500_000_2520_000,1558035004394149 +17212025549630306883_2500_000_2520_000,1558035001381105 +17212025549630306883_2500_000_2520_000,1558035012897961 +17212025549630306883_2500_000_2520_000,1558035011897875 +17212025549630306883_2500_000_2520_000,1558035000378455 +5444585006397501511_160_000_180_000,1557843369612501 +5444585006397501511_160_000_180_000,1557843356612649 +5444585006397501511_160_000_180_000,1557843357612587 +5444585006397501511_160_000_180_000,1557843366112688 +5444585006397501511_160_000_180_000,1557843369112577 +5444585006397501511_160_000_180_000,1557843356112502 +5444585006397501511_160_000_180_000,1557843357112699 +5444585006397501511_160_000_180_000,1557843359112424 +5444585006397501511_160_000_180_000,1557843368612608 +5444585006397501511_160_000_180_000,1557843358612418 +5444585006397501511_160_000_180_000,1557843359612545 +5444585006397501511_160_000_180_000,1557843365112636 +5444585006397501511_160_000_180_000,1557843365612657 +5444585006397501511_160_000_180_000,1557843367112626 +5444585006397501511_160_000_180_000,1557843366612681 +5444585006397501511_160_000_180_000,1557843367612623 +5444585006397501511_160_000_180_000,1557843358112458 +5444585006397501511_160_000_180_000,1557843355112397 +5444585006397501511_160_000_180_000,1557843355612457 +5444585006397501511_160_000_180_000,1557843368112622 +17595457728136868510_860_000_880_000,1568570142949954 +17595457728136868510_860_000_880_000,1568570144449980 +17595457728136868510_860_000_880_000,1568570133450011 +17595457728136868510_860_000_880_000,1568570132449985 +17595457728136868510_860_000_880_000,1568570146949999 +17595457728136868510_860_000_880_000,1568570145450102 +17595457728136868510_860_000_880_000,1568570136950003 +17595457728136868510_860_000_880_000,1568570146449992 +17595457728136868510_860_000_880_000,1568570145950029 +17595457728136868510_860_000_880_000,1568570134450024 +17595457728136868510_860_000_880_000,1568570135449980 +17595457728136868510_860_000_880_000,1568570133950026 +17595457728136868510_860_000_880_000,1568570143449845 +17595457728136868510_860_000_880_000,1568570143949863 +17595457728136868510_860_000_880_000,1568570144950031 +17595457728136868510_860_000_880_000,1568570132950020 +17595457728136868510_860_000_880_000,1568570142449990 +17595457728136868510_860_000_880_000,1568570135950008 +17595457728136868510_860_000_880_000,1568570134950042 +17595457728136868510_860_000_880_000,1568570136450044 +10534368980139017457_4480_000_4500_000,1557548163747266 +10534368980139017457_4480_000_4500_000,1557548173324042 +10534368980139017457_4480_000_4500_000,1557548171335072 +10534368980139017457_4480_000_4500_000,1557548171831184 +10534368980139017457_4480_000_4500_000,1557548172327947 +10534368980139017457_4480_000_4500_000,1557548160747474 +10534368980139017457_4480_000_4500_000,1557548159747321 +10534368980139017457_4480_000_4500_000,1557548170342486 +10534368980139017457_4480_000_4500_000,1557548169845348 +10534368980139017457_4480_000_4500_000,1557548170838932 +10534368980139017457_4480_000_4500_000,1557548162247217 +10534368980139017457_4480_000_4500_000,1557548169346776 +10534368980139017457_4480_000_4500_000,1557548173822919 +10534368980139017457_4480_000_4500_000,1557548162747305 +10534368980139017457_4480_000_4500_000,1557548160247434 +10534368980139017457_4480_000_4500_000,1557548163247304 +10534368980139017457_4480_000_4500_000,1557548159247329 +10534368980139017457_4480_000_4500_000,1557548161247379 +10534368980139017457_4480_000_4500_000,1557548161747254 +10534368980139017457_4480_000_4500_000,1557548172825501 +4593468568253300598_1620_000_1640_000,1558034119947167 +4593468568253300598_1620_000_1640_000,1558034131947521 +4593468568253300598_1620_000_1640_000,1558034130447767 +4593468568253300598_1620_000_1640_000,1558034123947147 +4593468568253300598_1620_000_1640_000,1558034123447155 +4593468568253300598_1620_000_1640_000,1558034131447564 +4593468568253300598_1620_000_1640_000,1558034132447509 +4593468568253300598_1620_000_1640_000,1558034133947605 +4593468568253300598_1620_000_1640_000,1558034130947609 +4593468568253300598_1620_000_1640_000,1558034120947198 +4593468568253300598_1620_000_1640_000,1558034129947874 +4593468568253300598_1620_000_1640_000,1558034121947243 +4593468568253300598_1620_000_1640_000,1558034134447535 +4593468568253300598_1620_000_1640_000,1558034122447204 +4593468568253300598_1620_000_1640_000,1558034120447070 +4593468568253300598_1620_000_1640_000,1558034132947552 +4593468568253300598_1620_000_1640_000,1558034121447241 +4593468568253300598_1620_000_1640_000,1558034124447344 +4593468568253300598_1620_000_1640_000,1558034122947127 +4593468568253300598_1620_000_1640_000,1558034133447706 +5810494922060252082_3720_000_3740_000,1557324791637281 +5810494922060252082_3720_000_3740_000,1557324799137989 +5810494922060252082_3720_000_3740_000,1557324792137386 +5810494922060252082_3720_000_3740_000,1557324793137531 +5810494922060252082_3720_000_3740_000,1557324802137484 +5810494922060252082_3720_000_3740_000,1557324802637768 +5810494922060252082_3720_000_3740_000,1557324793637492 +5810494922060252082_3720_000_3740_000,1557324789137692 +5810494922060252082_3720_000_3740_000,1557324803137953 +5810494922060252082_3720_000_3740_000,1557324800137837 +5810494922060252082_3720_000_3740_000,1557324789637648 +5810494922060252082_3720_000_3740_000,1557324800637433 +5810494922060252082_3720_000_3740_000,1557324792637516 +5810494922060252082_3720_000_3740_000,1557324803638106 +5810494922060252082_3720_000_3740_000,1557324791137526 +5810494922060252082_3720_000_3740_000,1557324790637757 +5810494922060252082_3720_000_3740_000,1557324801637223 +5810494922060252082_3720_000_3740_000,1557324801137028 +5810494922060252082_3720_000_3740_000,1557324799638056 +5810494922060252082_3720_000_3740_000,1557324790137645 +2942662230423855469_880_000_900_000,1559348363724177 +2942662230423855469_880_000_900_000,1559348365724257 +2942662230423855469_880_000_900_000,1559348367224483 +2942662230423855469_880_000_900_000,1559348356718454 +2942662230423855469_880_000_900_000,1559348367724406 +2942662230423855469_880_000_900_000,1559348355717289 +2942662230423855469_880_000_900_000,1559348357719518 +2942662230423855469_880_000_900_000,1559348357218926 +2942662230423855469_880_000_900_000,1559348363224053 +2942662230423855469_880_000_900_000,1559348365224224 +2942662230423855469_880_000_900_000,1559348364724292 +2942662230423855469_880_000_900_000,1559348353716247 +2942662230423855469_880_000_900_000,1559348354716626 +2942662230423855469_880_000_900_000,1559348366224290 +2942662230423855469_880_000_900_000,1559348366724409 +2942662230423855469_880_000_900_000,1559348353216332 +2942662230423855469_880_000_900_000,1559348355216840 +2942662230423855469_880_000_900_000,1559348364224254 +2942662230423855469_880_000_900_000,1559348356217995 +2942662230423855469_880_000_900_000,1559348354216438 +5927928428387529213_1640_000_1660_000,1557240564663983 +5927928428387529213_1640_000_1660_000,1557240563163984 +5927928428387529213_1640_000_1660_000,1557240553162670 +5927928428387529213_1640_000_1660_000,1557240566163388 +5927928428387529213_1640_000_1660_000,1557240556162436 +5927928428387529213_1640_000_1660_000,1557240554162851 +5927928428387529213_1640_000_1660_000,1557240552662244 +5927928428387529213_1640_000_1660_000,1557240555162405 +5927928428387529213_1640_000_1660_000,1557240564164016 +5927928428387529213_1640_000_1660_000,1557240552162020 +5927928428387529213_1640_000_1660_000,1557240554662508 +5927928428387529213_1640_000_1660_000,1557240562163098 +5927928428387529213_1640_000_1660_000,1557240566663035 +5927928428387529213_1640_000_1660_000,1557240555662402 +5927928428387529213_1640_000_1660_000,1557240565663746 +5927928428387529213_1640_000_1660_000,1557240562663614 +5927928428387529213_1640_000_1660_000,1557240563664057 +5927928428387529213_1640_000_1660_000,1557240556662471 +5927928428387529213_1640_000_1660_000,1557240553662931 +5927928428387529213_1640_000_1660_000,1557240565163970 +3645211352574995740_3540_000_3560_000,1558018817992298 +3645211352574995740_3540_000_3560_000,1558018804471765 +3645211352574995740_3540_000_3560_000,1558018808472120 +3645211352574995740_3540_000_3560_000,1558018807972072 +3645211352574995740_3540_000_3560_000,1558018815475883 +3645211352574995740_3540_000_3560_000,1558018804971761 +3645211352574995740_3540_000_3560_000,1558018816984976 +3645211352574995740_3540_000_3560_000,1558018815978327 +3645211352574995740_3540_000_3560_000,1558018816481398 +3645211352574995740_3540_000_3560_000,1558018818494946 +3645211352574995740_3540_000_3560_000,1558018817488679 +3645211352574995740_3540_000_3560_000,1558018805471754 +3645211352574995740_3540_000_3560_000,1558018806471940 +3645211352574995740_3540_000_3560_000,1558018807472066 +3645211352574995740_3540_000_3560_000,1558018805971789 +3645211352574995740_3540_000_3560_000,1558018806972056 +3645211352574995740_3540_000_3560_000,1558018814473516 +3645211352574995740_3540_000_3560_000,1558018813973212 +3645211352574995740_3540_000_3560_000,1558018814974307 +3645211352574995740_3540_000_3560_000,1558018803972077 +3510690431623954420_7700_000_7720_000,1567022016599663 +3510690431623954420_7700_000_7720_000,1567022018599669 +3510690431623954420_7700_000_7720_000,1567022028099832 +3510690431623954420_7700_000_7720_000,1567022017099671 +3510690431623954420_7700_000_7720_000,1567022021099696 +3510690431623954420_7700_000_7720_000,1567022019599567 +3510690431623954420_7700_000_7720_000,1567022020599579 +3510690431623954420_7700_000_7720_000,1567022029600020 +3510690431623954420_7700_000_7720_000,1567022017599659 +3510690431623954420_7700_000_7720_000,1567022026599976 +3510690431623954420_7700_000_7720_000,1567022030099989 +3510690431623954420_7700_000_7720_000,1567022028599676 +3510690431623954420_7700_000_7720_000,1567022019099605 +3510690431623954420_7700_000_7720_000,1567022018099661 +3510690431623954420_7700_000_7720_000,1567022030599800 +3510690431623954420_7700_000_7720_000,1567022027599919 +3510690431623954420_7700_000_7720_000,1567022029099795 +3510690431623954420_7700_000_7720_000,1567022020099574 +3510690431623954420_7700_000_7720_000,1567022027099913 +3510690431623954420_7700_000_7720_000,1567022031099758 +39847154216997509_6440_000_6460_000,1568954824924191 +39847154216997509_6440_000_6460_000,1568954813424574 +39847154216997509_6440_000_6460_000,1568954813924259 +39847154216997509_6440_000_6460_000,1568954811424618 +39847154216997509_6440_000_6460_000,1568954822924591 +39847154216997509_6440_000_6460_000,1568954812924890 +39847154216997509_6440_000_6460_000,1568954820924315 +39847154216997509_6440_000_6460_000,1568954810424785 +39847154216997509_6440_000_6460_000,1568954811924639 +39847154216997509_6440_000_6460_000,1568954810924694 +39847154216997509_6440_000_6460_000,1568954814924374 +39847154216997509_6440_000_6460_000,1568954823424463 +39847154216997509_6440_000_6460_000,1568954824424244 +39847154216997509_6440_000_6460_000,1568954814424260 +39847154216997509_6440_000_6460_000,1568954821924250 +39847154216997509_6440_000_6460_000,1568954821424322 +39847154216997509_6440_000_6460_000,1568954820424237 +39847154216997509_6440_000_6460_000,1568954823924349 +39847154216997509_6440_000_6460_000,1568954812424884 +39847154216997509_6440_000_6460_000,1568954822424440 +8623236016759087157_3500_000_3520_000,1557324582561015 +8623236016759087157_3500_000_3520_000,1557324569137429 +8623236016759087157_3500_000_3520_000,1557324582058259 +8623236016759087157_3500_000_3520_000,1557324583062491 +8623236016759087157_3500_000_3520_000,1557324573638623 +8623236016759087157_3500_000_3520_000,1557324572138081 +8623236016759087157_3500_000_3520_000,1557324583562892 +8623236016759087157_3500_000_3520_000,1557324570637898 +8623236016759087157_3500_000_3520_000,1557324581052320 +8623236016759087157_3500_000_3520_000,1557324571638078 +8623236016759087157_3500_000_3520_000,1557324570137931 +8623236016759087157_3500_000_3520_000,1557324580549644 +8623236016759087157_3500_000_3520_000,1557324571138013 +8623236016759087157_3500_000_3520_000,1557324579042078 +8623236016759087157_3500_000_3520_000,1557324581555154 +8623236016759087157_3500_000_3520_000,1557324572638279 +8623236016759087157_3500_000_3520_000,1557324579544532 +8623236016759087157_3500_000_3520_000,1557324580047183 +8623236016759087157_3500_000_3520_000,1557324569637588 +8623236016759087157_3500_000_3520_000,1557324573138464 +8920841445900141920_1700_000_1720_000,1557859882950427 +8920841445900141920_1700_000_1720_000,1557859870947785 +8920841445900141920_1700_000_1720_000,1557859868947629 +8920841445900141920_1700_000_1720_000,1557859882449903 +8920841445900141920_1700_000_1720_000,1557859878447989 +8920841445900141920_1700_000_1720_000,1557859872447754 +8920841445900141920_1700_000_1720_000,1557859879448016 +8920841445900141920_1700_000_1720_000,1557859879948093 +8920841445900141920_1700_000_1720_000,1557859869447788 +8920841445900141920_1700_000_1720_000,1557859881448912 +8920841445900141920_1700_000_1720_000,1557859870447773 +8920841445900141920_1700_000_1720_000,1557859880448231 +8920841445900141920_1700_000_1720_000,1557859878947993 +8920841445900141920_1700_000_1720_000,1557859880948478 +8920841445900141920_1700_000_1720_000,1557859869947772 +8920841445900141920_1700_000_1720_000,1557859881949366 +8920841445900141920_1700_000_1720_000,1557859872947901 +8920841445900141920_1700_000_1720_000,1557859871947716 +8920841445900141920_1700_000_1720_000,1557859871447819 +8920841445900141920_1700_000_1720_000,1557859868447474 +1417898473608326362_2560_000_2580_000,1557546242797474 +1417898473608326362_2560_000_2580_000,1557546239797468 +1417898473608326362_2560_000_2580_000,1557546241797368 +1417898473608326362_2560_000_2580_000,1557546252797649 +1417898473608326362_2560_000_2580_000,1557546252297680 +1417898473608326362_2560_000_2580_000,1557546239297163 +1417898473608326362_2560_000_2580_000,1557546253797788 +1417898473608326362_2560_000_2580_000,1557546249297427 +1417898473608326362_2560_000_2580_000,1557546242297446 +1417898473608326362_2560_000_2580_000,1557546251297740 +1417898473608326362_2560_000_2580_000,1557546240297658 +1417898473608326362_2560_000_2580_000,1557546240797643 +1417898473608326362_2560_000_2580_000,1557546250297550 +1417898473608326362_2560_000_2580_000,1557546249797555 +1417898473608326362_2560_000_2580_000,1557546251797725 +1417898473608326362_2560_000_2580_000,1557546250797666 +1417898473608326362_2560_000_2580_000,1557546253297756 +1417898473608326362_2560_000_2580_000,1557546243797028 +1417898473608326362_2560_000_2580_000,1557546243297291 +1417898473608326362_2560_000_2580_000,1557546241297483 +9584760613582366524_1620_000_1640_000,1557879417399208 +9584760613582366524_1620_000_1640_000,1557879416899412 +9584760613582366524_1620_000_1640_000,1557879428399102 +9584760613582366524_1620_000_1640_000,1557879420399302 +9584760613582366524_1620_000_1640_000,1557879427399045 +9584760613582366524_1620_000_1640_000,1557879420899353 +9584760613582366524_1620_000_1640_000,1557879426899061 +9584760613582366524_1620_000_1640_000,1557879418899485 +9584760613582366524_1620_000_1640_000,1557879418399553 +9584760613582366524_1620_000_1640_000,1557879429898992 +9584760613582366524_1620_000_1640_000,1557879428899097 +9584760613582366524_1620_000_1640_000,1557879430898987 +9584760613582366524_1620_000_1640_000,1557879429399097 +9584760613582366524_1620_000_1640_000,1557879421399451 +9584760613582366524_1620_000_1640_000,1557879431398990 +9584760613582366524_1620_000_1640_000,1557879419899335 +9584760613582366524_1620_000_1640_000,1557879419399372 +9584760613582366524_1620_000_1640_000,1557879430398927 +9584760613582366524_1620_000_1640_000,1557879417899434 +9584760613582366524_1620_000_1640_000,1557879427899058 +6503078254504013503_3440_000_3460_000,1557855947547440 +6503078254504013503_3440_000_3460_000,1557855934472627 +6503078254504013503_3440_000_3460_000,1557855932972711 +6503078254504013503_3440_000_3460_000,1557855934972072 +6503078254504013503_3440_000_3460_000,1557855946547513 +6503078254504013503_3440_000_3460_000,1557855933972741 +6503078254504013503_3440_000_3460_000,1557855945047402 +6503078254504013503_3440_000_3460_000,1557855936962356 +6503078254504013503_3440_000_3460_000,1557855945547411 +6503078254504013503_3440_000_3460_000,1557855947047525 +6503078254504013503_3440_000_3460_000,1557855944547167 +6503078254504013503_3440_000_3460_000,1557855944046932 +6503078254504013503_3440_000_3460_000,1557855937459144 +6503078254504013503_3440_000_3460_000,1557855933472775 +6503078254504013503_3440_000_3460_000,1557855946047387 +6503078254504013503_3440_000_3460_000,1557855935470483 +6503078254504013503_3440_000_3460_000,1557855943047114 +6503078254504013503_3440_000_3460_000,1557855935968223 +6503078254504013503_3440_000_3460_000,1557855943547034 +6503078254504013503_3440_000_3460_000,1557855936465449 +11867874114645674271_600_000_620_000,1556074736854433 +11867874114645674271_600_000_620_000,1556074726349701 +11867874114645674271_600_000_620_000,1556074735851657 +11867874114645674271_600_000_620_000,1556074725849692 +11867874114645674271_600_000_620_000,1556074737859199 +11867874114645674271_600_000_620_000,1556074738362254 +11867874114645674271_600_000_620_000,1556074727849804 +11867874114645674271_600_000_620_000,1556074733850341 +11867874114645674271_600_000_620_000,1556074724350381 +11867874114645674271_600_000_620_000,1556074735350931 +11867874114645674271_600_000_620_000,1556074728349730 +11867874114645674271_600_000_620_000,1556074724849999 +11867874114645674271_600_000_620_000,1556074725349782 +11867874114645674271_600_000_620_000,1556074726849817 +11867874114645674271_600_000_620_000,1556074727349951 +11867874114645674271_600_000_620_000,1556074723850636 +11867874114645674271_600_000_620_000,1556074736352936 +11867874114645674271_600_000_620_000,1556074737356543 +11867874114645674271_600_000_620_000,1556074734850605 +2374138435300423201_2600_000_2620_000,1557546281297269 +2374138435300423201_2600_000_2620_000,1557546290797384 +2374138435300423201_2600_000_2620_000,1557546279797210 +2374138435300423201_2600_000_2620_000,1557546282797429 +2374138435300423201_2600_000_2620_000,1557546279297244 +2374138435300423201_2600_000_2620_000,1557546280797280 +2374138435300423201_2600_000_2620_000,1557546280297328 +2374138435300423201_2600_000_2620_000,1557546289297324 +2374138435300423201_2600_000_2620_000,1557546289797335 +2374138435300423201_2600_000_2620_000,1557546283297421 +2374138435300423201_2600_000_2620_000,1557546293797422 +2374138435300423201_2600_000_2620_000,1557546283797387 +2374138435300423201_2600_000_2620_000,1557546291297442 +2374138435300423201_2600_000_2620_000,1557546292797289 +2374138435300423201_2600_000_2620_000,1557546293297352 +2374138435300423201_2600_000_2620_000,1557546282297473 +2374138435300423201_2600_000_2620_000,1557546290297367 +2374138435300423201_2600_000_2620_000,1557546281797389 +2374138435300423201_2600_000_2620_000,1557546292297340 +2374138435300423201_2600_000_2620_000,1557546291797459 +16050146835908439029_4500_000_4520_000,1557862669362069 +16050146835908439029_4500_000_4520_000,1557862668362475 +16050146835908439029_4500_000_4520_000,1557862680862489 +16050146835908439029_4500_000_4520_000,1557862682362527 +16050146835908439029_4500_000_4520_000,1557862679362451 +16050146835908439029_4500_000_4520_000,1557862669862200 +16050146835908439029_4500_000_4520_000,1557862680362483 +16050146835908439029_4500_000_4520_000,1557862670362417 +16050146835908439029_4500_000_4520_000,1557862668862219 +16050146835908439029_4500_000_4520_000,1557862682862598 +16050146835908439029_4500_000_4520_000,1557862681362512 +16050146835908439029_4500_000_4520_000,1557862672362384 +16050146835908439029_4500_000_4520_000,1557862672862388 +16050146835908439029_4500_000_4520_000,1557862670862532 +16050146835908439029_4500_000_4520_000,1557862671862452 +16050146835908439029_4500_000_4520_000,1557862681862522 +16050146835908439029_4500_000_4520_000,1557862678862529 +16050146835908439029_4500_000_4520_000,1557862671362531 +16050146835908439029_4500_000_4520_000,1557862679862445 +16050146835908439029_4500_000_4520_000,1557862678362527 +3400465735719851775_1400_000_1420_000,1572136113149873 +3400465735719851775_1400_000_1420_000,1572136101149909 +3400465735719851775_1400_000_1420_000,1572136102649696 +3400465735719851775_1400_000_1420_000,1572136113649695 +3400465735719851775_1400_000_1420_000,1572136110649929 +3400465735719851775_1400_000_1420_000,1572136111649912 +3400465735719851775_1400_000_1420_000,1572136112649991 +3400465735719851775_1400_000_1420_000,1572136114149800 +3400465735719851775_1400_000_1420_000,1572136100649954 +3400465735719851775_1400_000_1420_000,1572136114649927 +3400465735719851775_1400_000_1420_000,1572136103149884 +3400465735719851775_1400_000_1420_000,1572136112149956 +3400465735719851775_1400_000_1420_000,1572136105149812 +3400465735719851775_1400_000_1420_000,1572136103650115 +3400465735719851775_1400_000_1420_000,1572136115150038 +3400465735719851775_1400_000_1420_000,1572136102149777 +3400465735719851775_1400_000_1420_000,1572136111149884 +3400465735719851775_1400_000_1420_000,1572136104150206 +3400465735719851775_1400_000_1420_000,1572136101649856 +3400465735719851775_1400_000_1420_000,1572136104650062 +13347759874869607317_1540_000_1560_000,1557240455162639 +13347759874869607317_1540_000_1560_000,1557240466662585 +13347759874869607317_1540_000_1560_000,1557240462162692 +13347759874869607317_1540_000_1560_000,1557240454162643 +13347759874869607317_1540_000_1560_000,1557240453162676 +13347759874869607317_1540_000_1560_000,1557240464162672 +13347759874869607317_1540_000_1560_000,1557240462662650 +13347759874869607317_1540_000_1560_000,1557240463162605 +13347759874869607317_1540_000_1560_000,1557240466162631 +13347759874869607317_1540_000_1560_000,1557240452662663 +13347759874869607317_1540_000_1560_000,1557240465662701 +13347759874869607317_1540_000_1560_000,1557240464662665 +13347759874869607317_1540_000_1560_000,1557240452162656 +13347759874869607317_1540_000_1560_000,1557240455662596 +13347759874869607317_1540_000_1560_000,1557240456662719 +13347759874869607317_1540_000_1560_000,1557240456162634 +13347759874869607317_1540_000_1560_000,1557240463662676 +13347759874869607317_1540_000_1560_000,1557240465162708 +13347759874869607317_1540_000_1560_000,1557240454662705 +13347759874869607317_1540_000_1560_000,1557240453662610 +792520390268391604_780_000_800_000,1557276779322398 +792520390268391604_780_000_800_000,1557276789819241 +792520390268391604_780_000_800_000,1557276781822470 +792520390268391604_780_000_800_000,1557276790814750 +792520390268391604_780_000_800_000,1557276787822228 +792520390268391604_780_000_800_000,1557276777822335 +792520390268391604_780_000_800_000,1557276779822444 +792520390268391604_780_000_800_000,1557276791809911 +792520390268391604_780_000_800_000,1557276787322288 +792520390268391604_780_000_800_000,1557276781322325 +792520390268391604_780_000_800_000,1557276778822345 +792520390268391604_780_000_800_000,1557276788821750 +792520390268391604_780_000_800_000,1557276791312453 +792520390268391604_780_000_800_000,1557276780822385 +792520390268391604_780_000_800_000,1557276789320894 +792520390268391604_780_000_800_000,1557276788322108 +792520390268391604_780_000_800_000,1557276778322306 +792520390268391604_780_000_800_000,1557276790317051 +792520390268391604_780_000_800_000,1557276780322459 +792520390268391604_780_000_800_000,1557276777322444 +12555145882162126399_1180_000_1200_000,1558016457446728 +12555145882162126399_1180_000_1200_000,1558016457946731 +12555145882162126399_1180_000_1200_000,1558016443947236 +12555145882162126399_1180_000_1200_000,1558016455946753 +12555145882162126399_1180_000_1200_000,1558016456946665 +12555145882162126399_1180_000_1200_000,1558016445447090 +12555145882162126399_1180_000_1200_000,1558016446446923 +12555145882162126399_1180_000_1200_000,1558016453946646 +12555145882162126399_1180_000_1200_000,1558016446946859 +12555145882162126399_1180_000_1200_000,1558016445947010 +12555145882162126399_1180_000_1200_000,1558016444947161 +12555145882162126399_1180_000_1200_000,1558016455446712 +12555145882162126399_1180_000_1200_000,1558016448446785 +12555145882162126399_1180_000_1200_000,1558016447946858 +12555145882162126399_1180_000_1200_000,1558016458446676 +12555145882162126399_1180_000_1200_000,1558016444447200 +12555145882162126399_1180_000_1200_000,1558016454446636 +12555145882162126399_1180_000_1200_000,1558016454946704 +12555145882162126399_1180_000_1200_000,1558016447446885 +12555145882162126399_1180_000_1200_000,1558016456446713 +2363225200168330815_760_000_780_000,1557363529737707 +2363225200168330815_760_000_780_000,1557363527237757 +2363225200168330815_760_000_780_000,1557363531737831 +2363225200168330815_760_000_780_000,1557363517737551 +2363225200168330815_760_000_780_000,1557363521737684 +2363225200168330815_760_000_780_000,1557363520237879 +2363225200168330815_760_000_780_000,1557363520737853 +2363225200168330815_760_000_780_000,1557363530737748 +2363225200168330815_760_000_780_000,1557363530237741 +2363225200168330815_760_000_780_000,1557363527737802 +2363225200168330815_760_000_780_000,1557363519238031 +2363225200168330815_760_000_780_000,1557363518738025 +2363225200168330815_760_000_780_000,1557363519737941 +2363225200168330815_760_000_780_000,1557363528237770 +2363225200168330815_760_000_780_000,1557363517237246 +2363225200168330815_760_000_780_000,1557363518237827 +2363225200168330815_760_000_780_000,1557363528737726 +2363225200168330815_760_000_780_000,1557363529237740 +2363225200168330815_760_000_780_000,1557363521237784 +2363225200168330815_760_000_780_000,1557363531237786 +3328513486129168664_2080_000_2100_000,1567831757349830 +3328513486129168664_2080_000_2100_000,1567831748349777 +3328513486129168664_2080_000_2100_000,1567831749349801 +3328513486129168664_2080_000_2100_000,1567831759349962 +3328513486129168664_2080_000_2100_000,1567831748849802 +3328513486129168664_2080_000_2100_000,1567831755849834 +3328513486129168664_2080_000_2100_000,1567831745849801 +3328513486129168664_2080_000_2100_000,1567831756349773 +3328513486129168664_2080_000_2100_000,1567831746849942 +3328513486129168664_2080_000_2100_000,1567831750350037 +3328513486129168664_2080_000_2100_000,1567831749849925 +3328513486129168664_2080_000_2100_000,1567831759849964 +3328513486129168664_2080_000_2100_000,1567831747849819 +3328513486129168664_2080_000_2100_000,1567831747349894 +3328513486129168664_2080_000_2100_000,1567831758849989 +3328513486129168664_2080_000_2100_000,1567831758350003 +3328513486129168664_2080_000_2100_000,1567831746349855 +3328513486129168664_2080_000_2100_000,1567831757849928 +3328513486129168664_2080_000_2100_000,1567831756849772 +3328513486129168664_2080_000_2100_000,1567831760349924 +4632556232973423919_2940_000_2960_000,1557266971471958 +4632556232973423919_2940_000_2960_000,1557266969972031 +4632556232973423919_2940_000_2960_000,1557266970971991 +4632556232973423919_2940_000_2960_000,1557266961970295 +4632556232973423919_2940_000_2960_000,1557266959471575 +4632556232973423919_2940_000_2960_000,1557266973972043 +4632556232973423919_2940_000_2960_000,1557266970471994 +4632556232973423919_2940_000_2960_000,1557266959971091 +4632556232973423919_2940_000_2960_000,1557266969472028 +4632556232973423919_2940_000_2960_000,1557266973471990 +4632556232973423919_2940_000_2960_000,1557266962470698 +4632556232973423919_2940_000_2960_000,1557266972972019 +4632556232973423919_2940_000_2960_000,1557266961470258 +4632556232973423919_2940_000_2960_000,1557266972472025 +4632556232973423919_2940_000_2960_000,1557266963471197 +4632556232973423919_2940_000_2960_000,1557266963971575 +4632556232973423919_2940_000_2960_000,1557266962971008 +4632556232973423919_2940_000_2960_000,1557266960970249 +4632556232973423919_2940_000_2960_000,1557266960470534 +4632556232973423919_2940_000_2960_000,1557266971972014 +7855150647548977812_3900_000_3920_000,1557963222297587 +7855150647548977812_3900_000_3920_000,1557963229797360 +7855150647548977812_3900_000_3920_000,1557963219297462 +7855150647548977812_3900_000_3920_000,1557963222797358 +7855150647548977812_3900_000_3920_000,1557963219797630 +7855150647548977812_3900_000_3920_000,1557963221297729 +7855150647548977812_3900_000_3920_000,1557963228797686 +7855150647548977812_3900_000_3920_000,1557963232797406 +7855150647548977812_3900_000_3920_000,1557963230297234 +7855150647548977812_3900_000_3920_000,1557963230797164 +7855150647548977812_3900_000_3920_000,1557963232297399 +7855150647548977812_3900_000_3920_000,1557963221797707 +7855150647548977812_3900_000_3920_000,1557963231797409 +7855150647548977812_3900_000_3920_000,1557963220297852 +7855150647548977812_3900_000_3920_000,1557963233297506 +7855150647548977812_3900_000_3920_000,1557963218797413 +7855150647548977812_3900_000_3920_000,1557963231297227 +7855150647548977812_3900_000_3920_000,1557963229297664 +7855150647548977812_3900_000_3920_000,1557963220797815 +7855150647548977812_3900_000_3920_000,1557963223297048 +6228701001600487900_720_000_740_000,1557196124797396 +6228701001600487900_720_000_740_000,1557196115797133 +6228701001600487900_720_000_740_000,1557196127297339 +6228701001600487900_720_000_740_000,1557196118797295 +6228701001600487900_720_000_740_000,1557196114797019 +6228701001600487900_720_000_740_000,1557196125297368 +6228701001600487900_720_000_740_000,1557196117797593 +6228701001600487900_720_000_740_000,1557196128297444 +6228701001600487900_720_000_740_000,1557196126797262 +6228701001600487900_720_000_740_000,1557196116297146 +6228701001600487900_720_000_740_000,1557196114297509 +6228701001600487900_720_000_740_000,1557196125797316 +6228701001600487900_720_000_740_000,1557196124297381 +6228701001600487900_720_000_740_000,1557196128797516 +6228701001600487900_720_000_740_000,1557196126297244 +6228701001600487900_720_000_740_000,1557196118297420 +6228701001600487900_720_000_740_000,1557196117297567 +6228701001600487900_720_000_740_000,1557196116797296 +6228701001600487900_720_000_740_000,1557196127797382 +6228701001600487900_720_000_740_000,1557196115297027 +5683383258122801095_1040_000_1060_000,1557363799237684 +5683383258122801095_1040_000_1060_000,1557363798737684 +5683383258122801095_1040_000_1060_000,1557363809737739 +5683383258122801095_1040_000_1060_000,1557363801237681 +5683383258122801095_1040_000_1060_000,1557363808237726 +5683383258122801095_1040_000_1060_000,1557363810237674 +5683383258122801095_1040_000_1060_000,1557363797237694 +5683383258122801095_1040_000_1060_000,1557363808737723 +5683383258122801095_1040_000_1060_000,1557363801737692 +5683383258122801095_1040_000_1060_000,1557363807737730 +5683383258122801095_1040_000_1060_000,1557363797737739 +5683383258122801095_1040_000_1060_000,1557363800737713 +5683383258122801095_1040_000_1060_000,1557363799737724 +5683383258122801095_1040_000_1060_000,1557363811737549 +5683383258122801095_1040_000_1060_000,1557363798237739 +5683383258122801095_1040_000_1060_000,1557363810737600 +5683383258122801095_1040_000_1060_000,1557363807237704 +5683383258122801095_1040_000_1060_000,1557363811237550 +5683383258122801095_1040_000_1060_000,1557363800237778 +5683383258122801095_1040_000_1060_000,1557363809237695 +14631629219048194483_2720_000_2740_000,1558017994972187 +14631629219048194483_2720_000_2740_000,1558017997472318 +14631629219048194483_2720_000_2740_000,1558017985972625 +14631629219048194483_2720_000_2740_000,1558017985472656 +14631629219048194483_2720_000_2740_000,1558017986972355 +14631629219048194483_2720_000_2740_000,1558017998472347 +14631629219048194483_2720_000_2740_000,1558017996472360 +14631629219048194483_2720_000_2740_000,1558017983971974 +14631629219048194483_2720_000_2740_000,1558017987972319 +14631629219048194483_2720_000_2740_000,1558017984472180 +14631629219048194483_2720_000_2740_000,1558017984972436 +14631629219048194483_2720_000_2740_000,1558017995972332 +14631629219048194483_2720_000_2740_000,1558017997972298 +14631629219048194483_2720_000_2740_000,1558017994472307 +14631629219048194483_2720_000_2740_000,1558017995472189 +14631629219048194483_2720_000_2740_000,1558017988472347 +14631629219048194483_2720_000_2740_000,1558017986472464 +14631629219048194483_2720_000_2740_000,1558017987472321 +14631629219048194483_2720_000_2740_000,1558017996972405 +14631629219048194483_2720_000_2740_000,1558017993972404 +2906594041697319079_3040_000_3060_000,1557267072486781 +2906594041697319079_3040_000_3060_000,1557267060487498 +2906594041697319079_3040_000_3060_000,1557267073986709 +2906594041697319079_3040_000_3060_000,1557267070986791 +2906594041697319079_3040_000_3060_000,1557267059987458 +2906594041697319079_3040_000_3060_000,1557267071486876 +2906594041697319079_3040_000_3060_000,1557267062487451 +2906594041697319079_3040_000_3060_000,1557267063987482 +2906594041697319079_3040_000_3060_000,1557267063487438 +2906594041697319079_3040_000_3060_000,1557267071986868 +2906594041697319079_3040_000_3060_000,1557267072986667 +2906594041697319079_3040_000_3060_000,1557267069487459 +2906594041697319079_3040_000_3060_000,1557267073486626 +2906594041697319079_3040_000_3060_000,1557267062987469 +2906594041697319079_3040_000_3060_000,1557267061487517 +2906594041697319079_3040_000_3060_000,1557267061987452 +2906594041697319079_3040_000_3060_000,1557267060987578 +2906594041697319079_3040_000_3060_000,1557267070487093 +2906594041697319079_3040_000_3060_000,1557267069987397 +2906594041697319079_3040_000_3060_000,1557267059487462 +2383902674438058857_4420_000_4440_000,1567796626524800 +2383902674438058857_4420_000_4440_000,1567796634024717 +2383902674438058857_4420_000_4440_000,1567796623525141 +2383902674438058857_4420_000_4440_000,1567796634524720 +2383902674438058857_4420_000_4440_000,1567796637024790 +2383902674438058857_4420_000_4440_000,1567796633524726 +2383902674438058857_4420_000_4440_000,1567796623025071 +2383902674438058857_4420_000_4440_000,1567796624525076 +2383902674438058857_4420_000_4440_000,1567796627024804 +2383902674438058857_4420_000_4440_000,1567796627524846 +2383902674438058857_4420_000_4440_000,1567796635024610 +2383902674438058857_4420_000_4440_000,1567796624025081 +2383902674438058857_4420_000_4440_000,1567796625524889 +2383902674438058857_4420_000_4440_000,1567796635524621 +2383902674438058857_4420_000_4440_000,1567796626024858 +2383902674438058857_4420_000_4440_000,1567796636024637 +2383902674438058857_4420_000_4440_000,1567796625024984 +2383902674438058857_4420_000_4440_000,1567796633024733 +2383902674438058857_4420_000_4440_000,1567796637524761 +2383902674438058857_4420_000_4440_000,1567796636524720 +6862795755554967162_2280_000_2300_000,1558152098797641 +6862795755554967162_2280_000_2300_000,1558152096797728 +6862795755554967162_2280_000_2300_000,1558152098297483 +6862795755554967162_2280_000_2300_000,1558152086297472 +6862795755554967162_2280_000_2300_000,1558152088297543 +6862795755554967162_2280_000_2300_000,1558152090297619 +6862795755554967162_2280_000_2300_000,1558152088797546 +6862795755554967162_2280_000_2300_000,1558152096297876 +6862795755554967162_2280_000_2300_000,1558152087797448 +6862795755554967162_2280_000_2300_000,1558152100297819 +6862795755554967162_2280_000_2300_000,1558152089297513 +6862795755554967162_2280_000_2300_000,1558152086797503 +6862795755554967162_2280_000_2300_000,1558152097297600 +6862795755554967162_2280_000_2300_000,1558152099297843 +6862795755554967162_2280_000_2300_000,1558152089797536 +6862795755554967162_2280_000_2300_000,1558152090797668 +6862795755554967162_2280_000_2300_000,1558152099797835 +6862795755554967162_2280_000_2300_000,1558152100797780 +6862795755554967162_2280_000_2300_000,1558152097797483 +6862795755554967162_2280_000_2300_000,1558152087297497 +8085856200343017603_4120_000_4140_000,1557963441810800 +8085856200343017603_4120_000_4140_000,1557963452811392 +8085856200343017603_4120_000_4140_000,1557963442310429 +8085856200343017603_4120_000_4140_000,1557963448811394 +8085856200343017603_4120_000_4140_000,1557963440312587 +8085856200343017603_4120_000_4140_000,1557963452311343 +8085856200343017603_4120_000_4140_000,1557963438812840 +8085856200343017603_4120_000_4140_000,1557963449311428 +8085856200343017603_4120_000_4140_000,1557963450311446 +8085856200343017603_4120_000_4140_000,1557963450811460 +8085856200343017603_4120_000_4140_000,1557963451311480 +8085856200343017603_4120_000_4140_000,1557963441311391 +8085856200343017603_4120_000_4140_000,1557963439312755 +8085856200343017603_4120_000_4140_000,1557963442810720 +8085856200343017603_4120_000_4140_000,1557963453311401 +8085856200343017603_4120_000_4140_000,1557963449811379 +8085856200343017603_4120_000_4140_000,1557963439812771 +8085856200343017603_4120_000_4140_000,1557963443310973 +8085856200343017603_4120_000_4140_000,1557963451811373 +8085856200343017603_4120_000_4140_000,1557963440812062 +15370024704033662533_1240_000_1260_000,1558016507522714 +15370024704033662533_1240_000_1260_000,1558016504022513 +15370024704033662533_1240_000_1260_000,1558016508023021 +15370024704033662533_1240_000_1260_000,1558016516522659 +15370024704033662533_1240_000_1260_000,1558016518522508 +15370024704033662533_1240_000_1260_000,1558016506522488 +15370024704033662533_1240_000_1260_000,1558016516022182 +15370024704033662533_1240_000_1260_000,1558016518022743 +15370024704033662533_1240_000_1260_000,1558016517022970 +15370024704033662533_1240_000_1260_000,1558016514522028 +15370024704033662533_1240_000_1260_000,1558016507022487 +15370024704033662533_1240_000_1260_000,1558016505022580 +15370024704033662533_1240_000_1260_000,1558016517522896 +15370024704033662533_1240_000_1260_000,1558016506022489 +15370024704033662533_1240_000_1260_000,1558016504522546 +15370024704033662533_1240_000_1260_000,1558016514022344 +15370024704033662533_1240_000_1260_000,1558016505522521 +15370024704033662533_1240_000_1260_000,1558016515022010 +15370024704033662533_1240_000_1260_000,1558016515522158 +15370024704033662533_1240_000_1260_000,1558016508523056 +13887882285811432765_740_000_760_000,1557427670612416 +13887882285811432765_740_000_760_000,1557427657104220 +13887882285811432765_740_000_760_000,1557427656098234 +13887882285811432765_740_000_760_000,1557427670112488 +13887882285811432765_740_000_760_000,1557427657607241 +13887882285811432765_740_000_760_000,1557427659611359 +13887882285811432765_740_000_760_000,1557427668112690 +13887882285811432765_740_000_760_000,1557427669112938 +13887882285811432765_740_000_760_000,1557427668612977 +13887882285811432765_740_000_760_000,1557427667112287 +13887882285811432765_740_000_760_000,1557427667612500 +13887882285811432765_740_000_760_000,1557427660611691 +13887882285811432765_740_000_760_000,1557427658610018 +13887882285811432765_740_000_760_000,1557427660111611 +13887882285811432765_740_000_760_000,1557427658109105 +13887882285811432765_740_000_760_000,1557427656601007 +13887882285811432765_740_000_760_000,1557427659110867 +13887882285811432765_740_000_760_000,1557427666112373 +13887882285811432765_740_000_760_000,1557427666612282 +13887882285811432765_740_000_760_000,1557427669612708 +7886090431228432618_1060_000_1080_000,1557427978090024 +7886090431228432618_1060_000_1080_000,1557427986587357 +7886090431228432618_1060_000_1080_000,1557427979089033 +7886090431228432618_1060_000_1080_000,1557427980587825 +7886090431228432618_1060_000_1080_000,1557427988586899 +7886090431228432618_1060_000_1080_000,1557427989086904 +7886090431228432618_1060_000_1080_000,1557427977091041 +7886090431228432618_1060_000_1080_000,1557427976591045 +7886090431228432618_1060_000_1080_000,1557427987587267 +7886090431228432618_1060_000_1080_000,1557427980088231 +7886090431228432618_1060_000_1080_000,1557427987087350 +7886090431228432618_1060_000_1080_000,1557427990587971 +7886090431228432618_1060_000_1080_000,1557427978589494 +7886090431228432618_1060_000_1080_000,1557427979588581 +7886090431228432618_1060_000_1080_000,1557427977590623 +7886090431228432618_1060_000_1080_000,1557427990087424 +7886090431228432618_1060_000_1080_000,1557427988087048 +7886090431228432618_1060_000_1080_000,1557427989587118 +7886090431228432618_1060_000_1080_000,1557427986087349 +7886090431228432618_1060_000_1080_000,1557427976090905 +11096867396355523348_1460_000_1480_000,1557240385647315 +11096867396355523348_1460_000_1480_000,1557240376147639 +11096867396355523348_1460_000_1480_000,1557240383646953 +11096867396355523348_1460_000_1480_000,1557240373147399 +11096867396355523348_1460_000_1480_000,1557240385147284 +11096867396355523348_1460_000_1480_000,1557240383147053 +11096867396355523348_1460_000_1480_000,1557240375647537 +11096867396355523348_1460_000_1480_000,1557240376647555 +11096867396355523348_1460_000_1480_000,1557240382647278 +11096867396355523348_1460_000_1480_000,1557240374147381 +11096867396355523348_1460_000_1480_000,1557240373647402 +11096867396355523348_1460_000_1480_000,1557240382147351 +11096867396355523348_1460_000_1480_000,1557240375147338 +11096867396355523348_1460_000_1480_000,1557240386147261 +11096867396355523348_1460_000_1480_000,1557240384647073 +11096867396355523348_1460_000_1480_000,1557240372647451 +11096867396355523348_1460_000_1480_000,1557240384146914 +11096867396355523348_1460_000_1480_000,1557240386647265 +11096867396355523348_1460_000_1480_000,1557240374647330 +11096867396355523348_1460_000_1480_000,1557240372147515 +5993415832220804439_1020_000_1040_000,1557427938162330 +5993415832220804439_1020_000_1040_000,1557427940162319 +5993415832220804439_1020_000_1040_000,1557427937662244 +5993415832220804439_1020_000_1040_000,1557427946662314 +5993415832220804439_1020_000_1040_000,1557427946162333 +5993415832220804439_1020_000_1040_000,1557427938662319 +5993415832220804439_1020_000_1040_000,1557427948162669 +5993415832220804439_1020_000_1040_000,1557427947162431 +5993415832220804439_1020_000_1040_000,1557427947662672 +5993415832220804439_1020_000_1040_000,1557427949662420 +5993415832220804439_1020_000_1040_000,1557427950162677 +5993415832220804439_1020_000_1040_000,1557427948662689 +5993415832220804439_1020_000_1040_000,1557427950662930 +5993415832220804439_1020_000_1040_000,1557427940662334 +5993415832220804439_1020_000_1040_000,1557427939662313 +5993415832220804439_1020_000_1040_000,1557427936661931 +5993415832220804439_1020_000_1040_000,1557427936161893 +5993415832220804439_1020_000_1040_000,1557427939162305 +5993415832220804439_1020_000_1040_000,1557427937162046 +5993415832220804439_1020_000_1040_000,1557427949162510 +684234579698396203_2540_000_2560_000,1557546221272675 +684234579698396203_2540_000_2560_000,1557546223272676 +684234579698396203_2540_000_2560_000,1557546229272374 +684234579698396203_2540_000_2560_000,1557546232272632 +684234579698396203_2540_000_2560_000,1557546222772668 +684234579698396203_2540_000_2560_000,1557546233775554 +684234579698396203_2540_000_2560_000,1557546230272562 +684234579698396203_2540_000_2560_000,1557546219772629 +684234579698396203_2540_000_2560_000,1557546231272784 +684234579698396203_2540_000_2560_000,1557546221772604 +684234579698396203_2540_000_2560_000,1557546229772445 +684234579698396203_2540_000_2560_000,1557546233273525 +684234579698396203_2540_000_2560_000,1557546220772768 +684234579698396203_2540_000_2560_000,1557546230772716 +684234579698396203_2540_000_2560_000,1557546223772715 +684234579698396203_2540_000_2560_000,1557546231772736 +684234579698396203_2540_000_2560_000,1557546232772749 +684234579698396203_2540_000_2560_000,1557546222272631 +684234579698396203_2540_000_2560_000,1557546220272744 +684234579698396203_2540_000_2560_000,1557546219272563 +16367045247642649300_3060_000_3080_000,1557267091988308 +16367045247642649300_3060_000_3080_000,1557267090487889 +16367045247642649300_3060_000_3080_000,1557267089487659 +16367045247642649300_3060_000_3080_000,1557267093487520 +16367045247642649300_3060_000_3080_000,1557267093987555 +16367045247642649300_3060_000_3080_000,1557267082986958 +16367045247642649300_3060_000_3080_000,1557267080987657 +16367045247642649300_3060_000_3080_000,1557267083987136 +16367045247642649300_3060_000_3080_000,1557267082487269 +16367045247642649300_3060_000_3080_000,1557267080487535 +16367045247642649300_3060_000_3080_000,1557267081987538 +16367045247642649300_3060_000_3080_000,1557267083486940 +16367045247642649300_3060_000_3080_000,1557267079987387 +16367045247642649300_3060_000_3080_000,1557267079487248 +16367045247642649300_3060_000_3080_000,1557267089987808 +16367045247642649300_3060_000_3080_000,1557267092987360 +16367045247642649300_3060_000_3080_000,1557267092487706 +16367045247642649300_3060_000_3080_000,1557267090987837 +16367045247642649300_3060_000_3080_000,1557267081487585 +16367045247642649300_3060_000_3080_000,1557267091488223 +10940141908690367388_4420_000_4440_000,1557325501087726 +10940141908690367388_4420_000_4440_000,1557325493087410 +10940141908690367388_4420_000_4440_000,1557325490587432 +10940141908690367388_4420_000_4440_000,1557325503087783 +10940141908690367388_4420_000_4440_000,1557325501587681 +10940141908690367388_4420_000_4440_000,1557325492087435 +10940141908690367388_4420_000_4440_000,1557325503587721 +10940141908690367388_4420_000_4440_000,1557325491587341 +10940141908690367388_4420_000_4440_000,1557325489587388 +10940141908690367388_4420_000_4440_000,1557325489087347 +10940141908690367388_4420_000_4440_000,1557325490087423 +10940141908690367388_4420_000_4440_000,1557325499587637 +10940141908690367388_4420_000_4440_000,1557325491087364 +10940141908690367388_4420_000_4440_000,1557325493587440 +10940141908690367388_4420_000_4440_000,1557325502087695 +10940141908690367388_4420_000_4440_000,1557325500087574 +10940141908690367388_4420_000_4440_000,1557325502587743 +10940141908690367388_4420_000_4440_000,1557325492587377 +10940141908690367388_4420_000_4440_000,1557325500587663 +10940141908690367388_4420_000_4440_000,1557325499087629 +15865907199900332614_760_000_780_000,1559313080537412 +15865907199900332614_760_000_780_000,1559313078037376 +15865907199900332614_760_000_780_000,1559313080037454 +15865907199900332614_760_000_780_000,1559313079537512 +15865907199900332614_760_000_780_000,1559313078537459 +15865907199900332614_760_000_780_000,1559313089537338 +15865907199900332614_760_000_780_000,1559313077537461 +15865907199900332614_760_000_780_000,1559313091537372 +15865907199900332614_760_000_780_000,1559313081037481 +15865907199900332614_760_000_780_000,1559313087537628 +15865907199900332614_760_000_780_000,1559313077037424 +15865907199900332614_760_000_780_000,1559313079037502 +15865907199900332614_760_000_780_000,1559313090537600 +15865907199900332614_760_000_780_000,1559313089037261 +15865907199900332614_760_000_780_000,1559313088037246 +15865907199900332614_760_000_780_000,1559313091037429 +15865907199900332614_760_000_780_000,1559313087037841 +15865907199900332614_760_000_780_000,1559313081537390 +15865907199900332614_760_000_780_000,1559313090037603 +15865907199900332614_760_000_780_000,1559313088537022 +16418654553014119039_4340_000_4360_000,1557548032247842 +16418654553014119039_4340_000_4360_000,1557548021247344 +16418654553014119039_4340_000_4360_000,1557548020747349 +16418654553014119039_4340_000_4360_000,1557548019247610 +16418654553014119039_4340_000_4360_000,1557548019747557 +16418654553014119039_4340_000_4360_000,1557548022747669 +16418654553014119039_4340_000_4360_000,1557548032748077 +16418654553014119039_4340_000_4360_000,1557548022247554 +16418654553014119039_4340_000_4360_000,1557548020247425 +16418654553014119039_4340_000_4360_000,1557548031247283 +16418654553014119039_4340_000_4360_000,1557548031747513 +16418654553014119039_4340_000_4360_000,1557548021747406 +16418654553014119039_4340_000_4360_000,1557548023747615 +16418654553014119039_4340_000_4360_000,1557548029247116 +16418654553014119039_4340_000_4360_000,1557548030247196 +16418654553014119039_4340_000_4360_000,1557548030747259 +16418654553014119039_4340_000_4360_000,1557548023247650 +16418654553014119039_4340_000_4360_000,1557548029747131 +16418654553014119039_4340_000_4360_000,1557548033248036 +16418654553014119039_4340_000_4360_000,1557548033747756 +2795127582672852315_4140_000_4160_000,1557963462811402 +2795127582672852315_4140_000_4160_000,1557963459811328 +2795127582672852315_4140_000_4160_000,1557963461311393 +2795127582672852315_4140_000_4160_000,1557963468811200 +2795127582672852315_4140_000_4160_000,1557963460311323 +2795127582672852315_4140_000_4160_000,1557963472811254 +2795127582672852315_4140_000_4160_000,1557963459311361 +2795127582672852315_4140_000_4160_000,1557963472311331 +2795127582672852315_4140_000_4160_000,1557963469811253 +2795127582672852315_4140_000_4160_000,1557963473311173 +2795127582672852315_4140_000_4160_000,1557963458811300 +2795127582672852315_4140_000_4160_000,1557963461811317 +2795127582672852315_4140_000_4160_000,1557963460811362 +2795127582672852315_4140_000_4160_000,1557963471811333 +2795127582672852315_4140_000_4160_000,1557963462311357 +2795127582672852315_4140_000_4160_000,1557963463311436 +2795127582672852315_4140_000_4160_000,1557963469311205 +2795127582672852315_4140_000_4160_000,1557963470811412 +2795127582672852315_4140_000_4160_000,1557963471311372 +2795127582672852315_4140_000_4160_000,1557963470311335 +10084636266401282188_1120_000_1140_000,1558407846397548 +10084636266401282188_1120_000_1140_000,1558407843897545 +10084636266401282188_1120_000_1140_000,1558407844397659 +10084636266401282188_1120_000_1140_000,1558407855397331 +10084636266401282188_1120_000_1140_000,1558407854397201 +10084636266401282188_1120_000_1140_000,1558407856897229 +10084636266401282188_1120_000_1140_000,1558407843397428 +10084636266401282188_1120_000_1140_000,1558407857397306 +10084636266401282188_1120_000_1140_000,1558407845897532 +10084636266401282188_1120_000_1140_000,1558407846897582 +10084636266401282188_1120_000_1140_000,1558407855897228 +10084636266401282188_1120_000_1140_000,1558407852897242 +10084636266401282188_1120_000_1140_000,1558407845397550 +10084636266401282188_1120_000_1140_000,1558407856397205 +10084636266401282188_1120_000_1140_000,1558407853897063 +10084636266401282188_1120_000_1140_000,1558407844897621 +10084636266401282188_1120_000_1140_000,1558407847397707 +10084636266401282188_1120_000_1140_000,1558407854897351 +10084636266401282188_1120_000_1140_000,1558407853397165 +10084636266401282188_1120_000_1140_000,1558407842897345 +2709541197299883157_1140_000_1160_000,1558407875897558 +2709541197299883157_1140_000_1160_000,1558407877397532 +2709541197299883157_1140_000_1160_000,1558407873397482 +2709541197299883157_1140_000_1160_000,1558407866897397 +2709541197299883157_1140_000_1160_000,1558407865397535 +2709541197299883157_1140_000_1160_000,1558407862897305 +2709541197299883157_1140_000_1160_000,1558407865897598 +2709541197299883157_1140_000_1160_000,1558407867397220 +2709541197299883157_1140_000_1160_000,1558407866397538 +2709541197299883157_1140_000_1160_000,1558407874397414 +2709541197299883157_1140_000_1160_000,1558407876897664 +2709541197299883157_1140_000_1160_000,1558407876397661 +2709541197299883157_1140_000_1160_000,1558407874897399 +2709541197299883157_1140_000_1160_000,1558407864897431 +2709541197299883157_1140_000_1160_000,1558407863397357 +2709541197299883157_1140_000_1160_000,1558407863897366 +2709541197299883157_1140_000_1160_000,1558407873897410 +2709541197299883157_1140_000_1160_000,1558407872897442 +2709541197299883157_1140_000_1160_000,1558407875397469 +2709541197299883157_1140_000_1160_000,1558407864397400 +13849332693800388551_960_000_980_000,1557264991038089 +13849332693800388551_960_000_980_000,1557264981037854 +13849332693800388551_960_000_980_000,1557264980537799 +13849332693800388551_960_000_980_000,1557264990038023 +13849332693800388551_960_000_980_000,1557264981537583 +13849332693800388551_960_000_980_000,1557264990537919 +13849332693800388551_960_000_980_000,1557264989537908 +13849332693800388551_960_000_980_000,1557264993538114 +13849332693800388551_960_000_980_000,1557264992037794 +13849332693800388551_960_000_980_000,1557264982537109 +13849332693800388551_960_000_980_000,1557264991537831 +13849332693800388551_960_000_980_000,1557264983537470 +13849332693800388551_960_000_980_000,1557264984037733 +13849332693800388551_960_000_980_000,1557264980037600 +13849332693800388551_960_000_980_000,1557264979537445 +13849332693800388551_960_000_980_000,1557264983037216 +13849332693800388551_960_000_980_000,1557264992537947 +13849332693800388551_960_000_980_000,1557264993038041 +13849332693800388551_960_000_980_000,1557264994038073 +13849332693800388551_960_000_980_000,1557264982037313 +10649066155322078676_1660_000_1680_000,1557240584087768 +10649066155322078676_1660_000_1680_000,1557240585587367 +10649066155322078676_1660_000_1680_000,1557240573663029 +10649066155322078676_1660_000_1680_000,1557240584587589 +10649066155322078676_1660_000_1680_000,1557240586087486 +10649066155322078676_1660_000_1680_000,1557240585087446 +10649066155322078676_1660_000_1680_000,1557240575671293 +10649066155322078676_1660_000_1680_000,1557240576677868 +10649066155322078676_1660_000_1680_000,1557240576174505 +10649066155322078676_1660_000_1680_000,1557240582087726 +10649066155322078676_1660_000_1680_000,1557240574666010 +10649066155322078676_1660_000_1680_000,1557240572662201 +10649066155322078676_1660_000_1680_000,1557240572162174 +10649066155322078676_1660_000_1680_000,1557240583587849 +10649066155322078676_1660_000_1680_000,1557240573162360 +10649066155322078676_1660_000_1680_000,1557240582587734 +10649066155322078676_1660_000_1680_000,1557240586587594 +10649066155322078676_1660_000_1680_000,1557240574164269 +10649066155322078676_1660_000_1680_000,1557240575168313 +10649066155322078676_1660_000_1680_000,1557240583087847 +14386836877680112549_4460_000_4480_000,1559179974137579 +14386836877680112549_4460_000_4480_000,1559179965637497 +14386836877680112549_4460_000_4480_000,1559179975137452 +14386836877680112549_4460_000_4480_000,1559179965137491 +14386836877680112549_4460_000_4480_000,1559179967137475 +14386836877680112549_4460_000_4480_000,1559179968137424 +14386836877680112549_4460_000_4480_000,1559179968637431 +14386836877680112549_4460_000_4480_000,1559179977137567 +14386836877680112549_4460_000_4480_000,1559179977637531 +14386836877680112549_4460_000_4480_000,1559179974637544 +14386836877680112549_4460_000_4480_000,1559179975637343 +14386836877680112549_4460_000_4480_000,1559179966637434 +14386836877680112549_4460_000_4480_000,1559179964137409 +14386836877680112549_4460_000_4480_000,1559179967637439 +14386836877680112549_4460_000_4480_000,1559179976637532 +14386836877680112549_4460_000_4480_000,1559179978137338 +14386836877680112549_4460_000_4480_000,1559179978637228 +14386836877680112549_4460_000_4480_000,1559179964637420 +14386836877680112549_4460_000_4480_000,1559179966137487 +14386836877680112549_4460_000_4480_000,1559179976137422 +1703056599550681101_4380_000_4400_000,1557548063747285 +1703056599550681101_4380_000_4400_000,1557548069747442 +1703056599550681101_4380_000_4400_000,1557548060747134 +1703056599550681101_4380_000_4400_000,1557548059247135 +1703056599550681101_4380_000_4400_000,1557548062747196 +1703056599550681101_4380_000_4400_000,1557548061747138 +1703056599550681101_4380_000_4400_000,1557548059747103 +1703056599550681101_4380_000_4400_000,1557548071747485 +1703056599550681101_4380_000_4400_000,1557548062247198 +1703056599550681101_4380_000_4400_000,1557548071247487 +1703056599550681101_4380_000_4400_000,1557548070747406 +1703056599550681101_4380_000_4400_000,1557548073247485 +1703056599550681101_4380_000_4400_000,1557548072747519 +1703056599550681101_4380_000_4400_000,1557548061247054 +1703056599550681101_4380_000_4400_000,1557548070247363 +1703056599550681101_4380_000_4400_000,1557548063247235 +1703056599550681101_4380_000_4400_000,1557548060247093 +1703056599550681101_4380_000_4400_000,1557548072247479 +1703056599550681101_4380_000_4400_000,1557548069247567 +1703056599550681101_4380_000_4400_000,1557548073747477 +9806821842001738961_4460_000_4480_000,1557548152749185 +9806821842001738961_4460_000_4480_000,1557548152249507 +9806821842001738961_4460_000_4480_000,1557548139248527 +9806821842001738961_4460_000_4480_000,1557548139748613 +9806821842001738961_4460_000_4480_000,1557548149748710 +9806821842001738961_4460_000_4480_000,1557548143745069 +9806821842001738961_4460_000_4480_000,1557548141247955 +9806821842001738961_4460_000_4480_000,1557548150749859 +9806821842001738961_4460_000_4480_000,1557548153248836 +9806821842001738961_4460_000_4480_000,1557548142746485 +9806821842001738961_4460_000_4480_000,1557548151749796 +9806821842001738961_4460_000_4480_000,1557548140248466 +9806821842001738961_4460_000_4480_000,1557548143245860 +9806821842001738961_4460_000_4480_000,1557548141747585 +9806821842001738961_4460_000_4480_000,1557548149247731 +9806821842001738961_4460_000_4480_000,1557548153748607 +9806821842001738961_4460_000_4480_000,1557548142247085 +9806821842001738961_4460_000_4480_000,1557548150249485 +9806821842001738961_4460_000_4480_000,1557548151249946 +9806821842001738961_4460_000_4480_000,1557548140748234 +4008112367880337022_3680_000_3700_000,1569854705325111 +4008112367880337022_3680_000_3700_000,1569854713325049 +4008112367880337022_3680_000_3700_000,1569854717325186 +4008112367880337022_3680_000_3700_000,1569854717825065 +4008112367880337022_3680_000_3700_000,1569854716325211 +4008112367880337022_3680_000_3700_000,1569854716825240 +4008112367880337022_3680_000_3700_000,1569854714325134 +4008112367880337022_3680_000_3700_000,1569854706825153 +4008112367880337022_3680_000_3700_000,1569854704325165 +4008112367880337022_3680_000_3700_000,1569854714825260 +4008112367880337022_3680_000_3700_000,1569854706325106 +4008112367880337022_3680_000_3700_000,1569854705825068 +4008112367880337022_3680_000_3700_000,1569854704825168 +4008112367880337022_3680_000_3700_000,1569854707325043 +4008112367880337022_3680_000_3700_000,1569854707824970 +4008112367880337022_3680_000_3700_000,1569854715325243 +4008112367880337022_3680_000_3700_000,1569854715825244 +4008112367880337022_3680_000_3700_000,1569854703825152 +4008112367880337022_3680_000_3700_000,1569854713825019 +4008112367880337022_3680_000_3700_000,1569854703325067 +3275806206237593341_1260_000_1280_000,1557544942819254 +3275806206237593341_1260_000_1280_000,1557544950297870 +3275806206237593341_1260_000_1280_000,1557544951297442 +3275806206237593341_1260_000_1280_000,1557544951797369 +3275806206237593341_1260_000_1280_000,1557544950797707 +3275806206237593341_1260_000_1280_000,1557544952297300 +3275806206237593341_1260_000_1280_000,1557544949299141 +3275806206237593341_1260_000_1280_000,1557544940320359 +3275806206237593341_1260_000_1280_000,1557544940820248 +3275806206237593341_1260_000_1280_000,1557544942319553 +3275806206237593341_1260_000_1280_000,1557544941320001 +3275806206237593341_1260_000_1280_000,1557544949798358 +3275806206237593341_1260_000_1280_000,1557544939320505 +3275806206237593341_1260_000_1280_000,1557544953797222 +3275806206237593341_1260_000_1280_000,1557544953297262 +3275806206237593341_1260_000_1280_000,1557544943318943 +3275806206237593341_1260_000_1280_000,1557544941819785 +3275806206237593341_1260_000_1280_000,1557544943818501 +3275806206237593341_1260_000_1280_000,1557544939820502 +3275806206237593341_1260_000_1280_000,1557544952797231 +16942495693882305487_4340_000_4360_000,1559179844137784 +16942495693882305487_4340_000_4360_000,1559179844637716 +16942495693882305487_4340_000_4360_000,1559179846637950 +16942495693882305487_4340_000_4360_000,1559179855137769 +16942495693882305487_4340_000_4360_000,1559179854137701 +16942495693882305487_4340_000_4360_000,1559179846137883 +16942495693882305487_4340_000_4360_000,1559179845637785 +16942495693882305487_4340_000_4360_000,1559179857137780 +16942495693882305487_4340_000_4360_000,1559179848137768 +16942495693882305487_4340_000_4360_000,1559179847637805 +16942495693882305487_4340_000_4360_000,1559179848637749 +16942495693882305487_4340_000_4360_000,1559179855637782 +16942495693882305487_4340_000_4360_000,1559179845137739 +16942495693882305487_4340_000_4360_000,1559179858137740 +16942495693882305487_4340_000_4360_000,1559179856637781 +16942495693882305487_4340_000_4360_000,1559179854637737 +16942495693882305487_4340_000_4360_000,1559179857637814 +16942495693882305487_4340_000_4360_000,1559179856137797 +16942495693882305487_4340_000_4360_000,1559179858637797 +16942495693882305487_4340_000_4360_000,1559179847137875 +5764319372514665214_2480_000_2500_000,1558034992472993 +5764319372514665214_2480_000_2500_000,1558034983472954 +5764319372514665214_2480_000_2500_000,1558034982972924 +5764319372514665214_2480_000_2500_000,1558034989972975 +5764319372514665214_2480_000_2500_000,1558034981473075 +5764319372514665214_2480_000_2500_000,1558034990472969 +5764319372514665214_2480_000_2500_000,1558034984472951 +5764319372514665214_2480_000_2500_000,1558034991472965 +5764319372514665214_2480_000_2500_000,1558034980973024 +5764319372514665214_2480_000_2500_000,1558034979972956 +5764319372514665214_2480_000_2500_000,1558034981973026 +5764319372514665214_2480_000_2500_000,1558034991973002 +5764319372514665214_2480_000_2500_000,1558034990972960 +5764319372514665214_2480_000_2500_000,1558034993973011 +5764319372514665214_2480_000_2500_000,1558034982472951 +5764319372514665214_2480_000_2500_000,1558034983972951 +5764319372514665214_2480_000_2500_000,1558034993473006 +5764319372514665214_2480_000_2500_000,1558034980472954 +5764319372514665214_2480_000_2500_000,1558034994473066 +5764319372514665214_2480_000_2500_000,1558034992972995 +3485136235103477552_600_000_620_000,1559312920037900 +3485136235103477552_600_000_620_000,1559312918536992 +3485136235103477552_600_000_620_000,1559312929037490 +3485136235103477552_600_000_620_000,1559312931537400 +3485136235103477552_600_000_620_000,1559312921537438 +3485136235103477552_600_000_620_000,1559312917537421 +3485136235103477552_600_000_620_000,1559312927536888 +3485136235103477552_600_000_620_000,1559312921037521 +3485136235103477552_600_000_620_000,1559312919537665 +3485136235103477552_600_000_620_000,1559312928037154 +3485136235103477552_600_000_620_000,1559312930537328 +3485136235103477552_600_000_620_000,1559312917037757 +3485136235103477552_600_000_620_000,1559312930037396 +3485136235103477552_600_000_620_000,1559312918037188 +3485136235103477552_600_000_620_000,1559312929537548 +3485136235103477552_600_000_620_000,1559312927037001 +3485136235103477552_600_000_620_000,1559312928537375 +3485136235103477552_600_000_620_000,1559312931037329 +3485136235103477552_600_000_620_000,1559312919037170 +3485136235103477552_600_000_620_000,1559312920537711 +13732041959462600641_720_000_740_000,1558742853976814 +13732041959462600641_720_000_740_000,1558742855976028 +13732041959462600641_720_000_740_000,1558742843475326 +13732041959462600641_720_000_740_000,1558742854976703 +13732041959462600641_720_000_740_000,1558742843975547 +13732041959462600641_720_000_740_000,1558742846475978 +13732041959462600641_720_000_740_000,1558742844975697 +13732041959462600641_720_000_740_000,1558742856975912 +13732041959462600641_720_000_740_000,1558742855476179 +13732041959462600641_720_000_740_000,1558742842975141 +13732041959462600641_720_000_740_000,1558742847476056 +13732041959462600641_720_000_740_000,1558742857475609 +13732041959462600641_720_000_740_000,1558742844475636 +13732041959462600641_720_000_740_000,1558742845475848 +13732041959462600641_720_000_740_000,1558742845975911 +13732041959462600641_720_000_740_000,1558742846976015 +13732041959462600641_720_000_740_000,1558742854477097 +13732041959462600641_720_000_740_000,1558742852976440 +13732041959462600641_720_000_740_000,1558742853476695 +13732041959462600641_720_000_740_000,1558742856476018 +8684065200957554260_2700_000_2720_000,1566246362851376 +8684065200957554260_2700_000_2720_000,1566246374351315 +8684065200957554260_2700_000_2720_000,1566246373851362 +8684065200957554260_2700_000_2720_000,1566246372351287 +8684065200957554260_2700_000_2720_000,1566246363351451 +8684065200957554260_2700_000_2720_000,1566246362351295 +8684065200957554260_2700_000_2720_000,1566246363851429 +8684065200957554260_2700_000_2720_000,1566246366351318 +8684065200957554260_2700_000_2720_000,1566246375351264 +8684065200957554260_2700_000_2720_000,1566246373351328 +8684065200957554260_2700_000_2720_000,1566246376351894 +8684065200957554260_2700_000_2720_000,1566246376852628 +8684065200957554260_2700_000_2720_000,1566246364851337 +8684065200957554260_2700_000_2720_000,1566246375851419 +8684065200957554260_2700_000_2720_000,1566246365351325 +8684065200957554260_2700_000_2720_000,1566246366851318 +8684065200957554260_2700_000_2720_000,1566246365851320 +8684065200957554260_2700_000_2720_000,1566246364351329 +8684065200957554260_2700_000_2720_000,1566246372851306 +8684065200957554260_2700_000_2720_000,1566246374851263 +10410418118434245359_5140_000_5160_000,1557326223047734 +10410418118434245359_5140_000_5160_000,1557326221547648 +10410418118434245359_5140_000_5160_000,1557326223547764 +10410418118434245359_5140_000_5160_000,1557326209047560 +10410418118434245359_5140_000_5160_000,1557326213047602 +10410418118434245359_5140_000_5160_000,1557326212047572 +10410418118434245359_5140_000_5160_000,1557326221047770 +10410418118434245359_5140_000_5160_000,1557326211047663 +10410418118434245359_5140_000_5160_000,1557326211547653 +10410418118434245359_5140_000_5160_000,1557326220547772 +10410418118434245359_5140_000_5160_000,1557326212547575 +10410418118434245359_5140_000_5160_000,1557326209547585 +10410418118434245359_5140_000_5160_000,1557326210047617 +10410418118434245359_5140_000_5160_000,1557326220047729 +10410418118434245359_5140_000_5160_000,1557326222047648 +10410418118434245359_5140_000_5160_000,1557326222547699 +10410418118434245359_5140_000_5160_000,1557326219047730 +10410418118434245359_5140_000_5160_000,1557326219547770 +10410418118434245359_5140_000_5160_000,1557326210547626 +10410418118434245359_5140_000_5160_000,1557326213547578 +7240042450405902042_580_000_600_000,1559312901037775 +7240042450405902042_580_000_600_000,1559312897037515 +7240042450405902042_580_000_600_000,1559312899537484 +7240042450405902042_580_000_600_000,1559312898537394 +7240042450405902042_580_000_600_000,1559312911537589 +7240042450405902042_580_000_600_000,1559312900037413 +7240042450405902042_580_000_600_000,1559312907037317 +7240042450405902042_580_000_600_000,1559312901538082 +7240042450405902042_580_000_600_000,1559312909537272 +7240042450405902042_580_000_600_000,1559312908537793 +7240042450405902042_580_000_600_000,1559312899037443 +7240042450405902042_580_000_600_000,1559312910036813 +7240042450405902042_580_000_600_000,1559312910537019 +7240042450405902042_580_000_600_000,1559312908037618 +7240042450405902042_580_000_600_000,1559312909037663 +7240042450405902042_580_000_600_000,1559312911037369 +7240042450405902042_580_000_600_000,1559312898037440 +7240042450405902042_580_000_600_000,1559312900537375 +7240042450405902042_580_000_600_000,1559312897537487 +7240042450405902042_580_000_600_000,1559312907537791 +5585555620508986875_720_000_740_000,1559313037538117 +5585555620508986875_720_000_740_000,1559313050537687 +5585555620508986875_720_000_740_000,1559313047537497 +5585555620508986875_720_000_740_000,1559313048037350 +5585555620508986875_720_000_740_000,1559313040037581 +5585555620508986875_720_000_740_000,1559313039037173 +5585555620508986875_720_000_740_000,1559313038037778 +5585555620508986875_720_000_740_000,1559313051537445 +5585555620508986875_720_000_740_000,1559313040537431 +5585555620508986875_720_000_740_000,1559313047037528 +5585555620508986875_720_000_740_000,1559313049537681 +5585555620508986875_720_000_740_000,1559313048537310 +5585555620508986875_720_000_740_000,1559313041537128 +5585555620508986875_720_000_740_000,1559313049037464 +5585555620508986875_720_000_740_000,1559313037038225 +5585555620508986875_720_000_740_000,1559313041037197 +5585555620508986875_720_000_740_000,1559313051037544 +5585555620508986875_720_000_740_000,1559313050037678 +5585555620508986875_720_000_740_000,1559313038537390 +5585555620508986875_720_000_740_000,1559313039537279 +2714318267497393311_480_000_500_000,1558150298237122 +2714318267497393311_480_000_500_000,1558150287237469 +2714318267497393311_480_000_500_000,1558150290237709 +2714318267497393311_480_000_500_000,1558150296737452 +2714318267497393311_480_000_500_000,1558150287737484 +2714318267497393311_480_000_500_000,1558150299237252 +2714318267497393311_480_000_500_000,1558150288237628 +2714318267497393311_480_000_500_000,1558150300237538 +2714318267497393311_480_000_500_000,1558150297737232 +2714318267497393311_480_000_500_000,1558150289737802 +2714318267497393311_480_000_500_000,1558150290737726 +2714318267497393311_480_000_500_000,1558150296237346 +2714318267497393311_480_000_500_000,1558150297237200 +2714318267497393311_480_000_500_000,1558150288737667 +2714318267497393311_480_000_500_000,1558150286237588 +2714318267497393311_480_000_500_000,1558150289237769 +2714318267497393311_480_000_500_000,1558150286737552 +2714318267497393311_480_000_500_000,1558150298737101 +2714318267497393311_480_000_500_000,1558150299737398 +2714318267497393311_480_000_500_000,1558150300737648 +13790309965076620852_6520_000_6540_000,1574126957899706 +13790309965076620852_6520_000_6540_000,1574126959900038 +13790309965076620852_6520_000_6540_000,1574126955399851 +13790309965076620852_6520_000_6540_000,1574126968399982 +13790309965076620852_6520_000_6540_000,1574126965399821 +13790309965076620852_6520_000_6540_000,1574126958399589 +13790309965076620852_6520_000_6540_000,1574126957399943 +13790309965076620852_6520_000_6540_000,1574126967399978 +13790309965076620852_6520_000_6540_000,1574126958899663 +13790309965076620852_6520_000_6540_000,1574126956399869 +13790309965076620852_6520_000_6540_000,1574126966400006 +13790309965076620852_6520_000_6540_000,1574126956899942 +13790309965076620852_6520_000_6540_000,1574126968900008 +13790309965076620852_6520_000_6540_000,1574126966900090 +13790309965076620852_6520_000_6540_000,1574126959399883 +13790309965076620852_6520_000_6540_000,1574126965899849 +13790309965076620852_6520_000_6540_000,1574126967900033 +13790309965076620852_6520_000_6540_000,1574126955899899 +13790309965076620852_6520_000_6540_000,1574126969400087 +13790309965076620852_6520_000_6540_000,1574126969900021 +17387485694427326992_760_000_780_000,1557843958062722 +17387485694427326992_760_000_780_000,1557843968062691 +17387485694427326992_760_000_780_000,1557843968562687 +17387485694427326992_760_000_780_000,1557843959062736 +17387485694427326992_760_000_780_000,1557843967562765 +17387485694427326992_760_000_780_000,1557843956562821 +17387485694427326992_760_000_780_000,1557843955062802 +17387485694427326992_760_000_780_000,1557843965062813 +17387485694427326992_760_000_780_000,1557843969062758 +17387485694427326992_760_000_780_000,1557843969562794 +17387485694427326992_760_000_780_000,1557843966062703 +17387485694427326992_760_000_780_000,1557843967062734 +17387485694427326992_760_000_780_000,1557843965562735 +17387485694427326992_760_000_780_000,1557843959562659 +17387485694427326992_760_000_780_000,1557843957062778 +17387485694427326992_760_000_780_000,1557843957562803 +17387485694427326992_760_000_780_000,1557843966562710 +17387485694427326992_760_000_780_000,1557843956062840 +17387485694427326992_760_000_780_000,1557843958562737 +17387485694427326992_760_000_780_000,1557843955562873 +9350911198443552989_680_000_700_000,1557363451237372 +9350911198443552989_680_000_700_000,1557363441737465 +9350911198443552989_680_000_700_000,1557363449237250 +9350911198443552989_680_000_700_000,1557363439737622 +9350911198443552989_680_000_700_000,1557363438237327 +9350911198443552989_680_000_700_000,1557363440237403 +9350911198443552989_680_000_700_000,1557363441237340 +9350911198443552989_680_000_700_000,1557363447237793 +9350911198443552989_680_000_700_000,1557363451737437 +9350911198443552989_680_000_700_000,1557363449737386 +9350911198443552989_680_000_700_000,1557363437237375 +9350911198443552989_680_000_700_000,1557363437737418 +9350911198443552989_680_000_700_000,1557363440737261 +9350911198443552989_680_000_700_000,1557363448737285 +9350911198443552989_680_000_700_000,1557363439237622 +9350911198443552989_680_000_700_000,1557363447737794 +9350911198443552989_680_000_700_000,1557363438737444 +9350911198443552989_680_000_700_000,1557363450237422 +9350911198443552989_680_000_700_000,1557363450737354 +9350911198443552989_680_000_700_000,1557363448237517 +6174376739759381004_3240_000_3260_000,1557877015199162 +6174376739759381004_3240_000_3260_000,1557877006199178 +6174376739759381004_3240_000_3260_000,1557877004199181 +6174376739759381004_3240_000_3260_000,1557877005699128 +6174376739759381004_3240_000_3260_000,1557877008199313 +6174376739759381004_3240_000_3260_000,1557877016199192 +6174376739759381004_3240_000_3260_000,1557877014699134 +6174376739759381004_3240_000_3260_000,1557877007699341 +6174376739759381004_3240_000_3260_000,1557877017199143 +6174376739759381004_3240_000_3260_000,1557877014199207 +6174376739759381004_3240_000_3260_000,1557877016699133 +6174376739759381004_3240_000_3260_000,1557877004699166 +6174376739759381004_3240_000_3260_000,1557877018699207 +6174376739759381004_3240_000_3260_000,1557877015699193 +6174376739759381004_3240_000_3260_000,1557877008699136 +6174376739759381004_3240_000_3260_000,1557877005199071 +6174376739759381004_3240_000_3260_000,1557877018199234 +6174376739759381004_3240_000_3260_000,1557877007199256 +6174376739759381004_3240_000_3260_000,1557877006699224 +6174376739759381004_3240_000_3260_000,1557877017699172 +12153647356523920032_2560_000_2580_000,1572710128774788 +12153647356523920032_2560_000_2580_000,1572710126774725 +12153647356523920032_2560_000_2580_000,1572710120774792 +12153647356523920032_2560_000_2580_000,1572710129274823 +12153647356523920032_2560_000_2580_000,1572710116774778 +12153647356523920032_2560_000_2580_000,1572710119774754 +12153647356523920032_2560_000_2580_000,1572710117774722 +12153647356523920032_2560_000_2580_000,1572710130274767 +12153647356523920032_2560_000_2580_000,1572710128274786 +12153647356523920032_2560_000_2580_000,1572710120274760 +12153647356523920032_2560_000_2580_000,1572710130774726 +12153647356523920032_2560_000_2580_000,1572710118274683 +12153647356523920032_2560_000_2580_000,1572710127274765 +12153647356523920032_2560_000_2580_000,1572710127774811 +12153647356523920032_2560_000_2580_000,1572710126274748 +12153647356523920032_2560_000_2580_000,1572710118774754 +12153647356523920032_2560_000_2580_000,1572710117274754 +12153647356523920032_2560_000_2580_000,1572710129774796 +12153647356523920032_2560_000_2580_000,1572710119274760 +12153647356523920032_2560_000_2580_000,1572710116274782 +11933765568165455008_2940_000_2960_000,1557198335387209 +11933765568165455008_2940_000_2960_000,1557198338387310 +11933765568165455008_2940_000_2960_000,1557198347387352 +11933765568165455008_2940_000_2960_000,1557198338887301 +11933765568165455008_2940_000_2960_000,1557198348387315 +11933765568165455008_2940_000_2960_000,1557198345387416 +11933765568165455008_2940_000_2960_000,1557198335887178 +11933765568165455008_2940_000_2960_000,1557198344387408 +11933765568165455008_2940_000_2960_000,1557198344887332 +11933765568165455008_2940_000_2960_000,1557198337387225 +11933765568165455008_2940_000_2960_000,1557198345887369 +11933765568165455008_2940_000_2960_000,1557198347887352 +11933765568165455008_2940_000_2960_000,1557198346887349 +11933765568165455008_2940_000_2960_000,1557198336387249 +11933765568165455008_2940_000_2960_000,1557198348887399 +11933765568165455008_2940_000_2960_000,1557198334887218 +11933765568165455008_2940_000_2960_000,1557198334387221 +11933765568165455008_2940_000_2960_000,1557198337887303 +11933765568165455008_2940_000_2960_000,1557198336887239 +11933765568165455008_2940_000_2960_000,1557198346387373 +10161761842905385678_760_000_780_000,1557196157797448 +10161761842905385678_760_000_780_000,1557196158797350 +10161761842905385678_760_000_780_000,1557196168297765 +10161761842905385678_760_000_780_000,1557196155797333 +10161761842905385678_760_000_780_000,1557196167797613 +10161761842905385678_760_000_780_000,1557196166297587 +10161761842905385678_760_000_780_000,1557196156797479 +10161761842905385678_760_000_780_000,1557196167297622 +10161761842905385678_760_000_780_000,1557196154797258 +10161761842905385678_760_000_780_000,1557196154297327 +10161761842905385678_760_000_780_000,1557196165297463 +10161761842905385678_760_000_780_000,1557196165797474 +10161761842905385678_760_000_780_000,1557196156297413 +10161761842905385678_760_000_780_000,1557196164297460 +10161761842905385678_760_000_780_000,1557196158297419 +10161761842905385678_760_000_780_000,1557196168797617 +10161761842905385678_760_000_780_000,1557196166797651 +10161761842905385678_760_000_780_000,1557196155297293 +10161761842905385678_760_000_780_000,1557196164797422 +10161761842905385678_760_000_780_000,1557196157297472 +6922883602463663456_2220_000_2240_000,1558152040772834 +6922883602463663456_2220_000_2240_000,1558152026756411 +6922883602463663456_2220_000_2240_000,1558152028764982 +6922883602463663456_2220_000_2240_000,1558152036772202 +6922883602463663456_2220_000_2240_000,1558152029768625 +6922883602463663456_2220_000_2240_000,1558152037272084 +6922883602463663456_2220_000_2240_000,1558152038772394 +6922883602463663456_2220_000_2240_000,1558152036272158 +6922883602463663456_2220_000_2240_000,1558152030771388 +6922883602463663456_2220_000_2240_000,1558152038272239 +6922883602463663456_2220_000_2240_000,1558152040272803 +6922883602463663456_2220_000_2240_000,1558152030270137 +6922883602463663456_2220_000_2240_000,1558152037772191 +6922883602463663456_2220_000_2240_000,1558152027760810 +6922883602463663456_2220_000_2240_000,1558152027258557 +6922883602463663456_2220_000_2240_000,1558152026254441 +6922883602463663456_2220_000_2240_000,1558152039272550 +6922883602463663456_2220_000_2240_000,1558152039772680 +6922883602463663456_2220_000_2240_000,1558152029266975 +6922883602463663456_2220_000_2240_000,1558152028262942 +3341890853207909601_1020_000_1040_000,1573927803625099 +3341890853207909601_1020_000_1040_000,1573927790125189 +3341890853207909601_1020_000_1040_000,1573927802125062 +3341890853207909601_1020_000_1040_000,1573927801625067 +3341890853207909601_1020_000_1040_000,1573927794625079 +3341890853207909601_1020_000_1040_000,1573927790625242 +3341890853207909601_1020_000_1040_000,1573927792624930 +3341890853207909601_1020_000_1040_000,1573927791125208 +3341890853207909601_1020_000_1040_000,1573927800624954 +3341890853207909601_1020_000_1040_000,1573927804625096 +3341890853207909601_1020_000_1040_000,1573927800124914 +3341890853207909601_1020_000_1040_000,1573927802625074 +3341890853207909601_1020_000_1040_000,1573927792124827 +3341890853207909601_1020_000_1040_000,1573927794125084 +3341890853207909601_1020_000_1040_000,1573927801125097 +3341890853207909601_1020_000_1040_000,1573927793624995 +3341890853207909601_1020_000_1040_000,1573927793124963 +3341890853207909601_1020_000_1040_000,1573927804125097 +3341890853207909601_1020_000_1040_000,1573927803125097 +3341890853207909601_1020_000_1040_000,1573927791625026 +17756183617755834457_1940_000_1960_000,1558017204447293 +17756183617755834457_1940_000_1960_000,1558017214436996 +17756183617755834457_1940_000_1960_000,1558017215429120 +17756183617755834457_1940_000_1960_000,1558017206446333 +17756183617755834457_1940_000_1960_000,1558017207446078 +17756183617755834457_1940_000_1960_000,1558017218421930 +17756183617755834457_1940_000_1960_000,1558017213940930 +17756183617755834457_1940_000_1960_000,1558017217922014 +17756183617755834457_1940_000_1960_000,1558017206945999 +17756183617755834457_1940_000_1960_000,1558017205447104 +17756183617755834457_1940_000_1960_000,1558017214932926 +17756183617755834457_1940_000_1960_000,1558017217422255 +17756183617755834457_1940_000_1960_000,1558017215925793 +17756183617755834457_1940_000_1960_000,1558017208447290 +17756183617755834457_1940_000_1960_000,1558017216423608 +17756183617755834457_1940_000_1960_000,1558017207946577 +17756183617755834457_1940_000_1960_000,1558017216922725 +17756183617755834457_1940_000_1960_000,1558017204947246 +17756183617755834457_1940_000_1960_000,1558017205946707 +17756183617755834457_1940_000_1960_000,1558017203947410 +2218963221891181906_4360_000_4380_000,1573932454073919 +2218963221891181906_4360_000_4380_000,1573932458574312 +2218963221891181906_4360_000_4380_000,1573932456574286 +2218963221891181906_4360_000_4380_000,1573932445149910 +2218963221891181906_4360_000_4380_000,1573932457574335 +2218963221891181906_4360_000_4380_000,1573932444649899 +2218963221891181906_4360_000_4380_000,1573932446649928 +2218963221891181906_4360_000_4380_000,1573932445649884 +2218963221891181906_4360_000_4380_000,1573932448150256 +2218963221891181906_4360_000_4380_000,1573932444149933 +2218963221891181906_4360_000_4380_000,1573932447149977 +2218963221891181906_4360_000_4380_000,1573932454574319 +2218963221891181906_4360_000_4380_000,1573932456074299 +2218963221891181906_4360_000_4380_000,1573932455574265 +2218963221891181906_4360_000_4380_000,1573932457074331 +2218963221891181906_4360_000_4380_000,1573932458074340 +2218963221891181906_4360_000_4380_000,1573932448650899 +2218963221891181906_4360_000_4380_000,1573932446149941 +2218963221891181906_4360_000_4380_000,1573932447650058 +2218963221891181906_4360_000_4380_000,1573932455074331 +10149575340910243572_2720_000_2740_000,1558035231962663 +10149575340910243572_2720_000_2740_000,1558035232462596 +10149575340910243572_2720_000_2740_000,1558035234462274 +10149575340910243572_2720_000_2740_000,1558035232962512 +10149575340910243572_2720_000_2740_000,1558035233462396 +10149575340910243572_2720_000_2740_000,1558035230462351 +10149575340910243572_2720_000_2740_000,1558035231462594 +10149575340910243572_2720_000_2740_000,1558035230962448 +10149575340910243572_2720_000_2740_000,1558035229962648 +10149575340910243572_2720_000_2740_000,1558035233962327 +3459095437766396887_1600_000_1620_000,1559177116218096 +3459095437766396887_1600_000_1620_000,1559177116717458 +3459095437766396887_1600_000_1620_000,1559177108222874 +3459095437766396887_1600_000_1620_000,1559177115219166 +3459095437766396887_1600_000_1620_000,1559177117217000 +3459095437766396887_1600_000_1620_000,1559177114719614 +3459095437766396887_1600_000_1620_000,1559177115718671 +3459095437766396887_1600_000_1620_000,1559177105721360 +3459095437766396887_1600_000_1620_000,1559177108722993 +3459095437766396887_1600_000_1620_000,1559177107221934 +3459095437766396887_1600_000_1620_000,1559177106221852 +3459095437766396887_1600_000_1620_000,1559177114219949 +3459095437766396887_1600_000_1620_000,1559177105220562 +3459095437766396887_1600_000_1620_000,1559177107722383 +3459095437766396887_1600_000_1620_000,1559177118216369 +3459095437766396887_1600_000_1620_000,1559177117716745 +3459095437766396887_1600_000_1620_000,1559177104218831 +3459095437766396887_1600_000_1620_000,1559177104719526 +3459095437766396887_1600_000_1620_000,1559177118715883 +3459095437766396887_1600_000_1620_000,1559177106721948 +8249122135171526629_520_000_540_000,1559184839587788 +8249122135171526629_520_000_540_000,1559184839087463 +8249122135171526629_520_000_540_000,1559184838086814 +8249122135171526629_520_000_540_000,1559184829585106 +8249122135171526629_520_000_540_000,1559184829085741 +8249122135171526629_520_000_540_000,1559184841587404 +8249122135171526629_520_000_540_000,1559184832087286 +8249122135171526629_520_000_540_000,1559184831086036 +8249122135171526629_520_000_540_000,1559184830585419 +8249122135171526629_520_000_540_000,1559184838587000 +8249122135171526629_520_000_540_000,1559184842087749 +8249122135171526629_520_000_540_000,1559184827587385 +8249122135171526629_520_000_540_000,1559184828087141 +8249122135171526629_520_000_540_000,1559184837586942 +8249122135171526629_520_000_540_000,1559184840587321 +8249122135171526629_520_000_540_000,1559184830085083 +8249122135171526629_520_000_540_000,1559184828586572 +8249122135171526629_520_000_540_000,1559184841087135 +8249122135171526629_520_000_540_000,1559184840087626 +8249122135171526629_520_000_540_000,1559184831586778 +1664548685643064400_2240_000_2260_000,1572730660024796 +1664548685643064400_2240_000_2260_000,1572730661524793 +1664548685643064400_2240_000_2260_000,1572730664024893 +1664548685643064400_2240_000_2260_000,1572730661024763 +1664548685643064400_2240_000_2260_000,1572730659524712 +1664548685643064400_2240_000_2260_000,1572730651024914 +1664548685643064400_2240_000_2260_000,1572730652024805 +1664548685643064400_2240_000_2260_000,1572730663524712 +1664548685643064400_2240_000_2260_000,1572730662524607 +1664548685643064400_2240_000_2260_000,1572730654024948 +1664548685643064400_2240_000_2260_000,1572730660524763 +1664548685643064400_2240_000_2260_000,1572730649525094 +1664548685643064400_2240_000_2260_000,1572730651524841 +1664548685643064400_2240_000_2260_000,1572730653024965 +1664548685643064400_2240_000_2260_000,1572730662024682 +1664548685643064400_2240_000_2260_000,1572730652524780 +1664548685643064400_2240_000_2260_000,1572730650524867 +1664548685643064400_2240_000_2260_000,1572730663024572 +1664548685643064400_2240_000_2260_000,1572730650024950 +1664548685643064400_2240_000_2260_000,1572730653525063 +4916600861562283346_3880_000_3900_000,1559179394137429 +4916600861562283346_3880_000_3900_000,1559179396137504 +4916600861562283346_3880_000_3900_000,1559179396637496 +4916600861562283346_3880_000_3900_000,1559179398137489 +4916600861562283346_3880_000_3900_000,1559179388637375 +4916600861562283346_3880_000_3900_000,1559179398637508 +4916600861562283346_3880_000_3900_000,1559179386637413 +4916600861562283346_3880_000_3900_000,1559179386137493 +4916600861562283346_3880_000_3900_000,1559179397137450 +4916600861562283346_3880_000_3900_000,1559179387637365 +4916600861562283346_3880_000_3900_000,1559179384137390 +4916600861562283346_3880_000_3900_000,1559179387137336 +4916600861562283346_3880_000_3900_000,1559179384637499 +4916600861562283346_3880_000_3900_000,1559179388137403 +4916600861562283346_3880_000_3900_000,1559179397637459 +4916600861562283346_3880_000_3900_000,1559179395137442 +4916600861562283346_3880_000_3900_000,1559179385137537 +4916600861562283346_3880_000_3900_000,1559179385637530 +4916600861562283346_3880_000_3900_000,1559179395637456 +4916600861562283346_3880_000_3900_000,1559179394637383 +10802932587105534078_1280_000_1300_000,1557888796948097 +10802932587105534078_1280_000_1300_000,1557888798448099 +10802932587105534078_1280_000_1300_000,1557888806449251 +10802932587105534078_1280_000_1300_000,1557888809449360 +10802932587105534078_1280_000_1300_000,1557888810448859 +10802932587105534078_1280_000_1300_000,1557888800447985 +10802932587105534078_1280_000_1300_000,1557888807948674 +10802932587105534078_1280_000_1300_000,1557888809949023 +10802932587105534078_1280_000_1300_000,1557888810949122 +10802932587105534078_1280_000_1300_000,1557888799948216 +10802932587105534078_1280_000_1300_000,1557888798948041 +10802932587105534078_1280_000_1300_000,1557888800948126 +10802932587105534078_1280_000_1300_000,1557888806949187 +10802932587105534078_1280_000_1300_000,1557888807448803 +10802932587105534078_1280_000_1300_000,1557888799448247 +10802932587105534078_1280_000_1300_000,1557888808449065 +10802932587105534078_1280_000_1300_000,1557888797948166 +10802932587105534078_1280_000_1300_000,1557888796448121 +10802932587105534078_1280_000_1300_000,1557888808949531 +10802932587105534078_1280_000_1300_000,1557888797448185 +13748565785898537200_680_000_700_000,1573621439474829 +13748565785898537200_680_000_700_000,1573621439974767 +13748565785898537200_680_000_700_000,1573621429474915 +13748565785898537200_680_000_700_000,1573621429974924 +13748565785898537200_680_000_700_000,1573621440974804 +13748565785898537200_680_000_700_000,1573621441474863 +13748565785898537200_680_000_700_000,1573621443974860 +13748565785898537200_680_000_700_000,1573621431474829 +13748565785898537200_680_000_700_000,1573621441974787 +13748565785898537200_680_000_700_000,1573621432474859 +13748565785898537200_680_000_700_000,1573621443474808 +13748565785898537200_680_000_700_000,1573621430974792 +13748565785898537200_680_000_700_000,1573621433974860 +13748565785898537200_680_000_700_000,1573621431974875 +13748565785898537200_680_000_700_000,1573621442974839 +13748565785898537200_680_000_700_000,1573621430474807 +13748565785898537200_680_000_700_000,1573621442474772 +13748565785898537200_680_000_700_000,1573621440474758 +13748565785898537200_680_000_700_000,1573621433474826 +13748565785898537200_680_000_700_000,1573621432974900 +14643284977980826278_520_000_540_000,1558150336737516 +14643284977980826278_520_000_540_000,1558150328237510 +14643284977980826278_520_000_540_000,1558150337237361 +14643284977980826278_520_000_540_000,1558150327737447 +14643284977980826278_520_000_540_000,1558150327237391 +14643284977980826278_520_000_540_000,1558150339737419 +14643284977980826278_520_000_540_000,1558150326737588 +14643284977980826278_520_000_540_000,1558150326237829 +14643284977980826278_520_000_540_000,1558150330737609 +14643284977980826278_520_000_540_000,1558150329237498 +14643284977980826278_520_000_540_000,1558150330237732 +14643284977980826278_520_000_540_000,1558150339237427 +14643284977980826278_520_000_540_000,1558150340237494 +14643284977980826278_520_000_540_000,1558150340737501 +14643284977980826278_520_000_540_000,1558150329737609 +14643284977980826278_520_000_540_000,1558150328737468 +14643284977980826278_520_000_540_000,1558150337737326 +14643284977980826278_520_000_540_000,1558150338237396 +14643284977980826278_520_000_540_000,1558150336237586 +14643284977980826278_520_000_540_000,1558150338737431 +4045613324047897473_940_000_960_000,1558493338074162 +4045613324047897473_940_000_960_000,1558493348073783 +4045613324047897473_940_000_960_000,1558493350074053 +4045613324047897473_940_000_960_000,1558493345573992 +4045613324047897473_940_000_960_000,1558493347574007 +4045613324047897473_940_000_960_000,1558493338574044 +4045613324047897473_940_000_960_000,1558493335574296 +4045613324047897473_940_000_960_000,1558493339573912 +4045613324047897473_940_000_960_000,1558493336574269 +4045613324047897473_940_000_960_000,1558493347074035 +4045613324047897473_940_000_960_000,1558493346574102 +4045613324047897473_940_000_960_000,1558493346073989 +4045613324047897473_940_000_960_000,1558493337574148 +4045613324047897473_940_000_960_000,1558493348573778 +4045613324047897473_940_000_960_000,1558493349074012 +4045613324047897473_940_000_960_000,1558493337074219 +4045613324047897473_940_000_960_000,1558493349574122 +4045613324047897473_940_000_960_000,1558493340074053 +4045613324047897473_940_000_960_000,1558493336074290 +4045613324047897473_940_000_960_000,1558493339073948 +2257381802419655779_820_000_840_000,1558402111847622 +2257381802419655779_820_000_840_000,1558402122847222 +2257381802419655779_820_000_840_000,1558402108847992 +2257381802419655779_820_000_840_000,1558402118847287 +2257381802419655779_820_000_840_000,1558402120847365 +2257381802419655779_820_000_840_000,1558402110847064 +2257381802419655779_820_000_840_000,1558402119847426 +2257381802419655779_820_000_840_000,1558402122347099 +2257381802419655779_820_000_840_000,1558402121847019 +2257381802419655779_820_000_840_000,1558402121347177 +2257381802419655779_820_000_840_000,1558402109847716 +2257381802419655779_820_000_840_000,1558402112347811 +2257381802419655779_820_000_840_000,1558402123347308 +2257381802419655779_820_000_840_000,1558402112847819 +2257381802419655779_820_000_840_000,1558402109347833 +2257381802419655779_820_000_840_000,1558402120347479 +2257381802419655779_820_000_840_000,1558402111347219 +2257381802419655779_820_000_840_000,1558402110347368 +2257381802419655779_820_000_840_000,1558402119347368 +2257381802419655779_820_000_840_000,1558402113347613 +4054036670499089296_2300_000_2320_000,1557187714649115 +4054036670499089296_2300_000_2320_000,1557187716649135 +4054036670499089296_2300_000_2320_000,1557187704649276 +4054036670499089296_2300_000_2320_000,1557187707149136 +4054036670499089296_2300_000_2320_000,1557187716149170 +4054036670499089296_2300_000_2320_000,1557187704149193 +4054036670499089296_2300_000_2320_000,1557187717148945 +4054036670499089296_2300_000_2320_000,1557187707649076 +4054036670499089296_2300_000_2320_000,1557187706649119 +4054036670499089296_2300_000_2320_000,1557187705149208 +4054036670499089296_2300_000_2320_000,1557187715649133 +4054036670499089296_2300_000_2320_000,1557187713649046 +4054036670499089296_2300_000_2320_000,1557187706149101 +4054036670499089296_2300_000_2320_000,1557187715149153 +4054036670499089296_2300_000_2320_000,1557187703148999 +4054036670499089296_2300_000_2320_000,1557187703649173 +4054036670499089296_2300_000_2320_000,1557187713149076 +4054036670499089296_2300_000_2320_000,1557187714149098 +4054036670499089296_2300_000_2320_000,1557187717649252 +4054036670499089296_2300_000_2320_000,1557187705649134 +12056192874455954437_140_000_160_000,1557843345612667 +12056192874455954437_140_000_160_000,1557843349612578 +12056192874455954437_140_000_160_000,1557843345112543 +12056192874455954437_140_000_160_000,1557843335112508 +12056192874455954437_140_000_160_000,1557843338612551 +12056192874455954437_140_000_160_000,1557843336612494 +12056192874455954437_140_000_160_000,1557843338112693 +12056192874455954437_140_000_160_000,1557843337112658 +12056192874455954437_140_000_160_000,1557843339612639 +12056192874455954437_140_000_160_000,1557843348612302 +12056192874455954437_140_000_160_000,1557843335612429 +12056192874455954437_140_000_160_000,1557843336112396 +12056192874455954437_140_000_160_000,1557843349112419 +12056192874455954437_140_000_160_000,1557843337612796 +12056192874455954437_140_000_160_000,1557843346612497 +12056192874455954437_140_000_160_000,1557843347612615 +12056192874455954437_140_000_160_000,1557843348112448 +12056192874455954437_140_000_160_000,1557843346112603 +12056192874455954437_140_000_160_000,1557843339112601 +12056192874455954437_140_000_160_000,1557843347112468 +13034900465317073842_1700_000_1720_000,1559143078524545 +13034900465317073842_1700_000_1720_000,1559143065016062 +13034900465317073842_1700_000_1720_000,1559143064015948 +13034900465317073842_1700_000_1720_000,1559143074021060 +13034900465317073842_1700_000_1720_000,1559143068016067 +13034900465317073842_1700_000_1720_000,1559143076523597 +13034900465317073842_1700_000_1720_000,1559143067016248 +13034900465317073842_1700_000_1720_000,1559143075522514 +13034900465317073842_1700_000_1720_000,1559143077023973 +13034900465317073842_1700_000_1720_000,1559143064515955 +13034900465317073842_1700_000_1720_000,1559143066516551 +13034900465317073842_1700_000_1720_000,1559143077524362 +13034900465317073842_1700_000_1720_000,1559143068516366 +13034900465317073842_1700_000_1720_000,1559143076023064 +13034900465317073842_1700_000_1720_000,1559143074521426 +13034900465317073842_1700_000_1720_000,1559143067516020 +13034900465317073842_1700_000_1720_000,1559143065516232 +13034900465317073842_1700_000_1720_000,1559143066016549 +13034900465317073842_1700_000_1720_000,1559143075021878 +13034900465317073842_1700_000_1720_000,1559143078024530 +7511993111693456743_3880_000_3900_000,1557963202297466 +7511993111693456743_3880_000_3900_000,1557963212297515 +7511993111693456743_3880_000_3900_000,1557963200297419 +7511993111693456743_3880_000_3900_000,1557963202797419 +7511993111693456743_3880_000_3900_000,1557963211297319 +7511993111693456743_3880_000_3900_000,1557963211797549 +7511993111693456743_3880_000_3900_000,1557963201297473 +7511993111693456743_3880_000_3900_000,1557963209797116 +7511993111693456743_3880_000_3900_000,1557963210297172 +7511993111693456743_3880_000_3900_000,1557963200797464 +7511993111693456743_3880_000_3900_000,1557963209297327 +7511993111693456743_3880_000_3900_000,1557963208797520 +7511993111693456743_3880_000_3900_000,1557963198797401 +7511993111693456743_3880_000_3900_000,1557963213297448 +7511993111693456743_3880_000_3900_000,1557963210797182 +7511993111693456743_3880_000_3900_000,1557963201797503 +7511993111693456743_3880_000_3900_000,1557963199297286 +7511993111693456743_3880_000_3900_000,1557963199797330 +7511993111693456743_3880_000_3900_000,1557963203297377 +7511993111693456743_3880_000_3900_000,1557963212797472 +9355489589631690177_4800_000_4820_000,1557342366562650 +9355489589631690177_4800_000_4820_000,1557342358062536 +9355489589631690177_4800_000_4820_000,1557342369562809 +9355489589631690177_4800_000_4820_000,1557342357562530 +9355489589631690177_4800_000_4820_000,1557342367062748 +9355489589631690177_4800_000_4820_000,1557342356562423 +9355489589631690177_4800_000_4820_000,1557342355562520 +9355489589631690177_4800_000_4820_000,1557342358562309 +9355489589631690177_4800_000_4820_000,1557342368562561 +9355489589631690177_4800_000_4820_000,1557342367562723 +9355489589631690177_4800_000_4820_000,1557342365562451 +9355489589631690177_4800_000_4820_000,1557342369062698 +9355489589631690177_4800_000_4820_000,1557342366062493 +9355489589631690177_4800_000_4820_000,1557342368062616 +9355489589631690177_4800_000_4820_000,1557342357062509 +9355489589631690177_4800_000_4820_000,1557342359062110 +9355489589631690177_4800_000_4820_000,1557342355062436 +9355489589631690177_4800_000_4820_000,1557342359562031 +9355489589631690177_4800_000_4820_000,1557342365062568 +9355489589631690177_4800_000_4820_000,1557342356062469 +3522804493060229409_3400_000_3420_000,1557855904472271 +3522804493060229409_3400_000_3420_000,1557855907472634 +3522804493060229409_3400_000_3420_000,1557855896472328 +3522804493060229409_3400_000_3420_000,1557855892972587 +3522804493060229409_3400_000_3420_000,1557855906972551 +3522804493060229409_3400_000_3420_000,1557855905472302 +3522804493060229409_3400_000_3420_000,1557855904972296 +3522804493060229409_3400_000_3420_000,1557855905972396 +3522804493060229409_3400_000_3420_000,1557855906472495 +3522804493060229409_3400_000_3420_000,1557855893972382 +3522804493060229409_3400_000_3420_000,1557855897472206 +3522804493060229409_3400_000_3420_000,1557855902972245 +3522804493060229409_3400_000_3420_000,1557855894972377 +3522804493060229409_3400_000_3420_000,1557855893472505 +3522804493060229409_3400_000_3420_000,1557855895472388 +3522804493060229409_3400_000_3420_000,1557855896972244 +3522804493060229409_3400_000_3420_000,1557855903472293 +3522804493060229409_3400_000_3420_000,1557855895972316 +3522804493060229409_3400_000_3420_000,1557855894472345 +3522804493060229409_3400_000_3420_000,1557855903972289 +8566480970798227989_500_000_520_000,1557239425612429 +8566480970798227989_500_000_520_000,1557239414112699 +8566480970798227989_500_000_520_000,1557239413112667 +8566480970798227989_500_000_520_000,1557239415112533 +8566480970798227989_500_000_520_000,1557239416612460 +8566480970798227989_500_000_520_000,1557239423112799 +8566480970798227989_500_000_520_000,1557239415612490 +8566480970798227989_500_000_520_000,1557239422112884 +8566480970798227989_500_000_520_000,1557239412612624 +8566480970798227989_500_000_520_000,1557239424612659 +8566480970798227989_500_000_520_000,1557239412112652 +8566480970798227989_500_000_520_000,1557239422612861 +8566480970798227989_500_000_520_000,1557239416112464 +8566480970798227989_500_000_520_000,1557239423612728 +8566480970798227989_500_000_520_000,1557239413612747 +8566480970798227989_500_000_520_000,1557239426112320 +8566480970798227989_500_000_520_000,1557239426612303 +8566480970798227989_500_000_520_000,1557239414612596 +8566480970798227989_500_000_520_000,1557239425112554 +8566480970798227989_500_000_520_000,1557239424112739 +6278307160249415497_1700_000_1720_000,1558034213921937 +6278307160249415497_1700_000_1720_000,1558034201922721 +6278307160249415497_1700_000_1720_000,1558034202422649 +6278307160249415497_1700_000_1720_000,1558034202922472 +6278307160249415497_1700_000_1720_000,1558034204422154 +6278307160249415497_1700_000_1720_000,1558034214422280 +6278307160249415497_1700_000_1720_000,1558034213421817 +6278307160249415497_1700_000_1720_000,1558034211421372 +6278307160249415497_1700_000_1720_000,1558034203922216 +6278307160249415497_1700_000_1720_000,1558034200922728 +6278307160249415497_1700_000_1720_000,1558034212921821 +6278307160249415497_1700_000_1720_000,1558034210421304 +6278307160249415497_1700_000_1720_000,1558034201422689 +6278307160249415497_1700_000_1720_000,1558034211921700 +6278307160249415497_1700_000_1720_000,1558034209921189 +6278307160249415497_1700_000_1720_000,1558034212421831 +6278307160249415497_1700_000_1720_000,1558034200422683 +6278307160249415497_1700_000_1720_000,1558034210921320 +6278307160249415497_1700_000_1720_000,1558034203422353 +6278307160249415497_1700_000_1720_000,1558034199922726 +13787943721654585343_1220_000_1240_000,1558483374422389 +13787943721654585343_1220_000_1240_000,1558483360422540 +13787943721654585343_1220_000_1240_000,1558483362922326 +13787943721654585343_1220_000_1240_000,1558483361422280 +13787943721654585343_1220_000_1240_000,1558483370422349 +13787943721654585343_1220_000_1240_000,1558483359922533 +13787943721654585343_1220_000_1240_000,1558483372922276 +13787943721654585343_1220_000_1240_000,1558483364422414 +13787943721654585343_1220_000_1240_000,1558483369922463 +13787943721654585343_1220_000_1240_000,1558483373422253 +13787943721654585343_1220_000_1240_000,1558483360922432 +13787943721654585343_1220_000_1240_000,1558483370922205 +13787943721654585343_1220_000_1240_000,1558483371922349 +13787943721654585343_1220_000_1240_000,1558483371422242 +13787943721654585343_1220_000_1240_000,1558483361922245 +13787943721654585343_1220_000_1240_000,1558483362422314 +13787943721654585343_1220_000_1240_000,1558483363422326 +13787943721654585343_1220_000_1240_000,1558483363922364 +13787943721654585343_1220_000_1240_000,1558483372422320 +13787943721654585343_1220_000_1240_000,1558483373922325 +10998289306141768318_1280_000_1300_000,1558483433397038 +10998289306141768318_1280_000_1300_000,1558483430411803 +10998289306141768318_1280_000_1300_000,1558483420422343 +10998289306141768318_1280_000_1300_000,1558483434396435 +10998289306141768318_1280_000_1300_000,1558483421422280 +10998289306141768318_1280_000_1300_000,1558483423422502 +10998289306141768318_1280_000_1300_000,1558483430908205 +10998289306141768318_1280_000_1300_000,1558483424422579 +10998289306141768318_1280_000_1300_000,1558483433896475 +10998289306141768318_1280_000_1300_000,1558483423922620 +10998289306141768318_1280_000_1300_000,1558483419922414 +10998289306141768318_1280_000_1300_000,1558483422422324 +10998289306141768318_1280_000_1300_000,1558483431404397 +10998289306141768318_1280_000_1300_000,1558483431901030 +10998289306141768318_1280_000_1300_000,1558483429915076 +10998289306141768318_1280_000_1300_000,1558483420922273 +10998289306141768318_1280_000_1300_000,1558483421922318 +10998289306141768318_1280_000_1300_000,1558483422922327 +10998289306141768318_1280_000_1300_000,1558483432398938 +10998289306141768318_1280_000_1300_000,1558483432897848 +7435516779413778621_4440_000_4460_000,1557325510087987 +7435516779413778621_4440_000_4460_000,1557325509088023 +7435516779413778621_4440_000_4460_000,1557325509588017 +7435516779413778621_4440_000_4460_000,1557325522112585 +7435516779413778621_4440_000_4460_000,1557325511088136 +7435516779413778621_4440_000_4460_000,1557325513590433 +7435516779413778621_4440_000_4460_000,1557325512588488 +7435516779413778621_4440_000_4460_000,1557325521112794 +7435516779413778621_4440_000_4460_000,1557325513089176 +7435516779413778621_4440_000_4460_000,1557325522612689 +7435516779413778621_4440_000_4460_000,1557325520112870 +7435516779413778621_4440_000_4460_000,1557325523612525 +7435516779413778621_4440_000_4460_000,1557325511588133 +7435516779413778621_4440_000_4460_000,1557325521612655 +7435516779413778621_4440_000_4460_000,1557325519113921 +7435516779413778621_4440_000_4460_000,1557325520612844 +7435516779413778621_4440_000_4460_000,1557325510588071 +7435516779413778621_4440_000_4460_000,1557325523112680 +7435516779413778621_4440_000_4460_000,1557325519613322 +7435516779413778621_4440_000_4460_000,1557325512088233 +13944616099709049906_1020_000_1040_000,1558493425524322 +13944616099709049906_1020_000_1040_000,1558493417024071 +13944616099709049906_1020_000_1040_000,1558493426024320 +13944616099709049906_1020_000_1040_000,1558493416024098 +13944616099709049906_1020_000_1040_000,1558493429524171 +13944616099709049906_1020_000_1040_000,1558493426524287 +13944616099709049906_1020_000_1040_000,1558493419024193 +13944616099709049906_1020_000_1040_000,1558493430024138 +13944616099709049906_1020_000_1040_000,1558493427524280 +13944616099709049906_1020_000_1040_000,1558493415524136 +13944616099709049906_1020_000_1040_000,1558493427024273 +13944616099709049906_1020_000_1040_000,1558493429024223 +13944616099709049906_1020_000_1040_000,1558493428524220 +13944616099709049906_1020_000_1040_000,1558493420024171 +13944616099709049906_1020_000_1040_000,1558493418024131 +13944616099709049906_1020_000_1040_000,1558493418524161 +13944616099709049906_1020_000_1040_000,1558493417524102 +13944616099709049906_1020_000_1040_000,1558493419524165 +13944616099709049906_1020_000_1040_000,1558493416524077 +13944616099709049906_1020_000_1040_000,1558493428024253 +8229317157758012712_3860_000_3880_000,1559179375137657 +8229317157758012712_3860_000_3880_000,1559179375637448 +8229317157758012712_3860_000_3880_000,1559179366637361 +8229317157758012712_3860_000_3880_000,1559179368137382 +8229317157758012712_3860_000_3880_000,1559179367137366 +8229317157758012712_3860_000_3880_000,1559179376137327 +8229317157758012712_3860_000_3880_000,1559179378637568 +8229317157758012712_3860_000_3880_000,1559179374137643 +8229317157758012712_3860_000_3880_000,1559179374637715 +8229317157758012712_3860_000_3880_000,1559179376637419 +8229317157758012712_3860_000_3880_000,1559179364137325 +8229317157758012712_3860_000_3880_000,1559179377637503 +8229317157758012712_3860_000_3880_000,1559179366137360 +8229317157758012712_3860_000_3880_000,1559179368637389 +8229317157758012712_3860_000_3880_000,1559179377137484 +8229317157758012712_3860_000_3880_000,1559179364637326 +8229317157758012712_3860_000_3880_000,1559179365137367 +8229317157758012712_3860_000_3880_000,1559179367637354 +8229317157758012712_3860_000_3880_000,1559179378137535 +8229317157758012712_3860_000_3880_000,1559179365637366 +5638240639308158118_4220_000_4240_000,1555267988099080 +5638240639308158118_4220_000_4240_000,1555267981099003 +5638240639308158118_4220_000_4240_000,1555267980599134 +5638240639308158118_4220_000_4240_000,1555267989099215 +5638240639308158118_4220_000_4240_000,1555267977599158 +5638240639308158118_4220_000_4240_000,1555267987599108 +5638240639308158118_4220_000_4240_000,1555267986599172 +5638240639308158118_4220_000_4240_000,1555267979599132 +5638240639308158118_4220_000_4240_000,1555267988599141 +5638240639308158118_4220_000_4240_000,1555267990098844 +5638240639308158118_4220_000_4240_000,1555267990598105 +5638240639308158118_4220_000_4240_000,1555267979099131 +5638240639308158118_4220_000_4240_000,1555267978599123 +5638240639308158118_4220_000_4240_000,1555267987099206 +5638240639308158118_4220_000_4240_000,1555267976599172 +5638240639308158118_4220_000_4240_000,1555267977099159 +5638240639308158118_4220_000_4240_000,1555267989599152 +5638240639308158118_4220_000_4240_000,1555267980099165 +5638240639308158118_4220_000_4240_000,1555267978099155 +5638240639308158118_4220_000_4240_000,1555267991096855 +15272375112495403395_620_000_640_000,1559189217599985 +15272375112495403395_620_000_640_000,1559189230099846 +15272375112495403395_620_000_640_000,1559189221600285 +15272375112495403395_620_000_640_000,1559189228599908 +15272375112495403395_620_000_640_000,1559189228100026 +15272375112495403395_620_000_640_000,1559189231099755 +15272375112495403395_620_000_640_000,1559189229599850 +15272375112495403395_620_000_640_000,1559189217099978 +15272375112495403395_620_000_640_000,1559189220599788 +15272375112495403395_620_000_640_000,1559189229099841 +15272375112495403395_620_000_640_000,1559189227100268 +15272375112495403395_620_000_640_000,1559189231599710 +15272375112495403395_620_000_640_000,1559189218599758 +15272375112495403395_620_000_640_000,1559189219599785 +15272375112495403395_620_000_640_000,1559189218099858 +15272375112495403395_620_000_640_000,1559189230599799 +15272375112495403395_620_000_640_000,1559189219099720 +15272375112495403395_620_000_640_000,1559189221099879 +15272375112495403395_620_000_640_000,1559189227600224 +15272375112495403395_620_000_640_000,1559189220099860 +8993680275027614595_2520_000_2540_000,1555280202399639 +8993680275027614595_2520_000_2540_000,1555280199899606 +8993680275027614595_2520_000_2540_000,1555280211375470 +8993680275027614595_2520_000_2540_000,1555280199399568 +8993680275027614595_2520_000_2540_000,1555280212875223 +8993680275027614595_2520_000_2540_000,1555280208875515 +8993680275027614595_2520_000_2540_000,1555280202899788 +8993680275027614595_2520_000_2540_000,1555280210374654 +8993680275027614595_2520_000_2540_000,1555280210875023 +8993680275027614595_2520_000_2540_000,1555280200899582 +8993680275027614595_2520_000_2540_000,1555280201399518 +8993680275027614595_2520_000_2540_000,1555280212375553 +8993680275027614595_2520_000_2540_000,1555280209874639 +8993680275027614595_2520_000_2540_000,1555280211875697 +8993680275027614595_2520_000_2540_000,1555280209374829 +8993680275027614595_2520_000_2540_000,1555280203399700 +8993680275027614595_2520_000_2540_000,1555280201899495 +8993680275027614595_2520_000_2540_000,1555280213374830 +8993680275027614595_2520_000_2540_000,1555280200399612 +8993680275027614595_2520_000_2540_000,1555280198899595 +8688567562597583972_940_000_960_000,1555217344950039 +8688567562597583972_940_000_960_000,1555217347949948 +8688567562597583972_940_000_960_000,1555217356449802 +8688567562597583972_940_000_960_000,1555217353949933 +8688567562597583972_940_000_960_000,1555217345450004 +8688567562597583972_940_000_960_000,1555217346449944 +8688567562597583972_940_000_960_000,1555217343450016 +8688567562597583972_940_000_960_000,1555217344449945 +8688567562597583972_940_000_960_000,1555217346949874 +8688567562597583972_940_000_960_000,1555217355449905 +8688567562597583972_940_000_960_000,1555217353449883 +8688567562597583972_940_000_960_000,1555217355949898 +8688567562597583972_940_000_960_000,1555217354949900 +8688567562597583972_940_000_960_000,1555217357449853 +8688567562597583972_940_000_960_000,1555217345949937 +8688567562597583972_940_000_960_000,1555217354449934 +8688567562597583972_940_000_960_000,1555217356949774 +8688567562597583972_940_000_960_000,1555217343949948 +8688567562597583972_940_000_960_000,1555217357949939 +8688567562597583972_940_000_960_000,1555217347449863 +7247823803417339098_2320_000_2340_000,1557197726848807 +7247823803417339098_2320_000_2340_000,1557197726349233 +7247823803417339098_2320_000_2340_000,1557197727348551 +7247823803417339098_2320_000_2340_000,1557197714347252 +7247823803417339098_2320_000_2340_000,1557197716347129 +7247823803417339098_2320_000_2340_000,1557197725349846 +7247823803417339098_2320_000_2340_000,1557197718347455 +7247823803417339098_2320_000_2340_000,1557197716847198 +7247823803417339098_2320_000_2340_000,1557197715847235 +7247823803417339098_2320_000_2340_000,1557197724349365 +7247823803417339098_2320_000_2340_000,1557197714847182 +7247823803417339098_2320_000_2340_000,1557197717847546 +7247823803417339098_2320_000_2340_000,1557197728348372 +7247823803417339098_2320_000_2340_000,1557197715347156 +7247823803417339098_2320_000_2340_000,1557197727848417 +7247823803417339098_2320_000_2340_000,1557197718847355 +7247823803417339098_2320_000_2340_000,1557197728848372 +7247823803417339098_2320_000_2340_000,1557197724849707 +7247823803417339098_2320_000_2340_000,1557197725849623 +7247823803417339098_2320_000_2340_000,1557197717347349 +2601205676330128831_4880_000_4900_000,1555183240199075 +2601205676330128831_4880_000_4900_000,1555183251775192 +2601205676330128831_4880_000_4900_000,1555183242695259 +2601205676330128831_4880_000_4900_000,1555183239698969 +2601205676330128831_4880_000_4900_000,1555183252774590 +2601205676330128831_4880_000_4900_000,1555183239198898 +2601205676330128831_4880_000_4900_000,1555183241697881 +2601205676330128831_4880_000_4900_000,1555183250274996 +2601205676330128831_4880_000_4900_000,1555183248775035 +2601205676330128831_4880_000_4900_000,1555183242196604 +2601205676330128831_4880_000_4900_000,1555183241198707 +2601205676330128831_4880_000_4900_000,1555183252274928 +2601205676330128831_4880_000_4900_000,1555183253274584 +2601205676330128831_4880_000_4900_000,1555183249775067 +2601205676330128831_4880_000_4900_000,1555183238698908 +2601205676330128831_4880_000_4900_000,1555183240699040 +2601205676330128831_4880_000_4900_000,1555183243193747 +2601205676330128831_4880_000_4900_000,1555183251275298 +2601205676330128831_4880_000_4900_000,1555183249275187 +2601205676330128831_4880_000_4900_000,1555183250775187 +14737335824319407706_1980_000_2000_000,1556068257625722 +14737335824319407706_1980_000_2000_000,1556068264624994 +14737335824319407706_1980_000_2000_000,1556068253125108 +14737335824319407706_1980_000_2000_000,1556068256626068 +14737335824319407706_1980_000_2000_000,1556068256125917 +14737335824319407706_1980_000_2000_000,1556068267124989 +14737335824319407706_1980_000_2000_000,1556068254125759 +14737335824319407706_1980_000_2000_000,1556068265124999 +14737335824319407706_1980_000_2000_000,1556068263125013 +14737335824319407706_1980_000_2000_000,1556068266125077 +14737335824319407706_1980_000_2000_000,1556068254626070 +14737335824319407706_1980_000_2000_000,1556068265625046 +14737335824319407706_1980_000_2000_000,1556068255126360 +14737335824319407706_1980_000_2000_000,1556068267624889 +14737335824319407706_1980_000_2000_000,1556068255626085 +14737335824319407706_1980_000_2000_000,1556068266625069 +14737335824319407706_1980_000_2000_000,1556068264124922 +14737335824319407706_1980_000_2000_000,1556068257126022 +14737335824319407706_1980_000_2000_000,1556068253625378 +14737335824319407706_1980_000_2000_000,1556068263624987 +10504764403039842352_460_000_480_000,1558060925875055 +10504764403039842352_460_000_480_000,1558060940374703 +10504764403039842352_460_000_480_000,1558060939874709 +10504764403039842352_460_000_480_000,1558060937374792 +10504764403039842352_460_000_480_000,1558060927874686 +10504764403039842352_460_000_480_000,1558060926874887 +10504764403039842352_460_000_480_000,1558060930375221 +10504764403039842352_460_000_480_000,1558060926375083 +10504764403039842352_460_000_480_000,1558060935875120 +10504764403039842352_460_000_480_000,1558060936375015 +10504764403039842352_460_000_480_000,1558060936874787 +10504764403039842352_460_000_480_000,1558060938875168 +10504764403039842352_460_000_480_000,1558060928875075 +10504764403039842352_460_000_480_000,1558060937874938 +10504764403039842352_460_000_480_000,1558060928374842 +10504764403039842352_460_000_480_000,1558060929375235 +10504764403039842352_460_000_480_000,1558060938375035 +10504764403039842352_460_000_480_000,1558060939374902 +4140965781175793864_460_000_480_000,1559189068049919 +4140965781175793864_460_000_480_000,1559189060549423 +4140965781175793864_460_000_480_000,1559189058052659 +4140965781175793864_460_000_480_000,1559189070549944 +4140965781175793864_460_000_480_000,1559189071550057 +4140965781175793864_460_000_480_000,1559189067049957 +4140965781175793864_460_000_480_000,1559189061049573 +4140965781175793864_460_000_480_000,1559189059549297 +4140965781175793864_460_000_480_000,1559189067549997 +4140965781175793864_460_000_480_000,1559189058551289 +4140965781175793864_460_000_480_000,1559189057056840 +4140965781175793864_460_000_480_000,1559189069550001 +4140965781175793864_460_000_480_000,1559189068549926 +4140965781175793864_460_000_480_000,1559189069049952 +4140965781175793864_460_000_480_000,1559189059049934 +4140965781175793864_460_000_480_000,1559189057554573 +4140965781175793864_460_000_480_000,1559189070049942 +4140965781175793864_460_000_480_000,1559189061549638 +4140965781175793864_460_000_480_000,1559189071050027 +4140965781175793864_460_000_480_000,1559189060049248 +14188689528137485670_2660_000_2680_000,1555687836099829 +14188689528137485670_2660_000_2680_000,1555687847574536 +14188689528137485670_2660_000_2680_000,1555687834599917 +14188689528137485670_2660_000_2680_000,1555687835599804 +14188689528137485670_2660_000_2680_000,1555687844576878 +14188689528137485670_2660_000_2680_000,1555687838099816 +14188689528137485670_2660_000_2680_000,1555687846574299 +14188689528137485670_2660_000_2680_000,1555687836599840 +14188689528137485670_2660_000_2680_000,1555687837099812 +14188689528137485670_2660_000_2680_000,1555687848074544 +14188689528137485670_2660_000_2680_000,1555687845075193 +14188689528137485670_2660_000_2680_000,1555687834099910 +14188689528137485670_2660_000_2680_000,1555687845574255 +14188689528137485670_2660_000_2680_000,1555687847074492 +14188689528137485670_2660_000_2680_000,1555687835099800 +14188689528137485670_2660_000_2680_000,1555687843582715 +14188689528137485670_2660_000_2680_000,1555687837599851 +14188689528137485670_2660_000_2680_000,1555687833599780 +14188689528137485670_2660_000_2680_000,1555687846074113 +14188689528137485670_2660_000_2680_000,1555687844079474 +18149616047892103767_2460_000_2480_000,1555706658299969 +18149616047892103767_2460_000_2480_000,1555706646800116 +18149616047892103767_2460_000_2480_000,1555706656800049 +18149616047892103767_2460_000_2480_000,1555706647300089 +18149616047892103767_2460_000_2480_000,1555706645799946 +18149616047892103767_2460_000_2480_000,1555706645299873 +18149616047892103767_2460_000_2480_000,1555706644299834 +18149616047892103767_2460_000_2480_000,1555706654299962 +18149616047892103767_2460_000_2480_000,1555706648799880 +18149616047892103767_2460_000_2480_000,1555706656300141 +18149616047892103767_2460_000_2480_000,1555706644799899 +18149616047892103767_2460_000_2480_000,1555706658800051 +18149616047892103767_2460_000_2480_000,1555706655300035 +18149616047892103767_2460_000_2480_000,1555706654799999 +18149616047892103767_2460_000_2480_000,1555706655800109 +18149616047892103767_2460_000_2480_000,1555706657299969 +18149616047892103767_2460_000_2480_000,1555706646300071 +18149616047892103767_2460_000_2480_000,1555706657799945 +18149616047892103767_2460_000_2480_000,1555706647800020 +18149616047892103767_2460_000_2480_000,1555706648299913 +5026942594071056992_3120_000_3140_000,1555462125499896 +5026942594071056992_3120_000_3140_000,1555462133999526 +5026942594071056992_3120_000_3140_000,1555462131999686 +5026942594071056992_3120_000_3140_000,1555462120999711 +5026942594071056992_3120_000_3140_000,1555462123499771 +5026942594071056992_3120_000_3140_000,1555462132499693 +5026942594071056992_3120_000_3140_000,1555462124499589 +5026942594071056992_3120_000_3140_000,1555462122500198 +5026942594071056992_3120_000_3140_000,1555462123999626 +5026942594071056992_3120_000_3140_000,1555462130999515 +5026942594071056992_3120_000_3140_000,1555462123000001 +5026942594071056992_3120_000_3140_000,1555462121499912 +5026942594071056992_3120_000_3140_000,1555462132999655 +5026942594071056992_3120_000_3140_000,1555462135499500 +5026942594071056992_3120_000_3140_000,1555462124999696 +5026942594071056992_3120_000_3140_000,1555462133499574 +5026942594071056992_3120_000_3140_000,1555462122000279 +5026942594071056992_3120_000_3140_000,1555462134999525 +5026942594071056992_3120_000_3140_000,1555462131499619 +5026942594071056992_3120_000_3140_000,1555462134499515 +11987368976578218644_1340_000_1360_000,1557240254147006 +11987368976578218644_1340_000_1360_000,1557240256647136 +11987368976578218644_1340_000_1360_000,1557240253147019 +11987368976578218644_1340_000_1360_000,1557240264121600 +11987368976578218644_1340_000_1360_000,1557240266622584 +11987368976578218644_1340_000_1360_000,1557240253646981 +11987368976578218644_1340_000_1360_000,1557240263622577 +11987368976578218644_1340_000_1360_000,1557240255647121 +11987368976578218644_1340_000_1360_000,1557240266122577 +11987368976578218644_1340_000_1360_000,1557240252646979 +11987368976578218644_1340_000_1360_000,1557240256147181 +11987368976578218644_1340_000_1360_000,1557240265622400 +11987368976578218644_1340_000_1360_000,1557240263124752 +11987368976578218644_1340_000_1360_000,1557240252147007 +11987368976578218644_1340_000_1360_000,1557240254647011 +11987368976578218644_1340_000_1360_000,1557240264621606 +11987368976578218644_1340_000_1360_000,1557240265121984 +11987368976578218644_1340_000_1360_000,1557240255147121 +11987368976578218644_1340_000_1360_000,1557240262627879 +11987368976578218644_1340_000_1360_000,1557240262131544 +17136775999940024630_4860_000_4880_000,1555381565899350 +17136775999940024630_4860_000_4880_000,1555381569399418 +17136775999940024630_4860_000_4880_000,1555381577399397 +17136775999940024630_4860_000_4880_000,1555381567899452 +17136775999940024630_4860_000_4880_000,1555381579899405 +17136775999940024630_4860_000_4880_000,1555381576399429 +17136775999940024630_4860_000_4880_000,1555381566399384 +17136775999940024630_4860_000_4880_000,1555381569899411 +17136775999940024630_4860_000_4880_000,1555381579399300 +17136775999940024630_4860_000_4880_000,1555381576899420 +17136775999940024630_4860_000_4880_000,1555381565399404 +17136775999940024630_4860_000_4880_000,1555381575399420 +17136775999940024630_4860_000_4880_000,1555381578399393 +17136775999940024630_4860_000_4880_000,1555381567399421 +17136775999940024630_4860_000_4880_000,1555381575899458 +17136775999940024630_4860_000_4880_000,1555381577899394 +17136775999940024630_4860_000_4880_000,1555381568399448 +17136775999940024630_4860_000_4880_000,1555381568899445 +17136775999940024630_4860_000_4880_000,1555381578899304 +10980133015080705026_780_000_800_000,1557159347347517 +10980133015080705026_780_000_800_000,1557159341347548 +10980133015080705026_780_000_800_000,1557159350347179 +10980133015080705026_780_000_800_000,1557159338347170 +10980133015080705026_780_000_800_000,1557159348347894 +10980133015080705026_780_000_800_000,1557159341847592 +10980133015080705026_780_000_800_000,1557159340347306 +10980133015080705026_780_000_800_000,1557159351347307 +10980133015080705026_780_000_800_000,1557159339347070 +10980133015080705026_780_000_800_000,1557159349347437 +10980133015080705026_780_000_800_000,1557159348847749 +10980133015080705026_780_000_800_000,1557159337346937 +10980133015080705026_780_000_800_000,1557159340847461 +10980133015080705026_780_000_800_000,1557159350847321 +10980133015080705026_780_000_800_000,1557159337847132 +10980133015080705026_780_000_800_000,1557159349847214 +10980133015080705026_780_000_800_000,1557159347847829 +10980133015080705026_780_000_800_000,1557159338847114 +10980133015080705026_780_000_800_000,1557159351847230 +10980133015080705026_780_000_800_000,1557159339847156 +17792628511034220885_2360_000_2380_000,1555038976374997 +17792628511034220885_2360_000_2380_000,1555038978374871 +17792628511034220885_2360_000_2380_000,1555038976874968 +17792628511034220885_2360_000_2380_000,1555038975875022 +17792628511034220885_2360_000_2380_000,1555038968860681 +17792628511034220885_2360_000_2380_000,1555038964850579 +17792628511034220885_2360_000_2380_000,1555038974875036 +17792628511034220885_2360_000_2380_000,1555038977374963 +17792628511034220885_2360_000_2380_000,1555038978874913 +17792628511034220885_2360_000_2380_000,1555038966351482 +17792628511034220885_2360_000_2380_000,1555038979375036 +17792628511034220885_2360_000_2380_000,1555038965850871 +17792628511034220885_2360_000_2380_000,1555038977874932 +17792628511034220885_2360_000_2380_000,1555038967353934 +17792628511034220885_2360_000_2380_000,1555038969363655 +17792628511034220885_2360_000_2380_000,1555038965350606 +17792628511034220885_2360_000_2380_000,1555038966852499 +17792628511034220885_2360_000_2380_000,1555038968358038 diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/waymo/preprocess_waymo.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/waymo/preprocess_waymo.py new file mode 100644 index 0000000000000000000000000000000000000000..33a309fc2a03c52b63a43863ac40632ad5daa0a6 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/preprocessing/waymo/preprocess_waymo.py @@ -0,0 +1,387 @@ +""" +Preprocessing Script for ScanNet 20/200 + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import warnings + +warnings.filterwarnings("ignore", category=DeprecationWarning) + +import os + +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + +import argparse +import numpy as np +import tensorflow.compat.v1 as tf +from pathlib import Path +from waymo_open_dataset.utils import frame_utils +from waymo_open_dataset.utils import transform_utils +from waymo_open_dataset.utils import range_image_utils +from waymo_open_dataset import dataset_pb2 as open_dataset +import glob +import multiprocessing as mp +from concurrent.futures import ProcessPoolExecutor +from itertools import repeat + + +def create_lidar(frame): + """Parse and save the lidar data in psd format. + Args: + frame (:obj:`Frame`): Open dataset frame proto. + """ + ( + range_images, + camera_projections, + segmentation_labels, + range_image_top_pose, + ) = frame_utils.parse_range_image_and_camera_projection(frame) + + points, cp_points, valid_masks = convert_range_image_to_point_cloud( + frame, + range_images, + camera_projections, + range_image_top_pose, + keep_polar_features=True, + ) + points_ri2, cp_points_ri2, valid_masks_ri2 = convert_range_image_to_point_cloud( + frame, + range_images, + camera_projections, + range_image_top_pose, + ri_index=1, + keep_polar_features=True, + ) + + # 3d points in vehicle frame. + points_all = np.concatenate(points, axis=0) + points_all_ri2 = np.concatenate(points_ri2, axis=0) + # point labels. + + points_all = np.concatenate([points_all, points_all_ri2], axis=0) + + velodyne = np.c_[points_all[:, 3:6], points_all[:, 1]] + velodyne = velodyne.reshape((velodyne.shape[0] * velodyne.shape[1])) + + valid_masks = [valid_masks, valid_masks_ri2] + return velodyne, valid_masks + + +def create_label(frame): + ( + range_images, + camera_projections, + segmentation_labels, + range_image_top_pose, + ) = frame_utils.parse_range_image_and_camera_projection(frame) + + point_labels = convert_range_image_to_point_cloud_labels( + frame, range_images, segmentation_labels + ) + point_labels_ri2 = convert_range_image_to_point_cloud_labels( + frame, range_images, segmentation_labels, ri_index=1 + ) + + # point labels. + point_labels_all = np.concatenate(point_labels, axis=0) + point_labels_all_ri2 = np.concatenate(point_labels_ri2, axis=0) + point_labels_all = np.concatenate([point_labels_all, point_labels_all_ri2], axis=0) + + labels = point_labels_all + return labels + + +def convert_range_image_to_cartesian( + frame, range_images, range_image_top_pose, ri_index=0, keep_polar_features=False +): + """Convert range images from polar coordinates to Cartesian coordinates. + + Args: + frame: open dataset frame + range_images: A dict of {laser_name, [range_image_first_return, + range_image_second_return]}. + range_image_top_pose: range image pixel pose for top lidar. + ri_index: 0 for the first return, 1 for the second return. + keep_polar_features: If true, keep the features from the polar range image + (i.e. range, intensity, and elongation) as the first features in the + output range image. + + Returns: + dict of {laser_name, (H, W, D)} range images in Cartesian coordinates. D + will be 3 if keep_polar_features is False (x, y, z) and 6 if + keep_polar_features is True (range, intensity, elongation, x, y, z). + """ + cartesian_range_images = {} + frame_pose = tf.convert_to_tensor( + value=np.reshape(np.array(frame.pose.transform), [4, 4]) + ) + + # [H, W, 6] + range_image_top_pose_tensor = tf.reshape( + tf.convert_to_tensor(value=range_image_top_pose.data), + range_image_top_pose.shape.dims, + ) + # [H, W, 3, 3] + range_image_top_pose_tensor_rotation = transform_utils.get_rotation_matrix( + range_image_top_pose_tensor[..., 0], + range_image_top_pose_tensor[..., 1], + range_image_top_pose_tensor[..., 2], + ) + range_image_top_pose_tensor_translation = range_image_top_pose_tensor[..., 3:] + range_image_top_pose_tensor = transform_utils.get_transform( + range_image_top_pose_tensor_rotation, range_image_top_pose_tensor_translation + ) + + for c in frame.context.laser_calibrations: + range_image = range_images[c.name][ri_index] + if len(c.beam_inclinations) == 0: # pylint: disable=g-explicit-length-test + beam_inclinations = range_image_utils.compute_inclination( + tf.constant([c.beam_inclination_min, c.beam_inclination_max]), + height=range_image.shape.dims[0], + ) + else: + beam_inclinations = tf.constant(c.beam_inclinations) + + beam_inclinations = tf.reverse(beam_inclinations, axis=[-1]) + extrinsic = np.reshape(np.array(c.extrinsic.transform), [4, 4]) + + range_image_tensor = tf.reshape( + tf.convert_to_tensor(value=range_image.data), range_image.shape.dims + ) + pixel_pose_local = None + frame_pose_local = None + if c.name == open_dataset.LaserName.TOP: + pixel_pose_local = range_image_top_pose_tensor + pixel_pose_local = tf.expand_dims(pixel_pose_local, axis=0) + frame_pose_local = tf.expand_dims(frame_pose, axis=0) + range_image_cartesian = range_image_utils.extract_point_cloud_from_range_image( + tf.expand_dims(range_image_tensor[..., 0], axis=0), + tf.expand_dims(extrinsic, axis=0), + tf.expand_dims(tf.convert_to_tensor(value=beam_inclinations), axis=0), + pixel_pose=pixel_pose_local, + frame_pose=frame_pose_local, + ) + + range_image_cartesian = tf.squeeze(range_image_cartesian, axis=0) + + if keep_polar_features: + # If we want to keep the polar coordinate features of range, intensity, + # and elongation, concatenate them to be the initial dimensions of the + # returned Cartesian range image. + range_image_cartesian = tf.concat( + [range_image_tensor[..., 0:3], range_image_cartesian], axis=-1 + ) + + cartesian_range_images[c.name] = range_image_cartesian + + return cartesian_range_images + + +def convert_range_image_to_point_cloud( + frame, + range_images, + camera_projections, + range_image_top_pose, + ri_index=0, + keep_polar_features=False, +): + """Convert range images to point cloud. + + Args: + frame: open dataset frame + range_images: A dict of {laser_name, [range_image_first_return, + range_image_second_return]}. + camera_projections: A dict of {laser_name, + [camera_projection_from_first_return, + camera_projection_from_second_return]}. + range_image_top_pose: range image pixel pose for top lidar. + ri_index: 0 for the first return, 1 for the second return. + keep_polar_features: If true, keep the features from the polar range image + (i.e. range, intensity, and elongation) as the first features in the + output range image. + + Returns: + points: {[N, 3]} list of 3d lidar points of length 5 (number of lidars). + (NOTE: Will be {[N, 6]} if keep_polar_features is true. + cp_points: {[N, 6]} list of camera projections of length 5 + (number of lidars). + """ + calibrations = sorted(frame.context.laser_calibrations, key=lambda c: c.name) + points = [] + cp_points = [] + valid_masks = [] + + cartesian_range_images = convert_range_image_to_cartesian( + frame, range_images, range_image_top_pose, ri_index, keep_polar_features + ) + + for c in calibrations: + range_image = range_images[c.name][ri_index] + range_image_tensor = tf.reshape( + tf.convert_to_tensor(value=range_image.data), range_image.shape.dims + ) + range_image_mask = range_image_tensor[..., 0] > 0 + + range_image_cartesian = cartesian_range_images[c.name] + points_tensor = tf.gather_nd( + range_image_cartesian, tf.compat.v1.where(range_image_mask) + ) + + cp = camera_projections[c.name][ri_index] + cp_tensor = tf.reshape(tf.convert_to_tensor(value=cp.data), cp.shape.dims) + cp_points_tensor = tf.gather_nd(cp_tensor, tf.compat.v1.where(range_image_mask)) + points.append(points_tensor.numpy()) + cp_points.append(cp_points_tensor.numpy()) + valid_masks.append(range_image_mask.numpy()) + + return points, cp_points, valid_masks + + +def convert_range_image_to_point_cloud_labels( + frame, range_images, segmentation_labels, ri_index=0 +): + """Convert segmentation labels from range images to point clouds. + + Args: + frame: open dataset frame + range_images: A dict of {laser_name, [range_image_first_return, + range_image_second_return]}. + segmentation_labels: A dict of {laser_name, [range_image_first_return, + range_image_second_return]}. + ri_index: 0 for the first return, 1 for the second return. + + Returns: + point_labels: {[N, 2]} list of 3d lidar points's segmentation labels. 0 for + points that are not labeled. + """ + calibrations = sorted(frame.context.laser_calibrations, key=lambda c: c.name) + point_labels = [] + for c in calibrations: + range_image = range_images[c.name][ri_index] + range_image_tensor = tf.reshape( + tf.convert_to_tensor(range_image.data), range_image.shape.dims + ) + range_image_mask = range_image_tensor[..., 0] > 0 + + if c.name in segmentation_labels: + sl = segmentation_labels[c.name][ri_index] + sl_tensor = tf.reshape(tf.convert_to_tensor(sl.data), sl.shape.dims) + sl_points_tensor = tf.gather_nd(sl_tensor, tf.where(range_image_mask)) + else: + num_valid_point = tf.math.reduce_sum(tf.cast(range_image_mask, tf.int32)) + sl_points_tensor = tf.zeros([num_valid_point, 2], dtype=tf.int32) + + point_labels.append(sl_points_tensor.numpy()) + return point_labels + + +def handle_process(file_path, output_root, test_frame_list): + file = os.path.basename(file_path) + split = os.path.basename(os.path.dirname(file_path)) + print(f"Parsing {split}/{file}") + save_path = Path(output_root) / split / file.split(".")[0] + + data_group = tf.data.TFRecordDataset(file_path, compression_type="") + for data in data_group: + frame = open_dataset.Frame() + frame.ParseFromString(bytearray(data.numpy())) + context_name = frame.context.name + timestamp = str(frame.timestamp_micros) + + if split != "testing": + # for training and validation frame, extract labelled frame + if not frame.lasers[0].ri_return1.segmentation_label_compressed: + continue + else: + # for testing frame, extract frame in test_frame_list + if f"{context_name},{timestamp}" not in test_frame_list: + continue + + os.makedirs(save_path / timestamp, exist_ok=True) + + # extract frame pass above check + point_cloud, valid_masks = create_lidar(frame) + point_cloud = point_cloud.reshape(-1, 4) + coord = point_cloud[:, :3] + strength = np.tanh(point_cloud[:, -1].reshape([-1, 1])) + pose = np.array(frame.pose.transform, np.float32).reshape(4, 4) + mask = np.array(valid_masks, dtype=object) + + np.save(save_path / timestamp / "coord.npy", coord) + np.save(save_path / timestamp / "strength.npy", strength) + np.save(save_path / timestamp / "pose.npy", pose) + + # save mask for reverse prediction + if split != "training": + np.save(save_path / timestamp / "mask.npy", mask) + + # save label + if split != "testing": + # ignore TYPE_UNDEFINED, ignore_index 0 -> -1 + label = create_label(frame)[:, 1].reshape([-1]) - 1 + np.save(save_path / timestamp / "segment.npy", label) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset_root", + required=True, + help="Path to the Waymo dataset", + ) + parser.add_argument( + "--output_root", + required=True, + help="Output path where train/val folders will be located", + ) + parser.add_argument( + "--splits", + required=True, + nargs="+", + choices=["training", "validation", "testing"], + help="Splits need to process ([training, validation, testing]).", + ) + parser.add_argument( + "--num_workers", + default=mp.cpu_count(), + type=int, + help="Num workers for preprocessing.", + ) + config = parser.parse_args() + + # load file list + file_list = glob.glob( + os.path.join(os.path.abspath(config.dataset_root), "*", "*.tfrecord") + ) + assert len(file_list) == 1150 + + # Create output directories + for split in config.splits: + os.makedirs(os.path.join(config.output_root, split), exist_ok=True) + + file_list = [ + file + for file in file_list + if os.path.basename(os.path.dirname(file)) in config.splits + ] + + # Load test frame list + test_frame_file = os.path.join( + os.path.dirname(__file__), "3d_semseg_test_set_frames.txt" + ) + test_frame_list = [x.rstrip() for x in (open(test_frame_file, "r").readlines())] + + # Preprocess data. + print("Processing scenes...") + pool = ProcessPoolExecutor(max_workers=config.num_workers) + _ = list( + pool.map( + handle_process, + file_list, + repeat(config.output_root), + repeat(test_frame_list), + ) + ) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/s3dis.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/s3dis.py new file mode 100644 index 0000000000000000000000000000000000000000..1cfc176380caaa42d35bbf517ae2f13bf5c86f90 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/s3dis.py @@ -0,0 +1,18 @@ +""" +S3DIS Dataset + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +from .defaults import DefaultDataset +from .builder import DATASETS + + +@DATASETS.register_module() +class S3DISDataset(DefaultDataset): + def get_data_name(self, idx): + remain, room_name = os.path.split(self.data_list[idx % len(self.data_list)]) + remain, area_name = os.path.split(remain) + return f"{area_name}-{room_name}" diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/scannet.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/scannet.py new file mode 100644 index 0000000000000000000000000000000000000000..d3f3ab8832e37d64c96cdf836741a4f0fc3ad9ff --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/scannet.py @@ -0,0 +1,116 @@ +""" +ScanNet20 / ScanNet200 / ScanNet Data Efficient Dataset + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import glob +import numpy as np +import torch +from copy import deepcopy +from torch.utils.data import Dataset +from collections.abc import Sequence + +from pointcept.utils.logger import get_root_logger +from pointcept.utils.cache import shared_dict +from .builder import DATASETS +from .defaults import DefaultDataset +from .transform import Compose, TRANSFORMS +from .preprocessing.scannet.meta_data.scannet200_constants import ( + VALID_CLASS_IDS_20, + VALID_CLASS_IDS_200, +) + + +@DATASETS.register_module() +class ScanNetDataset(DefaultDataset): + VALID_ASSETS = [ + "coord", + "color", + "normal", + "segment20", + "instance", + ] + class2id = np.array(VALID_CLASS_IDS_20) + + def __init__( + self, + lr_file=None, + la_file=None, + **kwargs, + ): + self.lr = np.loadtxt(lr_file, dtype=str) if lr_file is not None else None + self.la = torch.load(la_file) if la_file is not None else None + super().__init__(**kwargs) + + def get_data_list(self): + if self.lr is None: + data_list = super().get_data_list() + else: + data_list = [ + os.path.join(self.data_root, "train", name) for name in self.lr + ] + return data_list + + def get_data(self, idx): + data_path = self.data_list[idx % len(self.data_list)] + name = self.get_data_name(idx) + if self.cache: + cache_name = f"pointcept-{name}" + return shared_dict(cache_name) + + data_dict = {} + assets = os.listdir(data_path) + for asset in assets: + if not asset.endswith(".npy"): + continue + if asset[:-4] not in self.VALID_ASSETS: + continue + data_dict[asset[:-4]] = np.load(os.path.join(data_path, asset)) + data_dict["name"] = name + data_dict["coord"] = data_dict["coord"].astype(np.float32) + data_dict["color"] = data_dict["color"].astype(np.float32) + data_dict["normal"] = data_dict["normal"].astype(np.float32) + + if "segment20" in data_dict.keys(): + data_dict["segment"] = ( + data_dict.pop("segment20").reshape([-1]).astype(np.int32) + ) + elif "segment200" in data_dict.keys(): + data_dict["segment"] = ( + data_dict.pop("segment200").reshape([-1]).astype(np.int32) + ) + else: + data_dict["segment"] = ( + np.ones(data_dict["coord"].shape[0], dtype=np.int32) * -1 + ) + + if "instance" in data_dict.keys(): + data_dict["instance"] = ( + data_dict.pop("instance").reshape([-1]).astype(np.int32) + ) + else: + data_dict["instance"] = ( + np.ones(data_dict["coord"].shape[0], dtype=np.int32) * -1 + ) + if self.la: + sampled_index = self.la[self.get_data_name(idx)] + mask = np.ones_like(data_dict["segment"], dtype=bool) + mask[sampled_index] = False + data_dict["segment"][mask] = self.ignore_index + data_dict["sampled_index"] = sampled_index + return data_dict + + +@DATASETS.register_module() +class ScanNet200Dataset(ScanNetDataset): + VALID_ASSETS = [ + "coord", + "color", + "normal", + "segment200", + "instance", + ] + class2id = np.array(VALID_CLASS_IDS_200) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/scannet_pair.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/scannet_pair.py new file mode 100644 index 0000000000000000000000000000000000000000..b7fdf199aa90b113bb4e8c7643e5549f62d0c51a --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/scannet_pair.py @@ -0,0 +1,89 @@ +""" +ScanNet Pair Dataset (Frame-level contrastive view) + +Refer PointContrast + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import glob +import numpy as np +import torch +from copy import deepcopy +from torch.utils.data import Dataset + +from pointcept.utils.logger import get_root_logger +from .builder import DATASETS +from .transform import Compose, TRANSFORMS + + +@DATASETS.register_module() +class ScanNetPairDataset(Dataset): + def __init__( + self, + data_root="data/scannet_pair", + overlap_threshold=0.3, + view1_transform=None, + view2_transform=None, + loop=1, + **kwargs + ): + super(ScanNetPairDataset, self).__init__() + self.data_root = data_root + self.overlap_threshold = overlap_threshold + self.view1_transform = Compose(view1_transform) + self.view2_transform = Compose(view2_transform) + self.loop = loop + self.data_list = self.get_data_list() + logger = get_root_logger() + logger.info("Totally {} x {} samples.".format(len(self.data_list), self.loop)) + + def get_data_list(self): + data_list = [] + overlap_list = glob.glob( + os.path.join(self.data_root, "*", "pcd", "overlap.txt") + ) + for overlap_file in overlap_list: + with open(overlap_file) as f: + overlap = f.readlines() + overlap = [pair.strip().split() for pair in overlap] + data_list.extend( + [ + pair[:2] + for pair in overlap + if float(pair[2]) > self.overlap_threshold + ] + ) + return data_list + + def get_data(self, idx): + pair = self.data_list[idx % len(self.data_list)] + view1_dict = torch.load(self.data_root + pair[0]) + view2_dict = torch.load(self.data_root + pair[1]) + return view1_dict, view2_dict + + def get_data_name(self, idx): + return os.path.basename(self.data_list[idx % len(self.data_list)]).split(".")[0] + + def prepare_train_data(self, idx): + # load data + view1_dict, view2_dict = self.get_data(idx) + view1_dict = self.view1_transform(view1_dict) + view2_dict = self.view2_transform(view2_dict) + data_dict = dict() + for key, value in view1_dict.items(): + data_dict["view1_" + key] = value + for key, value in view2_dict.items(): + data_dict["view2_" + key] = value + return data_dict + + def prepare_test_data(self, idx): + raise NotImplementedError + + def __getitem__(self, idx): + return self.prepare_train_data(idx) + + def __len__(self): + return len(self.data_list) * self.loop diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/scannetpp.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/scannetpp.py new file mode 100644 index 0000000000000000000000000000000000000000..5954e484081b32f7c6408e5efcf08f1af7449917 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/scannetpp.py @@ -0,0 +1,78 @@ +""" +ScanNet++ dataset + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import numpy as np +import glob + +from pointcept.utils.cache import shared_dict + +from .builder import DATASETS +from .defaults import DefaultDataset + + +@DATASETS.register_module() +class ScanNetPPDataset(DefaultDataset): + VALID_ASSETS = [ + "coord", + "color", + "normal", + "segment", + "instance", + ] + + def __init__( + self, + multilabel=False, + **kwargs, + ): + super().__init__(**kwargs) + self.multilabel = multilabel + + def get_data(self, idx): + data_path = self.data_list[idx % len(self.data_list)] + name = self.get_data_name(idx) + if self.cache: + cache_name = f"pointcept-{name}" + return shared_dict(cache_name) + + data_dict = {} + assets = os.listdir(data_path) + for asset in assets: + if not asset.endswith(".npy"): + continue + if asset[:-4] not in self.VALID_ASSETS: + continue + data_dict[asset[:-4]] = np.load(os.path.join(data_path, asset)) + data_dict["name"] = name + + if "coord" in data_dict.keys(): + data_dict["coord"] = data_dict["coord"].astype(np.float32) + + if "color" in data_dict.keys(): + data_dict["color"] = data_dict["color"].astype(np.float32) + + if "normal" in data_dict.keys(): + data_dict["normal"] = data_dict["normal"].astype(np.float32) + + if not self.multilabel: + if "segment" in data_dict.keys(): + data_dict["segment"] = data_dict["segment"][:, 0].astype(np.int32) + else: + data_dict["segment"] = ( + np.ones(data_dict["coord"].shape[0], dtype=np.int32) * -1 + ) + + if "instance" in data_dict.keys(): + data_dict["instance"] = data_dict["instance"][:, 0].astype(np.int32) + else: + data_dict["instance"] = ( + np.ones(data_dict["coord"].shape[0], dtype=np.int32) * -1 + ) + else: + raise NotImplementedError + return data_dict diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/semantic_kitti.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/semantic_kitti.py new file mode 100644 index 0000000000000000000000000000000000000000..cd1354f6043792bf50d367c63168b6fd0455038b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/semantic_kitti.py @@ -0,0 +1,144 @@ +""" +Semantic KITTI dataset + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import numpy as np + +from .builder import DATASETS +from .defaults import DefaultDataset + + +@DATASETS.register_module() +class SemanticKITTIDataset(DefaultDataset): + def __init__(self, ignore_index=-1, **kwargs): + self.ignore_index = ignore_index + self.learning_map = self.get_learning_map(ignore_index) + self.learning_map_inv = self.get_learning_map_inv(ignore_index) + super().__init__(ignore_index=ignore_index, **kwargs) + + def get_data_list(self): + split2seq = dict( + train=[0, 1, 2, 3, 4, 5, 6, 7, 9, 10], + val=[8], + test=[11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21], + ) + if isinstance(self.split, str): + seq_list = split2seq[self.split] + elif isinstance(self.split, list): + seq_list = [] + for split in self.split: + seq_list += split2seq[split] + else: + raise NotImplementedError + + data_list = [] + for seq in seq_list: + seq = str(seq).zfill(2) + seq_folder = os.path.join(self.data_root, "dataset", "sequences", seq) + seq_files = sorted(os.listdir(os.path.join(seq_folder, "velodyne"))) + data_list += [ + os.path.join(seq_folder, "velodyne", file) for file in seq_files + ] + return data_list + + def get_data(self, idx): + data_path = self.data_list[idx % len(self.data_list)] + with open(data_path, "rb") as b: + scan = np.fromfile(b, dtype=np.float32).reshape(-1, 4) + coord = scan[:, :3] + strength = scan[:, -1].reshape([-1, 1]) + + label_file = data_path.replace("velodyne", "labels").replace(".bin", ".label") + if os.path.exists(label_file): + with open(label_file, "rb") as a: + segment = np.fromfile(a, dtype=np.int32).reshape(-1) + segment = np.vectorize(self.learning_map.__getitem__)( + segment & 0xFFFF + ).astype(np.int32) + else: + segment = np.zeros(scan.shape[0]).astype(np.int32) + data_dict = dict( + coord=coord, + strength=strength, + segment=segment, + name=self.get_data_name(idx), + ) + return data_dict + + def get_data_name(self, idx): + file_path = self.data_list[idx % len(self.data_list)] + dir_path, file_name = os.path.split(file_path) + sequence_name = os.path.basename(os.path.dirname(dir_path)) + frame_name = os.path.splitext(file_name)[0] + data_name = f"{sequence_name}_{frame_name}" + return data_name + + @staticmethod + def get_learning_map(ignore_index): + learning_map = { + 0: ignore_index, # "unlabeled" + 1: ignore_index, # "outlier" mapped to "unlabeled" --------------------------mapped + 10: 0, # "car" + 11: 1, # "bicycle" + 13: 4, # "bus" mapped to "other-vehicle" --------------------------mapped + 15: 2, # "motorcycle" + 16: 4, # "on-rails" mapped to "other-vehicle" ---------------------mapped + 18: 3, # "truck" + 20: 4, # "other-vehicle" + 30: 5, # "person" + 31: 6, # "bicyclist" + 32: 7, # "motorcyclist" + 40: 8, # "road" + 44: 9, # "parking" + 48: 10, # "sidewalk" + 49: 11, # "other-ground" + 50: 12, # "building" + 51: 13, # "fence" + 52: ignore_index, # "other-structure" mapped to "unlabeled" ------------------mapped + 60: 8, # "lane-marking" to "road" ---------------------------------mapped + 70: 14, # "vegetation" + 71: 15, # "trunk" + 72: 16, # "terrain" + 80: 17, # "pole" + 81: 18, # "traffic-sign" + 99: ignore_index, # "other-object" to "unlabeled" ----------------------------mapped + 252: 0, # "moving-car" to "car" ------------------------------------mapped + 253: 6, # "moving-bicyclist" to "bicyclist" ------------------------mapped + 254: 5, # "moving-person" to "person" ------------------------------mapped + 255: 7, # "moving-motorcyclist" to "motorcyclist" ------------------mapped + 256: 4, # "moving-on-rails" mapped to "other-vehicle" --------------mapped + 257: 4, # "moving-bus" mapped to "other-vehicle" -------------------mapped + 258: 3, # "moving-truck" to "truck" --------------------------------mapped + 259: 4, # "moving-other"-vehicle to "other-vehicle" ----------------mapped + } + return learning_map + + @staticmethod + def get_learning_map_inv(ignore_index): + learning_map_inv = { + ignore_index: ignore_index, # "unlabeled" + 0: 10, # "car" + 1: 11, # "bicycle" + 2: 15, # "motorcycle" + 3: 18, # "truck" + 4: 20, # "other-vehicle" + 5: 30, # "person" + 6: 31, # "bicyclist" + 7: 32, # "motorcyclist" + 8: 40, # "road" + 9: 44, # "parking" + 10: 48, # "sidewalk" + 11: 49, # "other-ground" + 12: 50, # "building" + 13: 51, # "fence" + 14: 70, # "vegetation" + 15: 71, # "trunk" + 16: 72, # "terrain" + 17: 80, # "pole" + 18: 81, # "traffic-sign" + } + return learning_map_inv diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/shapenet_part.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/shapenet_part.py new file mode 100644 index 0000000000000000000000000000000000000000..ca0c3d50adf98b643233e28b56631654479fde60 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/shapenet_part.py @@ -0,0 +1,160 @@ +""" +ShapeNet Part Dataset (Unmaintained) + +get processed shapenet part dataset +at "https://shapenet.cs.stanford.edu/media/shapenetcore_partanno_segmentation_benchmark_v0_normal.zip" + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import json +import torch +import numpy as np +from copy import deepcopy +from torch.utils.data import Dataset + +from pointcept.utils.logger import get_root_logger +from .builder import DATASETS +from .transform import Compose + + +@DATASETS.register_module() +class ShapeNetPartDataset(Dataset): + def __init__( + self, + split="train", + data_root="data/shapenetcore_partanno_segmentation_benchmark_v0_normal", + transform=None, + test_mode=False, + test_cfg=None, + loop=1, + ): + super(ShapeNetPartDataset, self).__init__() + self.data_root = data_root + self.split = split + self.transform = Compose(transform) + self.loop = ( + loop if not test_mode else 1 + ) # force make loop = 1 while in test mode + self.test_mode = test_mode + self.test_cfg = test_cfg if test_mode else None + self.cache = {} + + # load categories file + self.categories = [] + self.category2part = { + "Airplane": [0, 1, 2, 3], + "Bag": [4, 5], + "Cap": [6, 7], + "Car": [8, 9, 10, 11], + "Chair": [12, 13, 14, 15], + "Earphone": [16, 17, 18], + "Guitar": [19, 20, 21], + "Knife": [22, 23], + "Lamp": [24, 25, 26, 27], + "Laptop": [28, 29], + "Motorbike": [30, 31, 32, 33, 34, 35], + "Mug": [36, 37], + "Pistol": [38, 39, 40], + "Rocket": [41, 42, 43], + "Skateboard": [44, 45, 46], + "Table": [47, 48, 49], + } + self.token2category = {} + with open(os.path.join(self.data_root, "synsetoffset2category.txt"), "r") as f: + for line in f: + ls = line.strip().split() + self.token2category[ls[1]] = len(self.categories) + self.categories.append(ls[0]) + + if test_mode: + self.post_transform = Compose(self.test_cfg.post_transform) + self.aug_transform = [Compose(aug) for aug in self.test_cfg.aug_transform] + + # load data list + if isinstance(self.split, str): + self.data_list = self.load_data_list(self.split) + elif isinstance(self.split, list): + self.data_list = [] + for s in self.split: + self.data_list += self.load_data_list(s) + else: + raise NotImplementedError + + logger = get_root_logger() + logger.info( + "Totally {} x {} samples in {} set.".format( + len(self.data_idx), self.loop, split + ) + ) + + def load_data_list(self, split): + split_file = os.path.join( + self.data_root, + "train_test_split", + "shuffled_{}_file_list.json".format(split), + ) + if not os.path.isfile(split_file): + raise (RuntimeError("Split file do not exist: " + split_file + "\n")) + with open(split_file, "r") as f: + # drop "shape_data/" and append ".txt" + data_list = [ + os.path.join(self.data_root, data[11:] + ".txt") + for data in json.load(f) + ] + return data_list + + def prepare_train_data(self, idx): + # load data + data_idx = idx % len(self.data_list) + if data_idx in self.cache: + coord, norm, segment, cls_token = self.cache[data_idx] + else: + data = np.loadtxt(self.data_list[data_idx]).astype(np.float32) + cls_token = self.token2category[ + os.path.basename(os.path.dirname(self.data_list[data_idx])) + ] + coord, norm, segment = ( + data[:, :3], + data[:, 3:6], + data[:, 6].astype(np.int32), + ) + self.cache[data_idx] = (coord, norm, segment, cls_token) + + data_dict = dict(coord=coord, norm=norm, segment=segment, cls_token=cls_token) + data_dict = self.transform(data_dict) + return data_dict + + def prepare_test_data(self, idx): + # load data + data_idx = self.data_idx[idx % len(self.data_idx)] + data = np.loadtxt(self.data_list[data_idx]).astype(np.float32) + cls_token = self.token2category[ + os.path.basename(os.path.dirname(self.data_list[data_idx])) + ] + coord, norm, segment = data[:, :3], data[:, 3:6], data[:, 6].astype(np.int32) + + data_dict = dict(coord=coord, norm=norm, cls_token=cls_token) + data_dict = self.transform(data_dict) + data_dict_list = [] + for aug in self.aug_transform: + data_dict_list.append(self.post_transform(aug(deepcopy(data_dict)))) + data_dict = dict( + fragment_list=data_dict_list, segment=segment, name=self.get_data_name(idx) + ) + return data_dict + + def get_data_name(self, idx): + data_idx = self.data_idx[idx % len(self.data_idx)] + return os.path.basename(self.data_list[data_idx]).split(".")[0] + + def __getitem__(self, idx): + if self.test_mode: + return self.prepare_test_data(idx) + else: + return self.prepare_train_data(idx) + + def __len__(self): + return len(self.data_idx) * self.loop diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/structure3d.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/structure3d.py new file mode 100644 index 0000000000000000000000000000000000000000..81649e3d83275f0419546f37da810e7508b327e0 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/structure3d.py @@ -0,0 +1,38 @@ +""" +Structured3D Datasets + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import glob +from collections.abc import Sequence + +from .defaults import DefaultDataset +from .builder import DATASETS + + +@DATASETS.register_module() +class Structured3DDataset(DefaultDataset): + def get_data_list(self): + if isinstance(self.split, str): + data_list = glob.glob( + os.path.join(self.data_root, self.split, "scene_*/room_*") + ) + elif isinstance(self.split, Sequence): + data_list = [] + for split in self.split: + data_list += glob.glob( + os.path.join(self.data_root, split, "scene_*/room_*") + ) + else: + raise NotImplementedError + return data_list + + def get_data_name(self, idx): + file_path = self.data_list[idx % len(self.data_list)] + dir_path, room_name = os.path.split(file_path) + scene_name = os.path.basename(dir_path) + data_name = f"{scene_name}_{room_name}" + return data_name diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/transform.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/transform.py new file mode 100644 index 0000000000000000000000000000000000000000..d1abfe4f3d88decdac17861b31647c34f9520292 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/transform.py @@ -0,0 +1,1148 @@ +""" +3D Point Cloud Augmentation + +Inspirited by chrischoy/SpatioTemporalSegmentation + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import random +import numbers +import scipy +import scipy.ndimage +import scipy.interpolate +import scipy.stats +import numpy as np +import torch +import copy +from collections.abc import Sequence, Mapping + +from pointcept.utils.registry import Registry + +TRANSFORMS = Registry("transforms") + + +@TRANSFORMS.register_module() +class Collect(object): + def __init__(self, keys, offset_keys_dict=None, **kwargs): + """ + e.g. Collect(keys=[coord], feat_keys=[coord, color]) + """ + if offset_keys_dict is None: + offset_keys_dict = dict(offset="coord") + self.keys = keys + self.offset_keys = offset_keys_dict + self.kwargs = kwargs + + def __call__(self, data_dict): + data = dict() + if isinstance(self.keys, str): + self.keys = [self.keys] + for key in self.keys: + data[key] = data_dict[key] + for key, value in self.offset_keys.items(): + data[key] = torch.tensor([data_dict[value].shape[0]]) + for name, keys in self.kwargs.items(): + name = name.replace("_keys", "") + assert isinstance(keys, Sequence) + data[name] = torch.cat([data_dict[key].float() for key in keys], dim=1) + return data + + +@TRANSFORMS.register_module() +class Copy(object): + def __init__(self, keys_dict=None): + if keys_dict is None: + keys_dict = dict(coord="origin_coord", segment="origin_segment") + self.keys_dict = keys_dict + + def __call__(self, data_dict): + for key, value in self.keys_dict.items(): + if isinstance(data_dict[key], np.ndarray): + data_dict[value] = data_dict[key].copy() + elif isinstance(data_dict[key], torch.Tensor): + data_dict[value] = data_dict[key].clone().detach() + else: + data_dict[value] = copy.deepcopy(data_dict[key]) + return data_dict + + +@TRANSFORMS.register_module() +class ToTensor(object): + def __call__(self, data): + if isinstance(data, torch.Tensor): + return data + elif isinstance(data, str): + # note that str is also a kind of sequence, judgement should before sequence + return data + elif isinstance(data, int): + return torch.LongTensor([data]) + elif isinstance(data, float): + return torch.FloatTensor([data]) + elif isinstance(data, np.ndarray) and np.issubdtype(data.dtype, bool): + return torch.from_numpy(data) + elif isinstance(data, np.ndarray) and np.issubdtype(data.dtype, np.integer): + return torch.from_numpy(data).long() + elif isinstance(data, np.ndarray) and np.issubdtype(data.dtype, np.floating): + return torch.from_numpy(data).float() + elif isinstance(data, Mapping): + result = {sub_key: self(item) for sub_key, item in data.items()} + return result + elif isinstance(data, Sequence): + result = [self(item) for item in data] + return result + else: + raise TypeError(f"type {type(data)} cannot be converted to tensor.") + + +@TRANSFORMS.register_module() +class Add(object): + def __init__(self, keys_dict=None): + if keys_dict is None: + keys_dict = dict() + self.keys_dict = keys_dict + + def __call__(self, data_dict): + for key, value in self.keys_dict.items(): + data_dict[key] = value + return data_dict + + +@TRANSFORMS.register_module() +class NormalizeColor(object): + def __call__(self, data_dict): + if "color" in data_dict.keys(): + data_dict["color"] = data_dict["color"] / 127.5 - 1 + return data_dict + + +@TRANSFORMS.register_module() +class NormalizeCoord(object): + def __call__(self, data_dict): + if "coord" in data_dict.keys(): + # modified from pointnet2 + centroid = np.mean(data_dict["coord"], axis=0) + data_dict["coord"] -= centroid + m = np.max(np.sqrt(np.sum(data_dict["coord"] ** 2, axis=1))) + data_dict["coord"] = data_dict["coord"] / m + return data_dict + + +@TRANSFORMS.register_module() +class PositiveShift(object): + def __call__(self, data_dict): + if "coord" in data_dict.keys(): + coord_min = np.min(data_dict["coord"], 0) + data_dict["coord"] -= coord_min + return data_dict + + +@TRANSFORMS.register_module() +class CenterShift(object): + def __init__(self, apply_z=True): + self.apply_z = apply_z + + def __call__(self, data_dict): + if "coord" in data_dict.keys(): + x_min, y_min, z_min = data_dict["coord"].min(axis=0) + x_max, y_max, _ = data_dict["coord"].max(axis=0) + if self.apply_z: + shift = [(x_min + x_max) / 2, (y_min + y_max) / 2, z_min] + else: + shift = [(x_min + x_max) / 2, (y_min + y_max) / 2, 0] + data_dict["coord"] -= shift + return data_dict + + +@TRANSFORMS.register_module() +class RandomShift(object): + def __init__(self, shift=((-0.2, 0.2), (-0.2, 0.2), (0, 0))): + self.shift = shift + + def __call__(self, data_dict): + if "coord" in data_dict.keys(): + shift_x = np.random.uniform(self.shift[0][0], self.shift[0][1]) + shift_y = np.random.uniform(self.shift[1][0], self.shift[1][1]) + shift_z = np.random.uniform(self.shift[2][0], self.shift[2][1]) + data_dict["coord"] += [shift_x, shift_y, shift_z] + return data_dict + + +@TRANSFORMS.register_module() +class PointClip(object): + def __init__(self, point_cloud_range=(-80, -80, -3, 80, 80, 1)): + self.point_cloud_range = point_cloud_range + + def __call__(self, data_dict): + if "coord" in data_dict.keys(): + data_dict["coord"] = np.clip( + data_dict["coord"], + a_min=self.point_cloud_range[:3], + a_max=self.point_cloud_range[3:], + ) + return data_dict + + +@TRANSFORMS.register_module() +class RandomDropout(object): + def __init__(self, dropout_ratio=0.2, dropout_application_ratio=0.5): + """ + upright_axis: axis index among x,y,z, i.e. 2 for z + """ + self.dropout_ratio = dropout_ratio + self.dropout_application_ratio = dropout_application_ratio + + def __call__(self, data_dict): + if random.random() < self.dropout_application_ratio: + n = len(data_dict["coord"]) + idx = np.random.choice(n, int(n * (1 - self.dropout_ratio)), replace=False) + if "sampled_index" in data_dict: + # for ScanNet data efficient, we need to make sure labeled point is sampled. + idx = np.unique(np.append(idx, data_dict["sampled_index"])) + mask = np.zeros_like(data_dict["segment"]).astype(bool) + mask[data_dict["sampled_index"]] = True + data_dict["sampled_index"] = np.where(mask[idx])[0] + if "coord" in data_dict.keys(): + data_dict["coord"] = data_dict["coord"][idx] + if "color" in data_dict.keys(): + data_dict["color"] = data_dict["color"][idx] + if "normal" in data_dict.keys(): + data_dict["normal"] = data_dict["normal"][idx] + if "strength" in data_dict.keys(): + data_dict["strength"] = data_dict["strength"][idx] + if "segment" in data_dict.keys(): + data_dict["segment"] = data_dict["segment"][idx] + if "instance" in data_dict.keys(): + data_dict["instance"] = data_dict["instance"][idx] + return data_dict + + +@TRANSFORMS.register_module() +class RandomRotate(object): + def __init__(self, angle=None, center=None, axis="z", always_apply=False, p=0.5): + self.angle = [-1, 1] if angle is None else angle + self.axis = axis + self.always_apply = always_apply + self.p = p if not self.always_apply else 1 + self.center = center + + def __call__(self, data_dict): + if random.random() > self.p: + return data_dict + angle = np.random.uniform(self.angle[0], self.angle[1]) * np.pi + rot_cos, rot_sin = np.cos(angle), np.sin(angle) + if self.axis == "x": + rot_t = np.array([[1, 0, 0], [0, rot_cos, -rot_sin], [0, rot_sin, rot_cos]]) + elif self.axis == "y": + rot_t = np.array([[rot_cos, 0, rot_sin], [0, 1, 0], [-rot_sin, 0, rot_cos]]) + elif self.axis == "z": + rot_t = np.array([[rot_cos, -rot_sin, 0], [rot_sin, rot_cos, 0], [0, 0, 1]]) + else: + raise NotImplementedError + if "coord" in data_dict.keys(): + if self.center is None: + x_min, y_min, z_min = data_dict["coord"].min(axis=0) + x_max, y_max, z_max = data_dict["coord"].max(axis=0) + center = [(x_min + x_max) / 2, (y_min + y_max) / 2, (z_min + z_max) / 2] + else: + center = self.center + data_dict["coord"] -= center + data_dict["coord"] = np.dot(data_dict["coord"], np.transpose(rot_t)) + data_dict["coord"] += center + if "normal" in data_dict.keys(): + data_dict["normal"] = np.dot(data_dict["normal"], np.transpose(rot_t)) + return data_dict + + +@TRANSFORMS.register_module() +class RandomRotateTargetAngle(object): + def __init__( + self, angle=(1 / 2, 1, 3 / 2), center=None, axis="z", always_apply=False, p=0.75 + ): + self.angle = angle + self.axis = axis + self.always_apply = always_apply + self.p = p if not self.always_apply else 1 + self.center = center + + def __call__(self, data_dict): + if random.random() > self.p: + return data_dict + angle = np.random.choice(self.angle) * np.pi + rot_cos, rot_sin = np.cos(angle), np.sin(angle) + if self.axis == "x": + rot_t = np.array([[1, 0, 0], [0, rot_cos, -rot_sin], [0, rot_sin, rot_cos]]) + elif self.axis == "y": + rot_t = np.array([[rot_cos, 0, rot_sin], [0, 1, 0], [-rot_sin, 0, rot_cos]]) + elif self.axis == "z": + rot_t = np.array([[rot_cos, -rot_sin, 0], [rot_sin, rot_cos, 0], [0, 0, 1]]) + else: + raise NotImplementedError + if "coord" in data_dict.keys(): + if self.center is None: + x_min, y_min, z_min = data_dict["coord"].min(axis=0) + x_max, y_max, z_max = data_dict["coord"].max(axis=0) + center = [(x_min + x_max) / 2, (y_min + y_max) / 2, (z_min + z_max) / 2] + else: + center = self.center + data_dict["coord"] -= center + data_dict["coord"] = np.dot(data_dict["coord"], np.transpose(rot_t)) + data_dict["coord"] += center + if "normal" in data_dict.keys(): + data_dict["normal"] = np.dot(data_dict["normal"], np.transpose(rot_t)) + return data_dict + + +@TRANSFORMS.register_module() +class RandomScale(object): + def __init__(self, scale=None, anisotropic=False): + self.scale = scale if scale is not None else [0.95, 1.05] + self.anisotropic = anisotropic + + def __call__(self, data_dict): + if "coord" in data_dict.keys(): + scale = np.random.uniform( + self.scale[0], self.scale[1], 3 if self.anisotropic else 1 + ) + data_dict["coord"] *= scale + return data_dict + + +@TRANSFORMS.register_module() +class RandomFlip(object): + def __init__(self, p=0.5): + self.p = p + + def __call__(self, data_dict): + if np.random.rand() < self.p: + if "coord" in data_dict.keys(): + data_dict["coord"][:, 0] = -data_dict["coord"][:, 0] + if "normal" in data_dict.keys(): + data_dict["normal"][:, 0] = -data_dict["normal"][:, 0] + if np.random.rand() < self.p: + if "coord" in data_dict.keys(): + data_dict["coord"][:, 1] = -data_dict["coord"][:, 1] + if "normal" in data_dict.keys(): + data_dict["normal"][:, 1] = -data_dict["normal"][:, 1] + return data_dict + + +@TRANSFORMS.register_module() +class RandomJitter(object): + def __init__(self, sigma=0.01, clip=0.05): + assert clip > 0 + self.sigma = sigma + self.clip = clip + + def __call__(self, data_dict): + if "coord" in data_dict.keys(): + jitter = np.clip( + self.sigma * np.random.randn(data_dict["coord"].shape[0], 3), + -self.clip, + self.clip, + ) + data_dict["coord"] += jitter + return data_dict + + +@TRANSFORMS.register_module() +class ClipGaussianJitter(object): + def __init__(self, scalar=0.02, store_jitter=False): + self.scalar = scalar + self.mean = np.mean(3) + self.cov = np.identity(3) + self.quantile = 1.96 + self.store_jitter = store_jitter + + def __call__(self, data_dict): + if "coord" in data_dict.keys(): + jitter = np.random.multivariate_normal( + self.mean, self.cov, data_dict["coord"].shape[0] + ) + jitter = self.scalar * np.clip(jitter / 1.96, -1, 1) + data_dict["coord"] += jitter + if self.store_jitter: + data_dict["jitter"] = jitter + return data_dict + + +@TRANSFORMS.register_module() +class ChromaticAutoContrast(object): + def __init__(self, p=0.2, blend_factor=None): + self.p = p + self.blend_factor = blend_factor + + def __call__(self, data_dict): + if "color" in data_dict.keys() and np.random.rand() < self.p: + lo = np.min(data_dict["color"], 0, keepdims=True) + hi = np.max(data_dict["color"], 0, keepdims=True) + scale = 255 / (hi - lo) + contrast_feat = (data_dict["color"][:, :3] - lo) * scale + blend_factor = ( + np.random.rand() if self.blend_factor is None else self.blend_factor + ) + data_dict["color"][:, :3] = (1 - blend_factor) * data_dict["color"][ + :, :3 + ] + blend_factor * contrast_feat + return data_dict + + +@TRANSFORMS.register_module() +class ChromaticTranslation(object): + def __init__(self, p=0.95, ratio=0.05): + self.p = p + self.ratio = ratio + + def __call__(self, data_dict): + if "color" in data_dict.keys() and np.random.rand() < self.p: + tr = (np.random.rand(1, 3) - 0.5) * 255 * 2 * self.ratio + data_dict["color"][:, :3] = np.clip(tr + data_dict["color"][:, :3], 0, 255) + return data_dict + + +@TRANSFORMS.register_module() +class ChromaticJitter(object): + def __init__(self, p=0.95, std=0.005): + self.p = p + self.std = std + + def __call__(self, data_dict): + if "color" in data_dict.keys() and np.random.rand() < self.p: + noise = np.random.randn(data_dict["color"].shape[0], 3) + noise *= self.std * 255 + data_dict["color"][:, :3] = np.clip( + noise + data_dict["color"][:, :3], 0, 255 + ) + return data_dict + + +@TRANSFORMS.register_module() +class RandomColorGrayScale(object): + def __init__(self, p): + self.p = p + + @staticmethod + def rgb_to_grayscale(color, num_output_channels=1): + if color.shape[-1] < 3: + raise TypeError( + "Input color should have at least 3 dimensions, but found {}".format( + color.shape[-1] + ) + ) + + if num_output_channels not in (1, 3): + raise ValueError("num_output_channels should be either 1 or 3") + + r, g, b = color[..., 0], color[..., 1], color[..., 2] + gray = (0.2989 * r + 0.587 * g + 0.114 * b).astype(color.dtype) + gray = np.expand_dims(gray, axis=-1) + + if num_output_channels == 3: + gray = np.broadcast_to(gray, color.shape) + + return gray + + def __call__(self, data_dict): + if np.random.rand() < self.p: + data_dict["color"] = self.rgb_to_grayscale(data_dict["color"], 3) + return data_dict + + +@TRANSFORMS.register_module() +class RandomColorJitter(object): + """ + Random Color Jitter for 3D point cloud (refer torchvision) + """ + + def __init__(self, brightness=0, contrast=0, saturation=0, hue=0, p=0.95): + self.brightness = self._check_input(brightness, "brightness") + self.contrast = self._check_input(contrast, "contrast") + self.saturation = self._check_input(saturation, "saturation") + self.hue = self._check_input( + hue, "hue", center=0, bound=(-0.5, 0.5), clip_first_on_zero=False + ) + self.p = p + + @staticmethod + def _check_input( + value, name, center=1, bound=(0, float("inf")), clip_first_on_zero=True + ): + if isinstance(value, numbers.Number): + if value < 0: + raise ValueError( + "If {} is a single number, it must be non negative.".format(name) + ) + value = [center - float(value), center + float(value)] + if clip_first_on_zero: + value[0] = max(value[0], 0.0) + elif isinstance(value, (tuple, list)) and len(value) == 2: + if not bound[0] <= value[0] <= value[1] <= bound[1]: + raise ValueError("{} values should be between {}".format(name, bound)) + else: + raise TypeError( + "{} should be a single number or a list/tuple with length 2.".format( + name + ) + ) + + # if value is 0 or (1., 1.) for brightness/contrast/saturation + # or (0., 0.) for hue, do nothing + if value[0] == value[1] == center: + value = None + return value + + @staticmethod + def blend(color1, color2, ratio): + ratio = float(ratio) + bound = 255.0 + return ( + (ratio * color1 + (1.0 - ratio) * color2) + .clip(0, bound) + .astype(color1.dtype) + ) + + @staticmethod + def rgb2hsv(rgb): + r, g, b = rgb[..., 0], rgb[..., 1], rgb[..., 2] + maxc = np.max(rgb, axis=-1) + minc = np.min(rgb, axis=-1) + eqc = maxc == minc + cr = maxc - minc + s = cr / (np.ones_like(maxc) * eqc + maxc * (1 - eqc)) + cr_divisor = np.ones_like(maxc) * eqc + cr * (1 - eqc) + rc = (maxc - r) / cr_divisor + gc = (maxc - g) / cr_divisor + bc = (maxc - b) / cr_divisor + + hr = (maxc == r) * (bc - gc) + hg = ((maxc == g) & (maxc != r)) * (2.0 + rc - bc) + hb = ((maxc != g) & (maxc != r)) * (4.0 + gc - rc) + h = hr + hg + hb + h = (h / 6.0 + 1.0) % 1.0 + return np.stack((h, s, maxc), axis=-1) + + @staticmethod + def hsv2rgb(hsv): + h, s, v = hsv[..., 0], hsv[..., 1], hsv[..., 2] + i = np.floor(h * 6.0) + f = (h * 6.0) - i + i = i.astype(np.int32) + + p = np.clip((v * (1.0 - s)), 0.0, 1.0) + q = np.clip((v * (1.0 - s * f)), 0.0, 1.0) + t = np.clip((v * (1.0 - s * (1.0 - f))), 0.0, 1.0) + i = i % 6 + mask = np.expand_dims(i, axis=-1) == np.arange(6) + + a1 = np.stack((v, q, p, p, t, v), axis=-1) + a2 = np.stack((t, v, v, q, p, p), axis=-1) + a3 = np.stack((p, p, t, v, v, q), axis=-1) + a4 = np.stack((a1, a2, a3), axis=-1) + + return np.einsum("...na, ...nab -> ...nb", mask.astype(hsv.dtype), a4) + + def adjust_brightness(self, color, brightness_factor): + if brightness_factor < 0: + raise ValueError( + "brightness_factor ({}) is not non-negative.".format(brightness_factor) + ) + + return self.blend(color, np.zeros_like(color), brightness_factor) + + def adjust_contrast(self, color, contrast_factor): + if contrast_factor < 0: + raise ValueError( + "contrast_factor ({}) is not non-negative.".format(contrast_factor) + ) + mean = np.mean(RandomColorGrayScale.rgb_to_grayscale(color)) + return self.blend(color, mean, contrast_factor) + + def adjust_saturation(self, color, saturation_factor): + if saturation_factor < 0: + raise ValueError( + "saturation_factor ({}) is not non-negative.".format(saturation_factor) + ) + gray = RandomColorGrayScale.rgb_to_grayscale(color) + return self.blend(color, gray, saturation_factor) + + def adjust_hue(self, color, hue_factor): + if not (-0.5 <= hue_factor <= 0.5): + raise ValueError( + "hue_factor ({}) is not in [-0.5, 0.5].".format(hue_factor) + ) + orig_dtype = color.dtype + hsv = self.rgb2hsv(color / 255.0) + h, s, v = hsv[..., 0], hsv[..., 1], hsv[..., 2] + h = (h + hue_factor) % 1.0 + hsv = np.stack((h, s, v), axis=-1) + color_hue_adj = (self.hsv2rgb(hsv) * 255.0).astype(orig_dtype) + return color_hue_adj + + @staticmethod + def get_params(brightness, contrast, saturation, hue): + fn_idx = torch.randperm(4) + b = ( + None + if brightness is None + else np.random.uniform(brightness[0], brightness[1]) + ) + c = None if contrast is None else np.random.uniform(contrast[0], contrast[1]) + s = ( + None + if saturation is None + else np.random.uniform(saturation[0], saturation[1]) + ) + h = None if hue is None else np.random.uniform(hue[0], hue[1]) + return fn_idx, b, c, s, h + + def __call__(self, data_dict): + ( + fn_idx, + brightness_factor, + contrast_factor, + saturation_factor, + hue_factor, + ) = self.get_params(self.brightness, self.contrast, self.saturation, self.hue) + + for fn_id in fn_idx: + if ( + fn_id == 0 + and brightness_factor is not None + and np.random.rand() < self.p + ): + data_dict["color"] = self.adjust_brightness( + data_dict["color"], brightness_factor + ) + elif ( + fn_id == 1 and contrast_factor is not None and np.random.rand() < self.p + ): + data_dict["color"] = self.adjust_contrast( + data_dict["color"], contrast_factor + ) + elif ( + fn_id == 2 + and saturation_factor is not None + and np.random.rand() < self.p + ): + data_dict["color"] = self.adjust_saturation( + data_dict["color"], saturation_factor + ) + elif fn_id == 3 and hue_factor is not None and np.random.rand() < self.p: + data_dict["color"] = self.adjust_hue(data_dict["color"], hue_factor) + return data_dict + + +@TRANSFORMS.register_module() +class HueSaturationTranslation(object): + @staticmethod + def rgb_to_hsv(rgb): + # Translated from source of colorsys.rgb_to_hsv + # r,g,b should be a numpy arrays with values between 0 and 255 + # rgb_to_hsv returns an array of floats between 0.0 and 1.0. + rgb = rgb.astype("float") + hsv = np.zeros_like(rgb) + # in case an RGBA array was passed, just copy the A channel + hsv[..., 3:] = rgb[..., 3:] + r, g, b = rgb[..., 0], rgb[..., 1], rgb[..., 2] + maxc = np.max(rgb[..., :3], axis=-1) + minc = np.min(rgb[..., :3], axis=-1) + hsv[..., 2] = maxc + mask = maxc != minc + hsv[mask, 1] = (maxc - minc)[mask] / maxc[mask] + rc = np.zeros_like(r) + gc = np.zeros_like(g) + bc = np.zeros_like(b) + rc[mask] = (maxc - r)[mask] / (maxc - minc)[mask] + gc[mask] = (maxc - g)[mask] / (maxc - minc)[mask] + bc[mask] = (maxc - b)[mask] / (maxc - minc)[mask] + hsv[..., 0] = np.select( + [r == maxc, g == maxc], [bc - gc, 2.0 + rc - bc], default=4.0 + gc - rc + ) + hsv[..., 0] = (hsv[..., 0] / 6.0) % 1.0 + return hsv + + @staticmethod + def hsv_to_rgb(hsv): + # Translated from source of colorsys.hsv_to_rgb + # h,s should be a numpy arrays with values between 0.0 and 1.0 + # v should be a numpy array with values between 0.0 and 255.0 + # hsv_to_rgb returns an array of uints between 0 and 255. + rgb = np.empty_like(hsv) + rgb[..., 3:] = hsv[..., 3:] + h, s, v = hsv[..., 0], hsv[..., 1], hsv[..., 2] + i = (h * 6.0).astype("uint8") + f = (h * 6.0) - i + p = v * (1.0 - s) + q = v * (1.0 - s * f) + t = v * (1.0 - s * (1.0 - f)) + i = i % 6 + conditions = [s == 0.0, i == 1, i == 2, i == 3, i == 4, i == 5] + rgb[..., 0] = np.select(conditions, [v, q, p, p, t, v], default=v) + rgb[..., 1] = np.select(conditions, [v, v, v, q, p, p], default=t) + rgb[..., 2] = np.select(conditions, [v, p, t, v, v, q], default=p) + return rgb.astype("uint8") + + def __init__(self, hue_max=0.5, saturation_max=0.2): + self.hue_max = hue_max + self.saturation_max = saturation_max + + def __call__(self, data_dict): + if "color" in data_dict.keys(): + # Assume color[:, :3] is rgb + hsv = HueSaturationTranslation.rgb_to_hsv(data_dict["color"][:, :3]) + hue_val = (np.random.rand() - 0.5) * 2 * self.hue_max + sat_ratio = 1 + (np.random.rand() - 0.5) * 2 * self.saturation_max + hsv[..., 0] = np.remainder(hue_val + hsv[..., 0] + 1, 1) + hsv[..., 1] = np.clip(sat_ratio * hsv[..., 1], 0, 1) + data_dict["color"][:, :3] = np.clip( + HueSaturationTranslation.hsv_to_rgb(hsv), 0, 255 + ) + return data_dict + + +@TRANSFORMS.register_module() +class RandomColorDrop(object): + def __init__(self, p=0.2, color_augment=0.0): + self.p = p + self.color_augment = color_augment + + def __call__(self, data_dict): + if "color" in data_dict.keys() and np.random.rand() < self.p: + data_dict["color"] *= self.color_augment + return data_dict + + def __repr__(self): + return "RandomColorDrop(color_augment: {}, p: {})".format( + self.color_augment, self.p + ) + + +@TRANSFORMS.register_module() +class ElasticDistortion(object): + def __init__(self, distortion_params=None): + self.distortion_params = ( + [[0.2, 0.4], [0.8, 1.6]] if distortion_params is None else distortion_params + ) + + @staticmethod + def elastic_distortion(coords, granularity, magnitude): + """ + Apply elastic distortion on sparse coordinate space. + pointcloud: numpy array of (number of points, at least 3 spatial dims) + granularity: size of the noise grid (in same scale[m/cm] as the voxel grid) + magnitude: noise multiplier + """ + blurx = np.ones((3, 1, 1, 1)).astype("float32") / 3 + blury = np.ones((1, 3, 1, 1)).astype("float32") / 3 + blurz = np.ones((1, 1, 3, 1)).astype("float32") / 3 + coords_min = coords.min(0) + + # Create Gaussian noise tensor of the size given by granularity. + noise_dim = ((coords - coords_min).max(0) // granularity).astype(int) + 3 + noise = np.random.randn(*noise_dim, 3).astype(np.float32) + + # Smoothing. + for _ in range(2): + noise = scipy.ndimage.filters.convolve( + noise, blurx, mode="constant", cval=0 + ) + noise = scipy.ndimage.filters.convolve( + noise, blury, mode="constant", cval=0 + ) + noise = scipy.ndimage.filters.convolve( + noise, blurz, mode="constant", cval=0 + ) + + # Trilinear interpolate noise filters for each spatial dimensions. + ax = [ + np.linspace(d_min, d_max, d) + for d_min, d_max, d in zip( + coords_min - granularity, + coords_min + granularity * (noise_dim - 2), + noise_dim, + ) + ] + interp = scipy.interpolate.RegularGridInterpolator( + ax, noise, bounds_error=False, fill_value=0 + ) + coords += interp(coords) * magnitude + return coords + + def __call__(self, data_dict): + if "coord" in data_dict.keys() and self.distortion_params is not None: + if random.random() < 0.95: + for granularity, magnitude in self.distortion_params: + data_dict["coord"] = self.elastic_distortion( + data_dict["coord"], granularity, magnitude + ) + return data_dict + + +@TRANSFORMS.register_module() +class GridSample(object): + def __init__( + self, + grid_size=0.05, + hash_type="fnv", + mode="train", + keys=("coord", "color", "normal", "segment"), + return_inverse=False, + return_grid_coord=False, + return_min_coord=False, + return_displacement=False, + project_displacement=False, + ): + self.grid_size = grid_size + self.hash = self.fnv_hash_vec if hash_type == "fnv" else self.ravel_hash_vec + assert mode in ["train", "test"] + self.mode = mode + self.keys = keys + self.return_inverse = return_inverse + self.return_grid_coord = return_grid_coord + self.return_min_coord = return_min_coord + self.return_displacement = return_displacement + self.project_displacement = project_displacement + + def __call__(self, data_dict): + assert "coord" in data_dict.keys() + scaled_coord = data_dict["coord"] / np.array(self.grid_size) + grid_coord = np.floor(scaled_coord).astype(int) + min_coord = grid_coord.min(0) + grid_coord -= min_coord + scaled_coord -= min_coord + min_coord = min_coord * np.array(self.grid_size) + key = self.hash(grid_coord) + idx_sort = np.argsort(key) + key_sort = key[idx_sort] + _, inverse, count = np.unique(key_sort, return_inverse=True, return_counts=True) + if self.mode == "train": # train mode + idx_select = ( + np.cumsum(np.insert(count, 0, 0)[0:-1]) + + np.random.randint(0, count.max(), count.size) % count + ) + idx_unique = idx_sort[idx_select] + if "sampled_index" in data_dict: + # for ScanNet data efficient, we need to make sure labeled point is sampled. + idx_unique = np.unique( + np.append(idx_unique, data_dict["sampled_index"]) + ) + mask = np.zeros_like(data_dict["segment"]).astype(bool) + mask[data_dict["sampled_index"]] = True + data_dict["sampled_index"] = np.where(mask[idx_unique])[0] + if self.return_inverse: + data_dict["inverse"] = np.zeros_like(inverse) + data_dict["inverse"][idx_sort] = inverse + if self.return_grid_coord: + data_dict["grid_coord"] = grid_coord[idx_unique] + if self.return_min_coord: + data_dict["min_coord"] = min_coord.reshape([1, 3]) + if self.return_displacement: + displacement = ( + scaled_coord - grid_coord - 0.5 + ) # [0, 1] -> [-0.5, 0.5] displacement to center + if self.project_displacement: + displacement = np.sum( + displacement * data_dict["normal"], axis=-1, keepdims=True + ) + data_dict["displacement"] = displacement[idx_unique] + for key in self.keys: + data_dict[key] = data_dict[key][idx_unique] + return data_dict + + elif self.mode == "test": # test mode + data_part_list = [] + for i in range(count.max()): + idx_select = np.cumsum(np.insert(count, 0, 0)[0:-1]) + i % count + idx_part = idx_sort[idx_select] + data_part = dict(index=idx_part) + if self.return_inverse: + data_dict["inverse"] = np.zeros_like(inverse) + data_dict["inverse"][idx_sort] = inverse + if self.return_grid_coord: + data_part["grid_coord"] = grid_coord[idx_part] + if self.return_min_coord: + data_part["min_coord"] = min_coord.reshape([1, 3]) + if self.return_displacement: + displacement = ( + scaled_coord - grid_coord - 0.5 + ) # [0, 1] -> [-0.5, 0.5] displacement to center + if self.project_displacement: + displacement = np.sum( + displacement * data_dict["normal"], axis=-1, keepdims=True + ) + data_dict["displacement"] = displacement[idx_part] + for key in data_dict.keys(): + if key in self.keys: + data_part[key] = data_dict[key][idx_part] + else: + data_part[key] = data_dict[key] + data_part_list.append(data_part) + return data_part_list + else: + raise NotImplementedError + + @staticmethod + def ravel_hash_vec(arr): + """ + Ravel the coordinates after subtracting the min coordinates. + """ + assert arr.ndim == 2 + arr = arr.copy() + arr -= arr.min(0) + arr = arr.astype(np.uint64, copy=False) + arr_max = arr.max(0).astype(np.uint64) + 1 + + keys = np.zeros(arr.shape[0], dtype=np.uint64) + # Fortran style indexing + for j in range(arr.shape[1] - 1): + keys += arr[:, j] + keys *= arr_max[j + 1] + keys += arr[:, -1] + return keys + + @staticmethod + def fnv_hash_vec(arr): + """ + FNV64-1A + """ + assert arr.ndim == 2 + # Floor first for negative coordinates + arr = arr.copy() + arr = arr.astype(np.uint64, copy=False) + hashed_arr = np.uint64(14695981039346656037) * np.ones( + arr.shape[0], dtype=np.uint64 + ) + for j in range(arr.shape[1]): + hashed_arr *= np.uint64(1099511628211) + hashed_arr = np.bitwise_xor(hashed_arr, arr[:, j]) + return hashed_arr + + +@TRANSFORMS.register_module() +class SphereCrop(object): + def __init__(self, point_max=80000, sample_rate=None, mode="random"): + self.point_max = point_max + self.sample_rate = sample_rate + assert mode in ["random", "center", "all"] + self.mode = mode + + def __call__(self, data_dict): + point_max = ( + int(self.sample_rate * data_dict["coord"].shape[0]) + if self.sample_rate is not None + else self.point_max + ) + + assert "coord" in data_dict.keys() + if self.mode == "all": + # TODO: Optimize + if "index" not in data_dict.keys(): + data_dict["index"] = np.arange(data_dict["coord"].shape[0]) + data_part_list = [] + # coord_list, color_list, dist2_list, idx_list, offset_list = [], [], [], [], [] + if data_dict["coord"].shape[0] > point_max: + coord_p, idx_uni = np.random.rand( + data_dict["coord"].shape[0] + ) * 1e-3, np.array([]) + while idx_uni.size != data_dict["index"].shape[0]: + init_idx = np.argmin(coord_p) + dist2 = np.sum( + np.power(data_dict["coord"] - data_dict["coord"][init_idx], 2), + 1, + ) + idx_crop = np.argsort(dist2)[:point_max] + + data_crop_dict = dict() + if "coord" in data_dict.keys(): + data_crop_dict["coord"] = data_dict["coord"][idx_crop] + if "grid_coord" in data_dict.keys(): + data_crop_dict["grid_coord"] = data_dict["grid_coord"][idx_crop] + if "normal" in data_dict.keys(): + data_crop_dict["normal"] = data_dict["normal"][idx_crop] + if "color" in data_dict.keys(): + data_crop_dict["color"] = data_dict["color"][idx_crop] + if "displacement" in data_dict.keys(): + data_crop_dict["displacement"] = data_dict["displacement"][ + idx_crop + ] + if "strength" in data_dict.keys(): + data_crop_dict["strength"] = data_dict["strength"][idx_crop] + data_crop_dict["weight"] = dist2[idx_crop] + data_crop_dict["index"] = data_dict["index"][idx_crop] + data_part_list.append(data_crop_dict) + + delta = np.square( + 1 - data_crop_dict["weight"] / np.max(data_crop_dict["weight"]) + ) + coord_p[idx_crop] += delta + idx_uni = np.unique( + np.concatenate((idx_uni, data_crop_dict["index"])) + ) + else: + data_crop_dict = data_dict.copy() + data_crop_dict["weight"] = np.zeros(data_dict["coord"].shape[0]) + data_crop_dict["index"] = data_dict["index"] + data_part_list.append(data_crop_dict) + return data_part_list + # mode is "random" or "center" + elif data_dict["coord"].shape[0] > point_max: + if self.mode == "random": + center = data_dict["coord"][ + np.random.randint(data_dict["coord"].shape[0]) + ] + elif self.mode == "center": + center = data_dict["coord"][data_dict["coord"].shape[0] // 2] + else: + raise NotImplementedError + idx_crop = np.argsort(np.sum(np.square(data_dict["coord"] - center), 1))[ + :point_max + ] + if "coord" in data_dict.keys(): + data_dict["coord"] = data_dict["coord"][idx_crop] + if "origin_coord" in data_dict.keys(): + data_dict["origin_coord"] = data_dict["origin_coord"][idx_crop] + if "grid_coord" in data_dict.keys(): + data_dict["grid_coord"] = data_dict["grid_coord"][idx_crop] + if "color" in data_dict.keys(): + data_dict["color"] = data_dict["color"][idx_crop] + if "normal" in data_dict.keys(): + data_dict["normal"] = data_dict["normal"][idx_crop] + if "segment" in data_dict.keys(): + data_dict["segment"] = data_dict["segment"][idx_crop] + if "instance" in data_dict.keys(): + data_dict["instance"] = data_dict["instance"][idx_crop] + if "displacement" in data_dict.keys(): + data_dict["displacement"] = data_dict["displacement"][idx_crop] + if "strength" in data_dict.keys(): + data_dict["strength"] = data_dict["strength"][idx_crop] + return data_dict + + +@TRANSFORMS.register_module() +class ShufflePoint(object): + def __call__(self, data_dict): + assert "coord" in data_dict.keys() + shuffle_index = np.arange(data_dict["coord"].shape[0]) + np.random.shuffle(shuffle_index) + if "coord" in data_dict.keys(): + data_dict["coord"] = data_dict["coord"][shuffle_index] + if "grid_coord" in data_dict.keys(): + data_dict["grid_coord"] = data_dict["grid_coord"][shuffle_index] + if "displacement" in data_dict.keys(): + data_dict["displacement"] = data_dict["displacement"][shuffle_index] + if "color" in data_dict.keys(): + data_dict["color"] = data_dict["color"][shuffle_index] + if "normal" in data_dict.keys(): + data_dict["normal"] = data_dict["normal"][shuffle_index] + if "segment" in data_dict.keys(): + data_dict["segment"] = data_dict["segment"][shuffle_index] + if "instance" in data_dict.keys(): + data_dict["instance"] = data_dict["instance"][shuffle_index] + return data_dict + + +@TRANSFORMS.register_module() +class CropBoundary(object): + def __call__(self, data_dict): + assert "segment" in data_dict + segment = data_dict["segment"].flatten() + mask = (segment != 0) * (segment != 1) + if "coord" in data_dict.keys(): + data_dict["coord"] = data_dict["coord"][mask] + if "grid_coord" in data_dict.keys(): + data_dict["grid_coord"] = data_dict["grid_coord"][mask] + if "color" in data_dict.keys(): + data_dict["color"] = data_dict["color"][mask] + if "normal" in data_dict.keys(): + data_dict["normal"] = data_dict["normal"][mask] + if "segment" in data_dict.keys(): + data_dict["segment"] = data_dict["segment"][mask] + if "instance" in data_dict.keys(): + data_dict["instance"] = data_dict["instance"][mask] + return data_dict + + +@TRANSFORMS.register_module() +class ContrastiveViewsGenerator(object): + def __init__( + self, + view_keys=("coord", "color", "normal", "origin_coord"), + view_trans_cfg=None, + ): + self.view_keys = view_keys + self.view_trans = Compose(view_trans_cfg) + + def __call__(self, data_dict): + view1_dict = dict() + view2_dict = dict() + for key in self.view_keys: + view1_dict[key] = data_dict[key].copy() + view2_dict[key] = data_dict[key].copy() + view1_dict = self.view_trans(view1_dict) + view2_dict = self.view_trans(view2_dict) + for key, value in view1_dict.items(): + data_dict["view1_" + key] = value + for key, value in view2_dict.items(): + data_dict["view2_" + key] = value + return data_dict + + +@TRANSFORMS.register_module() +class InstanceParser(object): + def __init__(self, segment_ignore_index=(-1, 0, 1), instance_ignore_index=-1): + self.segment_ignore_index = segment_ignore_index + self.instance_ignore_index = instance_ignore_index + + def __call__(self, data_dict): + coord = data_dict["coord"] + segment = data_dict["segment"] + instance = data_dict["instance"] + mask = ~np.in1d(segment, self.segment_ignore_index) + # mapping ignored instance to ignore index + instance[~mask] = self.instance_ignore_index + # reorder left instance + unique, inverse = np.unique(instance[mask], return_inverse=True) + instance_num = len(unique) + instance[mask] = inverse + # init instance information + centroid = np.ones((coord.shape[0], 3)) * self.instance_ignore_index + bbox = np.ones((instance_num, 8)) * self.instance_ignore_index + vacancy = [ + index for index in self.segment_ignore_index if index >= 0 + ] # vacate class index + + for instance_id in range(instance_num): + mask_ = instance == instance_id + coord_ = coord[mask_] + bbox_min = coord_.min(0) + bbox_max = coord_.max(0) + bbox_centroid = coord_.mean(0) + bbox_center = (bbox_max + bbox_min) / 2 + bbox_size = bbox_max - bbox_min + bbox_theta = np.zeros(1, dtype=coord_.dtype) + bbox_class = np.array([segment[mask_][0]], dtype=coord_.dtype) + # shift class index to fill vacate class index caused by segment ignore index + bbox_class -= np.greater(bbox_class, vacancy).sum() + + centroid[mask_] = bbox_centroid + bbox[instance_id] = np.concatenate( + [bbox_center, bbox_size, bbox_theta, bbox_class] + ) # 3 + 3 + 1 + 1 = 8 + data_dict["instance"] = instance + data_dict["instance_centroid"] = centroid + data_dict["bbox"] = bbox + return data_dict + + +class Compose(object): + def __init__(self, cfg=None): + self.cfg = cfg if cfg is not None else [] + self.transforms = [] + for t_cfg in self.cfg: + self.transforms.append(TRANSFORMS.build(t_cfg)) + + def __call__(self, data_dict): + for t in self.transforms: + data_dict = t(data_dict) + return data_dict diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/utils.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3abb9bf88c81f5eae302468ffc91c62bd942a002 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/utils.py @@ -0,0 +1,59 @@ +""" +Utils for Datasets + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import random +from collections.abc import Mapping, Sequence +import numpy as np +import torch +from torch.utils.data.dataloader import default_collate + + +def collate_fn(batch): + """ + collate function for point cloud which support dict and list, + 'coord' is necessary to determine 'offset' + """ + if not isinstance(batch, Sequence): + raise TypeError(f"{batch.dtype} is not supported.") + + if isinstance(batch[0], torch.Tensor): + return torch.cat(list(batch)) + elif isinstance(batch[0], str): + # str is also a kind of Sequence, judgement should before Sequence + return list(batch) + elif isinstance(batch[0], Sequence): + for data in batch: + data.append(torch.tensor([data[0].shape[0]])) + batch = [collate_fn(samples) for samples in zip(*batch)] + batch[-1] = torch.cumsum(batch[-1], dim=0).int() + return batch + elif isinstance(batch[0], Mapping): + batch = {key: collate_fn([d[key] for d in batch]) for key in batch[0]} + for key in batch.keys(): + if "offset" in key: + batch[key] = torch.cumsum(batch[key], dim=0) + return batch + else: + return default_collate(batch) + + +def point_collate_fn(batch, mix_prob=0): + assert isinstance( + batch[0], Mapping + ) # currently, only support input_dict, rather than input_list + batch = collate_fn(batch) + if "offset" in batch.keys(): + # Mix3d (https://arxiv.org/pdf/2110.02210.pdf) + if random.random() < mix_prob: + batch["offset"] = torch.cat( + [batch["offset"][1:-1:2], batch["offset"][-1].unsqueeze(0)], dim=0 + ) + return batch + + +def gaussian_kernel(dist2: np.array, a: float = 1, c: float = 5): + return a * np.exp(-dist2 / (2 * c**2)) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/datasets/waymo.py b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/waymo.py new file mode 100644 index 0000000000000000000000000000000000000000..93e7f7ee1a0f18f9c932248b5d1d192dcb78f5f5 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/datasets/waymo.py @@ -0,0 +1,104 @@ +""" +Waymo dataset + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import numpy as np +import glob + +from .builder import DATASETS +from .defaults import DefaultDataset + + +@DATASETS.register_module() +class WaymoDataset(DefaultDataset): + def __init__( + self, + timestamp=(0,), + reference_label=True, + timing_embedding=False, + **kwargs, + ): + super().__init__(**kwargs) + assert timestamp[0] == 0 + self.timestamp = timestamp + self.reference_label = reference_label + self.timing_embedding = timing_embedding + self.data_list = sorted(self.data_list) + _, self.sequence_offset, self.sequence_index = np.unique( + [os.path.dirname(data) for data in self.data_list], + return_index=True, + return_inverse=True, + ) + self.sequence_offset = np.append(self.sequence_offset, len(self.data_list)) + + def get_data_list(self): + if isinstance(self.split, str): + self.split = [self.split] + data_list = [] + for split in self.split: + data_list += glob.glob(os.path.join(self.data_root, split, "*", "*")) + return data_list + + @staticmethod + def align_pose(coord, pose, target_pose): + coord = np.hstack((coord, np.ones_like(coord[:, :1]))) + pose_align = np.matmul(np.linalg.inv(target_pose), pose) + coord = (pose_align @ coord.T).T[:, :3] + return coord + + def get_single_frame(self, idx): + return super().get_data(idx) + + def get_data(self, idx): + idx = idx % len(self.data_list) + if self.timestamp == (0,): + return self.get_single_frame(idx) + + sequence_index = self.sequence_index[idx] + lower, upper = self.sequence_offset[[sequence_index, sequence_index + 1]] + major_frame = self.get_single_frame(idx) + name = major_frame.pop("name") + target_pose = major_frame.pop("pose") + for key in major_frame.keys(): + major_frame[key] = [major_frame[key]] + + for timestamp in self.timestamp[1:]: + refer_idx = timestamp + idx + if refer_idx < lower or upper <= refer_idx: + continue + refer_frame = self.get_single_frame(refer_idx) + refer_frame.pop("name") + pose = refer_frame.pop("pose") + refer_frame["coord"] = self.align_pose( + refer_frame["coord"], pose, target_pose + ) + if not self.reference_label: + refer_frame["segment"] = ( + np.ones_like(refer_frame["segment"]) * self.ignore_index + ) + + if self.timing_embedding: + refer_frame["strength"] = np.hstack( + ( + refer_frame["strength"], + np.ones_like(refer_frame["strength"]) * timestamp, + ) + ) + + for key in major_frame.keys(): + major_frame[key].append(refer_frame[key]) + for key in major_frame.keys(): + major_frame[key] = np.concatenate(major_frame[key], axis=0) + major_frame["name"] = name + return major_frame + + def get_data_name(self, idx): + file_path = self.data_list[idx % len(self.data_list)] + sequence_path, frame_name = os.path.split(file_path) + sequence_name = os.path.basename(sequence_path) + data_name = f"{sequence_name}_{frame_name}" + return data_name diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/engines/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/engines/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/engines/defaults.py b/submodules/PointTransformerV3/Pointcept/pointcept/engines/defaults.py new file mode 100644 index 0000000000000000000000000000000000000000..d45e7925a50acb03bb510c46ec4c566f6815cc05 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/engines/defaults.py @@ -0,0 +1,152 @@ +""" +Default training/testing logic + +modified from detectron2(https://github.com/facebookresearch/detectron2) + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import sys +import argparse +import multiprocessing as mp +from torch.nn.parallel import DistributedDataParallel + + +import pointcept.utils.comm as comm +from pointcept.utils.env import get_random_seed, set_seed +from pointcept.utils.config import Config, DictAction + + +def create_ddp_model(model, *, fp16_compression=False, **kwargs): + """ + Create a DistributedDataParallel model if there are >1 processes. + Args: + model: a torch.nn.Module + fp16_compression: add fp16 compression hooks to the ddp object. + See more at https://pytorch.org/docs/stable/ddp_comm_hooks.html#torch.distributed.algorithms.ddp_comm_hooks.default_hooks.fp16_compress_hook + kwargs: other arguments of :module:`torch.nn.parallel.DistributedDataParallel`. + """ + if comm.get_world_size() == 1: + return model + # kwargs['find_unused_parameters'] = True + if "device_ids" not in kwargs: + kwargs["device_ids"] = [comm.get_local_rank()] + if "output_device" not in kwargs: + kwargs["output_device"] = [comm.get_local_rank()] + ddp = DistributedDataParallel(model, **kwargs) + if fp16_compression: + from torch.distributed.algorithms.ddp_comm_hooks import default as comm_hooks + + ddp.register_comm_hook(state=None, hook=comm_hooks.fp16_compress_hook) + return ddp + + +def worker_init_fn(worker_id, num_workers, rank, seed): + """Worker init func for dataloader. + + The seed of each worker equals to num_worker * rank + worker_id + user_seed + + Args: + worker_id (int): Worker id. + num_workers (int): Number of workers. + rank (int): The rank of current process. + seed (int): The random seed to use. + """ + + worker_seed = num_workers * rank + worker_id + seed + set_seed(worker_seed) + + +def default_argument_parser(epilog=None): + parser = argparse.ArgumentParser( + epilog=epilog + or f""" + Examples: + Run on single machine: + $ {sys.argv[0]} --num-gpus 8 --config-file cfg.yaml + Change some config options: + $ {sys.argv[0]} --config-file cfg.yaml MODEL.WEIGHTS /path/to/weight.pth SOLVER.BASE_LR 0.001 + Run on multiple machines: + (machine0)$ {sys.argv[0]} --machine-rank 0 --num-machines 2 --dist-url [--other-flags] + (machine1)$ {sys.argv[0]} --machine-rank 1 --num-machines 2 --dist-url [--other-flags] + """, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--config-file", default="", metavar="FILE", help="path to config file" + ) + parser.add_argument( + "--num-gpus", type=int, default=1, help="number of gpus *per machine*" + ) + parser.add_argument( + "--num-machines", type=int, default=1, help="total number of machines" + ) + parser.add_argument( + "--machine-rank", + type=int, + default=0, + help="the rank of this machine (unique per machine)", + ) + # PyTorch still may leave orphan processes in multi-gpu training. + # Therefore we use a deterministic way to obtain port, + # so that users are aware of orphan processes by seeing the port occupied. + # port = 2 ** 15 + 2 ** 14 + hash(os.getuid() if sys.platform != "win32" else 1) % 2 ** 14 + parser.add_argument( + "--dist-url", + # default="tcp://127.0.0.1:{}".format(port), + default="auto", + help="initialization URL for pytorch distributed backend. See " + "https://pytorch.org/docs/stable/distributed.html for details.", + ) + parser.add_argument( + "--options", nargs="+", action=DictAction, help="custom options" + ) + return parser + + +def default_config_parser(file_path, options): + # config name protocol: dataset_name/model_name-exp_name + if os.path.isfile(file_path): + cfg = Config.fromfile(file_path) + else: + sep = file_path.find("-") + cfg = Config.fromfile(os.path.join(file_path[:sep], file_path[sep + 1 :])) + + if options is not None: + cfg.merge_from_dict(options) + + if cfg.seed is None: + cfg.seed = get_random_seed() + + cfg.data.train.loop = cfg.epoch // cfg.eval_epoch + + os.makedirs(os.path.join(cfg.save_path, "model"), exist_ok=True) + if not cfg.resume: + cfg.dump(os.path.join(cfg.save_path, "config.py")) + return cfg + + +def default_setup(cfg): + # scalar by world size + world_size = comm.get_world_size() + cfg.num_worker = cfg.num_worker if cfg.num_worker is not None else mp.cpu_count() + cfg.num_worker_per_gpu = cfg.num_worker // world_size + assert cfg.batch_size % world_size == 0 + assert cfg.batch_size_val is None or cfg.batch_size_val % world_size == 0 + assert cfg.batch_size_test is None or cfg.batch_size_test % world_size == 0 + cfg.batch_size_per_gpu = cfg.batch_size // world_size + cfg.batch_size_val_per_gpu = ( + cfg.batch_size_val // world_size if cfg.batch_size_val is not None else 1 + ) + cfg.batch_size_test_per_gpu = ( + cfg.batch_size_test // world_size if cfg.batch_size_test is not None else 1 + ) + # update data loop + assert cfg.epoch % cfg.eval_epoch == 0 + # settle random seed + rank = comm.get_rank() + seed = None if cfg.seed is None else cfg.seed * cfg.num_worker_per_gpu + rank + set_seed(seed) + return cfg diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ab2c4beb7f1938d9703e572ad8619fe88bff223 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/__init__.py @@ -0,0 +1,5 @@ +from .default import HookBase +from .misc import * +from .evaluator import * + +from .builder import build_hooks diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/builder.py b/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..2f4cce4871b0e18f3adc1f7430a8d5410442c77c --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/builder.py @@ -0,0 +1,18 @@ +""" +Hook Builder + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from pointcept.utils.registry import Registry + + +HOOKS = Registry("hooks") + + +def build_hooks(cfg): + hooks = [] + for hook_cfg in cfg: + hooks.append(HOOKS.build(hook_cfg)) + return hooks diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/default.py b/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/default.py new file mode 100644 index 0000000000000000000000000000000000000000..87a64415a5a66d2570dffbaa7b90707443be42e2 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/default.py @@ -0,0 +1,32 @@ +""" +Default Hook + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + + +class HookBase: + """ + Base class for hooks that can be registered with :class:`TrainerBase`. + """ + + trainer = None # A weak reference to the trainer object. + + def before_train(self): + pass + + def before_epoch(self): + pass + + def before_step(self): + pass + + def after_step(self): + pass + + def after_epoch(self): + pass + + def after_train(self): + pass diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/evaluator.py b/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..02b35b3abd83e0a7b59f532f1ca8aacf70afcce2 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/evaluator.py @@ -0,0 +1,581 @@ +""" +Evaluate Hook + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import numpy as np +import torch +import torch.distributed as dist +import pointops +from uuid import uuid4 + +import pointcept.utils.comm as comm +from pointcept.utils.misc import intersection_and_union_gpu + +from .default import HookBase +from .builder import HOOKS + + +@HOOKS.register_module() +class ClsEvaluator(HookBase): + def after_epoch(self): + if self.trainer.cfg.evaluate: + self.eval() + + def eval(self): + self.trainer.logger.info(">>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>") + self.trainer.model.eval() + for i, input_dict in enumerate(self.trainer.val_loader): + for key in input_dict.keys(): + if isinstance(input_dict[key], torch.Tensor): + input_dict[key] = input_dict[key].cuda(non_blocking=True) + with torch.no_grad(): + output_dict = self.trainer.model(input_dict) + output = output_dict["cls_logits"] + loss = output_dict["loss"] + pred = output.max(1)[1] + label = input_dict["category"] + intersection, union, target = intersection_and_union_gpu( + pred, + label, + self.trainer.cfg.data.num_classes, + self.trainer.cfg.data.ignore_index, + ) + if comm.get_world_size() > 1: + dist.all_reduce(intersection), dist.all_reduce(union), dist.all_reduce( + target + ) + intersection, union, target = ( + intersection.cpu().numpy(), + union.cpu().numpy(), + target.cpu().numpy(), + ) + # Here there is no need to sync since sync happened in dist.all_reduce + self.trainer.storage.put_scalar("val_intersection", intersection) + self.trainer.storage.put_scalar("val_union", union) + self.trainer.storage.put_scalar("val_target", target) + self.trainer.storage.put_scalar("val_loss", loss.item()) + self.trainer.logger.info( + "Test: [{iter}/{max_iter}] " + "Loss {loss:.4f} ".format( + iter=i + 1, max_iter=len(self.trainer.val_loader), loss=loss.item() + ) + ) + loss_avg = self.trainer.storage.history("val_loss").avg + intersection = self.trainer.storage.history("val_intersection").total + union = self.trainer.storage.history("val_union").total + target = self.trainer.storage.history("val_target").total + iou_class = intersection / (union + 1e-10) + acc_class = intersection / (target + 1e-10) + m_iou = np.mean(iou_class) + m_acc = np.mean(acc_class) + all_acc = sum(intersection) / (sum(target) + 1e-10) + self.trainer.logger.info( + "Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.".format( + m_iou, m_acc, all_acc + ) + ) + for i in range(self.trainer.cfg.data.num_classes): + self.trainer.logger.info( + "Class_{idx}-{name} Result: iou/accuracy {iou:.4f}/{accuracy:.4f}".format( + idx=i, + name=self.trainer.cfg.data.names[i], + iou=iou_class[i], + accuracy=acc_class[i], + ) + ) + current_epoch = self.trainer.epoch + 1 + if self.trainer.writer is not None: + self.trainer.writer.add_scalar("val/loss", loss_avg, current_epoch) + self.trainer.writer.add_scalar("val/mIoU", m_iou, current_epoch) + self.trainer.writer.add_scalar("val/mAcc", m_acc, current_epoch) + self.trainer.writer.add_scalar("val/allAcc", all_acc, current_epoch) + self.trainer.logger.info("<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<") + self.trainer.comm_info["current_metric_value"] = all_acc # save for saver + self.trainer.comm_info["current_metric_name"] = "allAcc" # save for saver + + def after_train(self): + self.trainer.logger.info( + "Best {}: {:.4f}".format("allAcc", self.trainer.best_metric_value) + ) + + +@HOOKS.register_module() +class SemSegEvaluator(HookBase): + def after_epoch(self): + if self.trainer.cfg.evaluate: + self.eval() + + def eval(self): + self.trainer.logger.info(">>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>") + self.trainer.model.eval() + for i, input_dict in enumerate(self.trainer.val_loader): + for key in input_dict.keys(): + if isinstance(input_dict[key], torch.Tensor): + input_dict[key] = input_dict[key].cuda(non_blocking=True) + with torch.no_grad(): + output_dict = self.trainer.model(input_dict) + output = output_dict["seg_logits"] + loss = output_dict["loss"] + pred = output.max(1)[1] + segment = input_dict["segment"] + if "origin_coord" in input_dict.keys(): + idx, _ = pointops.knn_query( + 1, + input_dict["coord"].float(), + input_dict["offset"].int(), + input_dict["origin_coord"].float(), + input_dict["origin_offset"].int(), + ) + pred = pred[idx.flatten().long()] + segment = input_dict["origin_segment"] + intersection, union, target = intersection_and_union_gpu( + pred, + segment, + self.trainer.cfg.data.num_classes, + self.trainer.cfg.data.ignore_index, + ) + if comm.get_world_size() > 1: + dist.all_reduce(intersection), dist.all_reduce(union), dist.all_reduce( + target + ) + intersection, union, target = ( + intersection.cpu().numpy(), + union.cpu().numpy(), + target.cpu().numpy(), + ) + # Here there is no need to sync since sync happened in dist.all_reduce + self.trainer.storage.put_scalar("val_intersection", intersection) + self.trainer.storage.put_scalar("val_union", union) + self.trainer.storage.put_scalar("val_target", target) + self.trainer.storage.put_scalar("val_loss", loss.item()) + info = "Test: [{iter}/{max_iter}] ".format( + iter=i + 1, max_iter=len(self.trainer.val_loader) + ) + if "origin_coord" in input_dict.keys(): + info = "Interp. " + info + self.trainer.logger.info( + info + + "Loss {loss:.4f} ".format( + iter=i + 1, max_iter=len(self.trainer.val_loader), loss=loss.item() + ) + ) + loss_avg = self.trainer.storage.history("val_loss").avg + intersection = self.trainer.storage.history("val_intersection").total + union = self.trainer.storage.history("val_union").total + target = self.trainer.storage.history("val_target").total + iou_class = intersection / (union + 1e-10) + acc_class = intersection / (target + 1e-10) + m_iou = np.mean(iou_class) + m_acc = np.mean(acc_class) + all_acc = sum(intersection) / (sum(target) + 1e-10) + self.trainer.logger.info( + "Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.".format( + m_iou, m_acc, all_acc + ) + ) + for i in range(self.trainer.cfg.data.num_classes): + self.trainer.logger.info( + "Class_{idx}-{name} Result: iou/accuracy {iou:.4f}/{accuracy:.4f}".format( + idx=i, + name=self.trainer.cfg.data.names[i], + iou=iou_class[i], + accuracy=acc_class[i], + ) + ) + current_epoch = self.trainer.epoch + 1 + if self.trainer.writer is not None: + self.trainer.writer.add_scalar("val/loss", loss_avg, current_epoch) + self.trainer.writer.add_scalar("val/mIoU", m_iou, current_epoch) + self.trainer.writer.add_scalar("val/mAcc", m_acc, current_epoch) + self.trainer.writer.add_scalar("val/allAcc", all_acc, current_epoch) + self.trainer.logger.info("<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<") + self.trainer.comm_info["current_metric_value"] = m_iou # save for saver + self.trainer.comm_info["current_metric_name"] = "mIoU" # save for saver + + def after_train(self): + self.trainer.logger.info( + "Best {}: {:.4f}".format("mIoU", self.trainer.best_metric_value) + ) + + +@HOOKS.register_module() +class InsSegEvaluator(HookBase): + def __init__(self, segment_ignore_index=(-1,), instance_ignore_index=-1): + self.segment_ignore_index = segment_ignore_index + self.instance_ignore_index = instance_ignore_index + + self.valid_class_names = None # update in before train + self.overlaps = np.append(np.arange(0.5, 0.95, 0.05), 0.25) + self.min_region_sizes = 100 + self.distance_threshes = float("inf") + self.distance_confs = -float("inf") + + def before_train(self): + self.valid_class_names = [ + self.trainer.cfg.data.names[i] + for i in range(self.trainer.cfg.data.num_classes) + if i not in self.segment_ignore_index + ] + + def after_epoch(self): + if self.trainer.cfg.evaluate: + self.eval() + + def associate_instances(self, pred, segment, instance): + segment = segment.cpu().numpy() + instance = instance.cpu().numpy() + void_mask = np.in1d(segment, self.segment_ignore_index) + + assert ( + pred["pred_classes"].shape[0] + == pred["pred_scores"].shape[0] + == pred["pred_masks"].shape[0] + ) + assert pred["pred_masks"].shape[1] == segment.shape[0] == instance.shape[0] + # get gt instances + gt_instances = dict() + for i in range(self.trainer.cfg.data.num_classes): + if i not in self.segment_ignore_index: + gt_instances[self.trainer.cfg.data.names[i]] = [] + instance_ids, idx, counts = np.unique( + instance, return_index=True, return_counts=True + ) + segment_ids = segment[idx] + for i in range(len(instance_ids)): + if instance_ids[i] == self.instance_ignore_index: + continue + if segment_ids[i] in self.segment_ignore_index: + continue + gt_inst = dict() + gt_inst["instance_id"] = instance_ids[i] + gt_inst["segment_id"] = segment_ids[i] + gt_inst["dist_conf"] = 0.0 + gt_inst["med_dist"] = -1.0 + gt_inst["vert_count"] = counts[i] + gt_inst["matched_pred"] = [] + gt_instances[self.trainer.cfg.data.names[segment_ids[i]]].append(gt_inst) + + # get pred instances and associate with gt + pred_instances = dict() + for i in range(self.trainer.cfg.data.num_classes): + if i not in self.segment_ignore_index: + pred_instances[self.trainer.cfg.data.names[i]] = [] + instance_id = 0 + for i in range(len(pred["pred_classes"])): + if pred["pred_classes"][i] in self.segment_ignore_index: + continue + pred_inst = dict() + pred_inst["uuid"] = uuid4() + pred_inst["instance_id"] = instance_id + pred_inst["segment_id"] = pred["pred_classes"][i] + pred_inst["confidence"] = pred["pred_scores"][i] + pred_inst["mask"] = np.not_equal(pred["pred_masks"][i], 0) + pred_inst["vert_count"] = np.count_nonzero(pred_inst["mask"]) + pred_inst["void_intersection"] = np.count_nonzero( + np.logical_and(void_mask, pred_inst["mask"]) + ) + if pred_inst["vert_count"] < self.min_region_sizes: + continue # skip if empty + segment_name = self.trainer.cfg.data.names[pred_inst["segment_id"]] + matched_gt = [] + for gt_idx, gt_inst in enumerate(gt_instances[segment_name]): + intersection = np.count_nonzero( + np.logical_and( + instance == gt_inst["instance_id"], pred_inst["mask"] + ) + ) + if intersection > 0: + gt_inst_ = gt_inst.copy() + pred_inst_ = pred_inst.copy() + gt_inst_["intersection"] = intersection + pred_inst_["intersection"] = intersection + matched_gt.append(gt_inst_) + gt_inst["matched_pred"].append(pred_inst_) + pred_inst["matched_gt"] = matched_gt + pred_instances[segment_name].append(pred_inst) + instance_id += 1 + return gt_instances, pred_instances + + def evaluate_matches(self, scenes): + overlaps = self.overlaps + min_region_sizes = [self.min_region_sizes] + dist_threshes = [self.distance_threshes] + dist_confs = [self.distance_confs] + + # results: class x overlap + ap_table = np.zeros( + (len(dist_threshes), len(self.valid_class_names), len(overlaps)), float + ) + for di, (min_region_size, distance_thresh, distance_conf) in enumerate( + zip(min_region_sizes, dist_threshes, dist_confs) + ): + for oi, overlap_th in enumerate(overlaps): + pred_visited = {} + for scene in scenes: + for _ in scene["pred"]: + for label_name in self.valid_class_names: + for p in scene["pred"][label_name]: + if "uuid" in p: + pred_visited[p["uuid"]] = False + for li, label_name in enumerate(self.valid_class_names): + y_true = np.empty(0) + y_score = np.empty(0) + hard_false_negatives = 0 + has_gt = False + has_pred = False + for scene in scenes: + pred_instances = scene["pred"][label_name] + gt_instances = scene["gt"][label_name] + # filter groups in ground truth + gt_instances = [ + gt + for gt in gt_instances + if gt["vert_count"] >= min_region_size + and gt["med_dist"] <= distance_thresh + and gt["dist_conf"] >= distance_conf + ] + if gt_instances: + has_gt = True + if pred_instances: + has_pred = True + + cur_true = np.ones(len(gt_instances)) + cur_score = np.ones(len(gt_instances)) * (-float("inf")) + cur_match = np.zeros(len(gt_instances), dtype=bool) + # collect matches + for gti, gt in enumerate(gt_instances): + found_match = False + for pred in gt["matched_pred"]: + # greedy assignments + if pred_visited[pred["uuid"]]: + continue + overlap = float(pred["intersection"]) / ( + gt["vert_count"] + + pred["vert_count"] + - pred["intersection"] + ) + if overlap > overlap_th: + confidence = pred["confidence"] + # if already have a prediction for this gt, + # the prediction with the lower score is automatically a false positive + if cur_match[gti]: + max_score = max(cur_score[gti], confidence) + min_score = min(cur_score[gti], confidence) + cur_score[gti] = max_score + # append false positive + cur_true = np.append(cur_true, 0) + cur_score = np.append(cur_score, min_score) + cur_match = np.append(cur_match, True) + # otherwise set score + else: + found_match = True + cur_match[gti] = True + cur_score[gti] = confidence + pred_visited[pred["uuid"]] = True + if not found_match: + hard_false_negatives += 1 + # remove non-matched ground truth instances + cur_true = cur_true[cur_match] + cur_score = cur_score[cur_match] + + # collect non-matched predictions as false positive + for pred in pred_instances: + found_gt = False + for gt in pred["matched_gt"]: + overlap = float(gt["intersection"]) / ( + gt["vert_count"] + + pred["vert_count"] + - gt["intersection"] + ) + if overlap > overlap_th: + found_gt = True + break + if not found_gt: + num_ignore = pred["void_intersection"] + for gt in pred["matched_gt"]: + if gt["segment_id"] in self.segment_ignore_index: + num_ignore += gt["intersection"] + # small ground truth instances + if ( + gt["vert_count"] < min_region_size + or gt["med_dist"] > distance_thresh + or gt["dist_conf"] < distance_conf + ): + num_ignore += gt["intersection"] + proportion_ignore = ( + float(num_ignore) / pred["vert_count"] + ) + # if not ignored append false positive + if proportion_ignore <= overlap_th: + cur_true = np.append(cur_true, 0) + confidence = pred["confidence"] + cur_score = np.append(cur_score, confidence) + + # append to overall results + y_true = np.append(y_true, cur_true) + y_score = np.append(y_score, cur_score) + + # compute average precision + if has_gt and has_pred: + # compute precision recall curve first + + # sorting and cumsum + score_arg_sort = np.argsort(y_score) + y_score_sorted = y_score[score_arg_sort] + y_true_sorted = y_true[score_arg_sort] + y_true_sorted_cumsum = np.cumsum(y_true_sorted) + + # unique thresholds + (thresholds, unique_indices) = np.unique( + y_score_sorted, return_index=True + ) + num_prec_recall = len(unique_indices) + 1 + + # prepare precision recall + num_examples = len(y_score_sorted) + # https://github.com/ScanNet/ScanNet/pull/26 + # all predictions are non-matched but also all of them are ignored and not counted as FP + # y_true_sorted_cumsum is empty + # num_true_examples = y_true_sorted_cumsum[-1] + num_true_examples = ( + y_true_sorted_cumsum[-1] + if len(y_true_sorted_cumsum) > 0 + else 0 + ) + precision = np.zeros(num_prec_recall) + recall = np.zeros(num_prec_recall) + + # deal with the first point + y_true_sorted_cumsum = np.append(y_true_sorted_cumsum, 0) + # deal with remaining + for idx_res, idx_scores in enumerate(unique_indices): + cumsum = y_true_sorted_cumsum[idx_scores - 1] + tp = num_true_examples - cumsum + fp = num_examples - idx_scores - tp + fn = cumsum + hard_false_negatives + p = float(tp) / (tp + fp) + r = float(tp) / (tp + fn) + precision[idx_res] = p + recall[idx_res] = r + + # first point in curve is artificial + precision[-1] = 1.0 + recall[-1] = 0.0 + + # compute average of precision-recall curve + recall_for_conv = np.copy(recall) + recall_for_conv = np.append(recall_for_conv[0], recall_for_conv) + recall_for_conv = np.append(recall_for_conv, 0.0) + + stepWidths = np.convolve( + recall_for_conv, [-0.5, 0, 0.5], "valid" + ) + # integrate is now simply a dot product + ap_current = np.dot(precision, stepWidths) + + elif has_gt: + ap_current = 0.0 + else: + ap_current = float("nan") + ap_table[di, li, oi] = ap_current + d_inf = 0 + o50 = np.where(np.isclose(self.overlaps, 0.5)) + o25 = np.where(np.isclose(self.overlaps, 0.25)) + oAllBut25 = np.where(np.logical_not(np.isclose(self.overlaps, 0.25))) + ap_scores = dict() + ap_scores["all_ap"] = np.nanmean(ap_table[d_inf, :, oAllBut25]) + ap_scores["all_ap_50%"] = np.nanmean(ap_table[d_inf, :, o50]) + ap_scores["all_ap_25%"] = np.nanmean(ap_table[d_inf, :, o25]) + ap_scores["classes"] = {} + for li, label_name in enumerate(self.valid_class_names): + ap_scores["classes"][label_name] = {} + ap_scores["classes"][label_name]["ap"] = np.average( + ap_table[d_inf, li, oAllBut25] + ) + ap_scores["classes"][label_name]["ap50%"] = np.average( + ap_table[d_inf, li, o50] + ) + ap_scores["classes"][label_name]["ap25%"] = np.average( + ap_table[d_inf, li, o25] + ) + return ap_scores + + def eval(self): + self.trainer.logger.info(">>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>") + self.trainer.model.eval() + scenes = [] + for i, input_dict in enumerate(self.trainer.val_loader): + assert ( + len(input_dict["offset"]) == 1 + ) # currently only support bs 1 for each GPU + for key in input_dict.keys(): + if isinstance(input_dict[key], torch.Tensor): + input_dict[key] = input_dict[key].cuda(non_blocking=True) + with torch.no_grad(): + output_dict = self.trainer.model(input_dict) + + loss = output_dict["loss"] + + segment = input_dict["segment"] + instance = input_dict["instance"] + # map to origin + if "origin_coord" in input_dict.keys(): + idx, _ = pointops.knn_query( + 1, + input_dict["coord"].float(), + input_dict["offset"].int(), + input_dict["origin_coord"].float(), + input_dict["origin_offset"].int(), + ) + idx = idx.cpu().flatten().long() + output_dict["pred_masks"] = output_dict["pred_masks"][:, idx] + segment = input_dict["origin_segment"] + instance = input_dict["origin_instance"] + + gt_instances, pred_instance = self.associate_instances( + output_dict, segment, instance + ) + scenes.append(dict(gt=gt_instances, pred=pred_instance)) + + self.trainer.storage.put_scalar("val_loss", loss.item()) + self.trainer.logger.info( + "Test: [{iter}/{max_iter}] " + "Loss {loss:.4f} ".format( + iter=i + 1, max_iter=len(self.trainer.val_loader), loss=loss.item() + ) + ) + + loss_avg = self.trainer.storage.history("val_loss").avg + comm.synchronize() + scenes_sync = comm.gather(scenes, dst=0) + scenes = [scene for scenes_ in scenes_sync for scene in scenes_] + ap_scores = self.evaluate_matches(scenes) + all_ap = ap_scores["all_ap"] + all_ap_50 = ap_scores["all_ap_50%"] + all_ap_25 = ap_scores["all_ap_25%"] + self.trainer.logger.info( + "Val result: mAP/AP50/AP25 {:.4f}/{:.4f}/{:.4f}.".format( + all_ap, all_ap_50, all_ap_25 + ) + ) + for i, label_name in enumerate(self.valid_class_names): + ap = ap_scores["classes"][label_name]["ap"] + ap_50 = ap_scores["classes"][label_name]["ap50%"] + ap_25 = ap_scores["classes"][label_name]["ap25%"] + self.trainer.logger.info( + "Class_{idx}-{name} Result: AP/AP50/AP25 {AP:.4f}/{AP50:.4f}/{AP25:.4f}".format( + idx=i, name=label_name, AP=ap, AP50=ap_50, AP25=ap_25 + ) + ) + current_epoch = self.trainer.epoch + 1 + if self.trainer.writer is not None: + self.trainer.writer.add_scalar("val/loss", loss_avg, current_epoch) + self.trainer.writer.add_scalar("val/mAP", all_ap, current_epoch) + self.trainer.writer.add_scalar("val/AP50", all_ap_50, current_epoch) + self.trainer.writer.add_scalar("val/AP25", all_ap_25, current_epoch) + self.trainer.logger.info("<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<") + self.trainer.comm_info["current_metric_value"] = all_ap_50 # save for saver + self.trainer.comm_info["current_metric_name"] = "AP50" # save for saver diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/misc.py b/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..155bf5541fc8e5406618a801ba4ccb1e369d4308 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/engines/hooks/misc.py @@ -0,0 +1,464 @@ +""" +Misc Hook + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import sys +import glob +import os +import shutil +import time +import torch +import torch.utils.data +from collections import OrderedDict + +if sys.version_info >= (3, 10): + from collections.abc import Sequence +else: + from collections import Sequence +from pointcept.utils.timer import Timer +from pointcept.utils.comm import is_main_process, synchronize, get_world_size +from pointcept.utils.cache import shared_dict +import pointcept.utils.comm as comm +from pointcept.engines.test import TESTERS + +from .default import HookBase +from .builder import HOOKS + + +@HOOKS.register_module() +class IterationTimer(HookBase): + def __init__(self, warmup_iter=1): + self._warmup_iter = warmup_iter + self._start_time = time.perf_counter() + self._iter_timer = Timer() + self._remain_iter = 0 + + def before_train(self): + self._start_time = time.perf_counter() + self._remain_iter = self.trainer.max_epoch * len(self.trainer.train_loader) + + def before_epoch(self): + self._iter_timer.reset() + + def before_step(self): + data_time = self._iter_timer.seconds() + self.trainer.storage.put_scalar("data_time", data_time) + + def after_step(self): + batch_time = self._iter_timer.seconds() + self._iter_timer.reset() + self.trainer.storage.put_scalar("batch_time", batch_time) + self._remain_iter -= 1 + remain_time = self._remain_iter * self.trainer.storage.history("batch_time").avg + t_m, t_s = divmod(remain_time, 60) + t_h, t_m = divmod(t_m, 60) + remain_time = "{:02d}:{:02d}:{:02d}".format(int(t_h), int(t_m), int(t_s)) + if "iter_info" in self.trainer.comm_info.keys(): + info = ( + "Data {data_time_val:.3f} ({data_time_avg:.3f}) " + "Batch {batch_time_val:.3f} ({batch_time_avg:.3f}) " + "Remain {remain_time} ".format( + data_time_val=self.trainer.storage.history("data_time").val, + data_time_avg=self.trainer.storage.history("data_time").avg, + batch_time_val=self.trainer.storage.history("batch_time").val, + batch_time_avg=self.trainer.storage.history("batch_time").avg, + remain_time=remain_time, + ) + ) + self.trainer.comm_info["iter_info"] += info + if self.trainer.comm_info["iter"] <= self._warmup_iter: + self.trainer.storage.history("data_time").reset() + self.trainer.storage.history("batch_time").reset() + + +@HOOKS.register_module() +class InformationWriter(HookBase): + def __init__(self): + self.curr_iter = 0 + self.model_output_keys = [] + + def before_train(self): + self.trainer.comm_info["iter_info"] = "" + self.curr_iter = self.trainer.start_epoch * len(self.trainer.train_loader) + + def before_step(self): + self.curr_iter += 1 + # MSC pretrain do not have offset information. Comment the code for support MSC + # info = "Train: [{epoch}/{max_epoch}][{iter}/{max_iter}] " \ + # "Scan {batch_size} ({points_num}) ".format( + # epoch=self.trainer.epoch + 1, max_epoch=self.trainer.max_epoch, + # iter=self.trainer.comm_info["iter"], max_iter=len(self.trainer.train_loader), + # batch_size=len(self.trainer.comm_info["input_dict"]["offset"]), + # points_num=self.trainer.comm_info["input_dict"]["offset"][-1] + # ) + info = "Train: [{epoch}/{max_epoch}][{iter}/{max_iter}] ".format( + epoch=self.trainer.epoch + 1, + max_epoch=self.trainer.max_epoch, + iter=self.trainer.comm_info["iter"] + 1, + max_iter=len(self.trainer.train_loader), + ) + self.trainer.comm_info["iter_info"] += info + + def after_step(self): + if "model_output_dict" in self.trainer.comm_info.keys(): + model_output_dict = self.trainer.comm_info["model_output_dict"] + self.model_output_keys = model_output_dict.keys() + for key in self.model_output_keys: + self.trainer.storage.put_scalar(key, model_output_dict[key].item()) + + for key in self.model_output_keys: + self.trainer.comm_info["iter_info"] += "{key}: {value:.4f} ".format( + key=key, value=self.trainer.storage.history(key).val + ) + lr = self.trainer.optimizer.state_dict()["param_groups"][0]["lr"] + self.trainer.comm_info["iter_info"] += "Lr: {lr:.5f}".format(lr=lr) + self.trainer.logger.info(self.trainer.comm_info["iter_info"]) + self.trainer.comm_info["iter_info"] = "" # reset iter info + if self.trainer.writer is not None: + self.trainer.writer.add_scalar("lr", lr, self.curr_iter) + for key in self.model_output_keys: + self.trainer.writer.add_scalar( + "train_batch/" + key, + self.trainer.storage.history(key).val, + self.curr_iter, + ) + + def after_epoch(self): + epoch_info = "Train result: " + for key in self.model_output_keys: + epoch_info += "{key}: {value:.4f} ".format( + key=key, value=self.trainer.storage.history(key).avg + ) + self.trainer.logger.info(epoch_info) + if self.trainer.writer is not None: + for key in self.model_output_keys: + self.trainer.writer.add_scalar( + "train/" + key, + self.trainer.storage.history(key).avg, + self.trainer.epoch + 1, + ) + + +@HOOKS.register_module() +class CheckpointSaver(HookBase): + def __init__(self, save_freq=None): + self.save_freq = save_freq # None or int, None indicate only save model last + + def after_epoch(self): + if is_main_process(): + is_best = False + if self.trainer.cfg.evaluate: + current_metric_value = self.trainer.comm_info["current_metric_value"] + current_metric_name = self.trainer.comm_info["current_metric_name"] + if current_metric_value > self.trainer.best_metric_value: + self.trainer.best_metric_value = current_metric_value + is_best = True + self.trainer.logger.info( + "Best validation {} updated to: {:.4f}".format( + current_metric_name, current_metric_value + ) + ) + self.trainer.logger.info( + "Currently Best {}: {:.4f}".format( + current_metric_name, self.trainer.best_metric_value + ) + ) + + filename = os.path.join( + self.trainer.cfg.save_path, "model", "model_last.pth" + ) + self.trainer.logger.info("Saving checkpoint to: " + filename) + torch.save( + { + "epoch": self.trainer.epoch + 1, + "state_dict": self.trainer.model.state_dict(), + "optimizer": self.trainer.optimizer.state_dict(), + "scheduler": self.trainer.scheduler.state_dict(), + "scaler": ( + self.trainer.scaler.state_dict() + if self.trainer.cfg.enable_amp + else None + ), + "best_metric_value": self.trainer.best_metric_value, + }, + filename + ".tmp", + ) + os.replace(filename + ".tmp", filename) + if is_best: + shutil.copyfile( + filename, + os.path.join(self.trainer.cfg.save_path, "model", "model_best.pth"), + ) + if self.save_freq and (self.trainer.epoch + 1) % self.save_freq == 0: + shutil.copyfile( + filename, + os.path.join( + self.trainer.cfg.save_path, + "model", + f"epoch_{self.trainer.epoch + 1}.pth", + ), + ) + + +@HOOKS.register_module() +class CheckpointLoader(HookBase): + def __init__(self, keywords="", replacement=None, strict=False): + self.keywords = keywords + self.replacement = replacement if replacement is not None else keywords + self.strict = strict + + def before_train(self): + self.trainer.logger.info("=> Loading checkpoint & weight ...") + if self.trainer.cfg.weight and os.path.isfile(self.trainer.cfg.weight): + self.trainer.logger.info(f"Loading weight at: {self.trainer.cfg.weight}") + checkpoint = torch.load( + self.trainer.cfg.weight, + map_location=lambda storage, loc: storage.cuda(), + ) + self.trainer.logger.info( + f"Loading layer weights with keyword: {self.keywords}, " + f"replace keyword with: {self.replacement}" + ) + weight = OrderedDict() + for key, value in checkpoint["state_dict"].items(): + if not key.startswith("module."): + key = "module." + key # xxx.xxx -> module.xxx.xxx + # Now all keys contain "module." no matter DDP or not. + if self.keywords in key: + key = key.replace(self.keywords, self.replacement) + if comm.get_world_size() == 1: + key = key[7:] # module.xxx.xxx -> xxx.xxx + weight[key] = value + load_state_info = self.trainer.model.load_state_dict( + weight, strict=self.strict + ) + self.trainer.logger.info(f"Missing keys: {load_state_info[0]}") + if self.trainer.cfg.resume: + self.trainer.logger.info( + f"Resuming train at eval epoch: {checkpoint['epoch']}" + ) + self.trainer.start_epoch = checkpoint["epoch"] + self.trainer.best_metric_value = checkpoint["best_metric_value"] + self.trainer.optimizer.load_state_dict(checkpoint["optimizer"]) + self.trainer.scheduler.load_state_dict(checkpoint["scheduler"]) + if self.trainer.cfg.enable_amp: + self.trainer.scaler.load_state_dict(checkpoint["scaler"]) + else: + self.trainer.logger.info(f"No weight found at: {self.trainer.cfg.weight}") + + +@HOOKS.register_module() +class PreciseEvaluator(HookBase): + def __init__(self, test_last=False): + self.test_last = test_last + + def after_train(self): + self.trainer.logger.info( + ">>>>>>>>>>>>>>>> Start Precise Evaluation >>>>>>>>>>>>>>>>" + ) + torch.cuda.empty_cache() + cfg = self.trainer.cfg + tester = TESTERS.build( + dict(type=cfg.test.type, cfg=cfg, model=self.trainer.model) + ) + if self.test_last: + self.trainer.logger.info("=> Testing on model_last ...") + else: + self.trainer.logger.info("=> Testing on model_best ...") + best_path = os.path.join( + self.trainer.cfg.save_path, "model", "model_best.pth" + ) + checkpoint = torch.load(best_path) + state_dict = checkpoint["state_dict"] + tester.model.load_state_dict(state_dict, strict=True) + tester.test() + + +@HOOKS.register_module() +class DataCacheOperator(HookBase): + def __init__(self, data_root, split): + self.data_root = data_root + self.split = split + self.data_list = self.get_data_list() + + def get_data_list(self): + if isinstance(self.split, str): + data_list = glob.glob(os.path.join(self.data_root, self.split)) + elif isinstance(self.split, Sequence): + data_list = [] + for split in self.split: + data_list += glob.glob(os.path.join(self.data_root, split)) + else: + raise NotImplementedError + return data_list + + def get_cache_name(self, data_path): + data_name = data_path.replace(os.path.dirname(self.data_root), "") + return "pointcept" + data_name.replace(os.path.sep, "-") + + def before_train(self): + self.trainer.logger.info( + f"=> Caching dataset: {self.data_root}, split: {self.split} ..." + ) + if is_main_process(): + dataset = self.trainer.train_loader.dataset + for i in range(len(dataset)): + data_dict = dataset[i] + name = data_dict["name"] + shared_dict(f"Pointcept-{name}", data_dict) + synchronize() + + +@HOOKS.register_module() +class RuntimeProfiler(HookBase): + def __init__( + self, + forward=True, + backward=True, + interrupt=False, + warm_up=2, + sort_by="cuda_time_total", + row_limit=30, + ): + self.forward = forward + self.backward = backward + self.interrupt = interrupt + self.warm_up = warm_up + self.sort_by = sort_by + self.row_limit = row_limit + + def before_train(self): + self.trainer.logger.info("Profiling runtime ...") + from torch.profiler import profile, record_function, ProfilerActivity + + for i, input_dict in enumerate(self.trainer.train_loader): + if i == self.warm_up + 1: + break + for key in input_dict.keys(): + if isinstance(input_dict[key], torch.Tensor): + input_dict[key] = input_dict[key].cuda(non_blocking=True) + if self.forward: + with profile( + activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], + record_shapes=True, + profile_memory=True, + with_stack=True, + ) as forward_prof: + with record_function("model_inference"): + output_dict = self.trainer.model(input_dict) + else: + output_dict = self.trainer.model(input_dict) + loss = output_dict["loss"] + if self.backward: + with profile( + activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], + record_shapes=True, + profile_memory=True, + with_stack=True, + ) as backward_prof: + with record_function("model_inference"): + loss.backward() + self.trainer.logger.info(f"Profile: [{i + 1}/{self.warm_up + 1}]") + if self.forward: + self.trainer.logger.info( + "Forward profile: \n" + + str( + forward_prof.key_averages().table( + sort_by=self.sort_by, row_limit=self.row_limit + ) + ) + ) + forward_prof.export_chrome_trace( + os.path.join(self.trainer.cfg.save_path, "forward_trace.json") + ) + + if self.backward: + self.trainer.logger.info( + "Backward profile: \n" + + str( + backward_prof.key_averages().table( + sort_by=self.sort_by, row_limit=self.row_limit + ) + ) + ) + backward_prof.export_chrome_trace( + os.path.join(self.trainer.cfg.save_path, "backward_trace.json") + ) + if self.interrupt: + sys.exit(0) + + +@HOOKS.register_module() +class RuntimeProfilerV2(HookBase): + def __init__( + self, + interrupt=False, + wait=1, + warmup=1, + active=10, + repeat=1, + sort_by="cuda_time_total", + row_limit=30, + ): + self.interrupt = interrupt + self.wait = wait + self.warmup = warmup + self.active = active + self.repeat = repeat + self.sort_by = sort_by + self.row_limit = row_limit + + def before_train(self): + self.trainer.logger.info("Profiling runtime ...") + from torch.profiler import ( + profile, + record_function, + ProfilerActivity, + schedule, + tensorboard_trace_handler, + ) + + prof = profile( + activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], + schedule=schedule( + wait=self.wait, + warmup=self.warmup, + active=self.active, + repeat=self.repeat, + ), + on_trace_ready=tensorboard_trace_handler(self.trainer.cfg.save_path), + record_shapes=True, + profile_memory=True, + with_stack=True, + ) + prof.start() + for i, input_dict in enumerate(self.trainer.train_loader): + if i >= (self.wait + self.warmup + self.active) * self.repeat: + break + for key in input_dict.keys(): + if isinstance(input_dict[key], torch.Tensor): + input_dict[key] = input_dict[key].cuda(non_blocking=True) + with record_function("model_forward"): + output_dict = self.trainer.model(input_dict) + loss = output_dict["loss"] + with record_function("model_backward"): + loss.backward() + prof.step() + self.trainer.logger.info( + f"Profile: [{i + 1}/{(self.wait + self.warmup + self.active) * self.repeat}]" + ) + self.trainer.logger.info( + "Profile: \n" + + str( + prof.key_averages().table( + sort_by=self.sort_by, row_limit=self.row_limit + ) + ) + ) + prof.stop() + + if self.interrupt: + sys.exit(0) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/engines/launch.py b/submodules/PointTransformerV3/Pointcept/pointcept/engines/launch.py new file mode 100644 index 0000000000000000000000000000000000000000..99a8351fe5ab4393c1fab75c3bd546ba66641986 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/engines/launch.py @@ -0,0 +1,137 @@ +""" +Launcher + +modified from detectron2(https://github.com/facebookresearch/detectron2) + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import logging +from datetime import timedelta +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from pointcept.utils import comm + +__all__ = ["DEFAULT_TIMEOUT", "launch"] + +DEFAULT_TIMEOUT = timedelta(minutes=60) + + +def _find_free_port(): + import socket + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Binding to port 0 will cause the OS to find an available port for us + sock.bind(("", 0)) + port = sock.getsockname()[1] + sock.close() + # NOTE: there is still a chance the port could be taken by other processes. + return port + + +def launch( + main_func, + num_gpus_per_machine, + num_machines=1, + machine_rank=0, + dist_url=None, + cfg=(), + timeout=DEFAULT_TIMEOUT, +): + """ + Launch multi-gpu or distributed training. + This function must be called on all machines involved in the training. + It will spawn child processes (defined by ``num_gpus_per_machine``) on each machine. + Args: + main_func: a function that will be called by `main_func(*args)` + num_gpus_per_machine (int): number of GPUs per machine + num_machines (int): the total number of machines + machine_rank (int): the rank of this machine + dist_url (str): url to connect to for distributed jobs, including protocol + e.g. "tcp://127.0.0.1:8686". + Can be set to "auto" to automatically select a free port on localhost + timeout (timedelta): timeout of the distributed workers + args (tuple): arguments passed to main_func + """ + world_size = num_machines * num_gpus_per_machine + if world_size > 1: + if dist_url == "auto": + assert ( + num_machines == 1 + ), "dist_url=auto not supported in multi-machine jobs." + port = _find_free_port() + dist_url = f"tcp://127.0.0.1:{port}" + if num_machines > 1 and dist_url.startswith("file://"): + logger = logging.getLogger(__name__) + logger.warning( + "file:// is not a reliable init_method in multi-machine jobs. Prefer tcp://" + ) + + mp.spawn( + _distributed_worker, + nprocs=num_gpus_per_machine, + args=( + main_func, + world_size, + num_gpus_per_machine, + machine_rank, + dist_url, + cfg, + timeout, + ), + daemon=False, + ) + else: + main_func(*cfg) + + +def _distributed_worker( + local_rank, + main_func, + world_size, + num_gpus_per_machine, + machine_rank, + dist_url, + cfg, + timeout=DEFAULT_TIMEOUT, +): + assert ( + torch.cuda.is_available() + ), "cuda is not available. Please check your installation." + global_rank = machine_rank * num_gpus_per_machine + local_rank + try: + dist.init_process_group( + backend="NCCL", + init_method=dist_url, + world_size=world_size, + rank=global_rank, + timeout=timeout, + ) + except Exception as e: + logger = logging.getLogger(__name__) + logger.error("Process group URL: {}".format(dist_url)) + raise e + + # Setup the local process group (which contains ranks within the same machine) + assert comm._LOCAL_PROCESS_GROUP is None + num_machines = world_size // num_gpus_per_machine + for i in range(num_machines): + ranks_on_i = list( + range(i * num_gpus_per_machine, (i + 1) * num_gpus_per_machine) + ) + pg = dist.new_group(ranks_on_i) + if i == machine_rank: + comm._LOCAL_PROCESS_GROUP = pg + + assert num_gpus_per_machine <= torch.cuda.device_count() + torch.cuda.set_device(local_rank) + + # synchronize is needed here to prevent a possible timeout after calling init_process_group + # See: https://github.com/facebookresearch/maskrcnn-benchmark/issues/172 + comm.synchronize() + + main_func(*cfg) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/engines/test.py b/submodules/PointTransformerV3/Pointcept/pointcept/engines/test.py new file mode 100644 index 0000000000000000000000000000000000000000..d74872d8c18382ca44b09211c2f3def70fc8dfe7 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/engines/test.py @@ -0,0 +1,640 @@ +""" +Tester + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import time +import numpy as np +from collections import OrderedDict +import torch +import torch.distributed as dist +import torch.nn.functional as F +import torch.utils.data + +from .defaults import create_ddp_model +import pointcept.utils.comm as comm +from pointcept.datasets import build_dataset, collate_fn +from pointcept.models import build_model +from pointcept.utils.logger import get_root_logger +from pointcept.utils.registry import Registry +from pointcept.utils.misc import ( + AverageMeter, + intersection_and_union, + intersection_and_union_gpu, + make_dirs, +) + + +TESTERS = Registry("testers") + + +class TesterBase: + def __init__(self, cfg, model=None, test_loader=None, verbose=False) -> None: + torch.multiprocessing.set_sharing_strategy("file_system") + self.logger = get_root_logger( + log_file=os.path.join(cfg.save_path, "test.log"), + file_mode="a" if cfg.resume else "w", + ) + self.logger.info("=> Loading config ...") + self.cfg = cfg + self.verbose = verbose + if self.verbose: + self.logger.info(f"Save path: {cfg.save_path}") + self.logger.info(f"Config:\n{cfg.pretty_text}") + if model is None: + self.logger.info("=> Building model ...") + self.model = self.build_model() + else: + self.model = model + if test_loader is None: + self.logger.info("=> Building test dataset & dataloader ...") + self.test_loader = self.build_test_loader() + else: + self.test_loader = test_loader + + def build_model(self): + model = build_model(self.cfg.model) + n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) + self.logger.info(f"Num params: {n_parameters}") + model = create_ddp_model( + model.cuda(), + broadcast_buffers=False, + find_unused_parameters=self.cfg.find_unused_parameters, + ) + if os.path.isfile(self.cfg.weight): + self.logger.info(f"Loading weight at: {self.cfg.weight}") + checkpoint = torch.load(self.cfg.weight) + weight = OrderedDict() + for key, value in checkpoint["state_dict"].items(): + if key.startswith("module."): + if comm.get_world_size() == 1: + key = key[7:] # module.xxx.xxx -> xxx.xxx + else: + if comm.get_world_size() > 1: + key = "module." + key # xxx.xxx -> module.xxx.xxx + weight[key] = value + model.load_state_dict(weight, strict=True) + self.logger.info( + "=> Loaded weight '{}' (epoch {})".format( + self.cfg.weight, checkpoint["epoch"] + ) + ) + else: + raise RuntimeError("=> No checkpoint found at '{}'".format(self.cfg.weight)) + return model + + def build_test_loader(self): + test_dataset = build_dataset(self.cfg.data.test) + if comm.get_world_size() > 1: + test_sampler = torch.utils.data.distributed.DistributedSampler(test_dataset) + else: + test_sampler = None + test_loader = torch.utils.data.DataLoader( + test_dataset, + batch_size=self.cfg.batch_size_test_per_gpu, + shuffle=False, + num_workers=self.cfg.batch_size_test_per_gpu, + pin_memory=True, + sampler=test_sampler, + collate_fn=self.__class__.collate_fn, + ) + return test_loader + + def test(self): + raise NotImplementedError + + @staticmethod + def collate_fn(batch): + raise collate_fn(batch) + + +@TESTERS.register_module() +class SemSegTester(TesterBase): + def test(self): + assert self.test_loader.batch_size == 1 + logger = get_root_logger() + logger.info(">>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>") + + batch_time = AverageMeter() + intersection_meter = AverageMeter() + union_meter = AverageMeter() + target_meter = AverageMeter() + self.model.eval() + + save_path = os.path.join(self.cfg.save_path, "result") + make_dirs(save_path) + # create submit folder only on main process + if ( + self.cfg.data.test.type == "ScanNetDataset" + or self.cfg.data.test.type == "ScanNet200Dataset" + or self.cfg.data.test.type == "ScanNetPPDataset" + ) and comm.is_main_process(): + make_dirs(os.path.join(save_path, "submit")) + elif ( + self.cfg.data.test.type == "SemanticKITTIDataset" and comm.is_main_process() + ): + make_dirs(os.path.join(save_path, "submit")) + elif self.cfg.data.test.type == "NuScenesDataset" and comm.is_main_process(): + import json + + make_dirs(os.path.join(save_path, "submit", "lidarseg", "test")) + make_dirs(os.path.join(save_path, "submit", "test")) + submission = dict( + meta=dict( + use_camera=False, + use_lidar=True, + use_radar=False, + use_map=False, + use_external=False, + ) + ) + with open( + os.path.join(save_path, "submit", "test", "submission.json"), "w" + ) as f: + json.dump(submission, f, indent=4) + comm.synchronize() + record = {} + # fragment inference + for idx, data_dict in enumerate(self.test_loader): + end = time.time() + data_dict = data_dict[0] # current assume batch size is 1 + fragment_list = data_dict.pop("fragment_list") + segment = data_dict.pop("segment") + data_name = data_dict.pop("name") + pred_save_path = os.path.join(save_path, "{}_pred.npy".format(data_name)) + if os.path.isfile(pred_save_path): + logger.info( + "{}/{}: {}, loaded pred and label.".format( + idx + 1, len(self.test_loader), data_name + ) + ) + pred = np.load(pred_save_path) + if "origin_segment" in data_dict.keys(): + segment = data_dict["origin_segment"] + else: + pred = torch.zeros((segment.size, self.cfg.data.num_classes)).cuda() + for i in range(len(fragment_list)): + fragment_batch_size = 1 + s_i, e_i = i * fragment_batch_size, min( + (i + 1) * fragment_batch_size, len(fragment_list) + ) + input_dict = collate_fn(fragment_list[s_i:e_i]) + for key in input_dict.keys(): + if isinstance(input_dict[key], torch.Tensor): + input_dict[key] = input_dict[key].cuda(non_blocking=True) + idx_part = input_dict["index"] + with torch.no_grad(): + pred_part = self.model(input_dict)["seg_logits"] # (n, k) + pred_part = F.softmax(pred_part, -1) + if self.cfg.empty_cache: + torch.cuda.empty_cache() + bs = 0 + for be in input_dict["offset"]: + pred[idx_part[bs:be], :] += pred_part[bs:be] + bs = be + + logger.info( + "Test: {}/{}-{data_name}, Batch: {batch_idx}/{batch_num}".format( + idx + 1, + len(self.test_loader), + data_name=data_name, + batch_idx=i, + batch_num=len(fragment_list), + ) + ) + if self.cfg.data.test.type == "ScanNetPPDataset": + pred = pred.topk(3, dim=1)[1].data.cpu().numpy() + else: + pred = pred.max(1)[1].data.cpu().numpy() + if "origin_segment" in data_dict.keys(): + assert "inverse" in data_dict.keys() + pred = pred[data_dict["inverse"]] + segment = data_dict["origin_segment"] + np.save(pred_save_path, pred) + if ( + self.cfg.data.test.type == "ScanNetDataset" + or self.cfg.data.test.type == "ScanNet200Dataset" + ): + np.savetxt( + os.path.join(save_path, "submit", "{}.txt".format(data_name)), + self.test_loader.dataset.class2id[pred].reshape([-1, 1]), + fmt="%d", + ) + elif self.cfg.data.test.type == "ScanNetPPDataset": + np.savetxt( + os.path.join(save_path, "submit", "{}.txt".format(data_name)), + pred.astype(np.int32), + delimiter=",", + fmt="%d", + ) + pred = pred[:, 0] # for mIoU, TODO: support top3 mIoU + elif self.cfg.data.test.type == "SemanticKITTIDataset": + # 00_000000 -> 00, 000000 + sequence_name, frame_name = data_name.split("_") + os.makedirs( + os.path.join( + save_path, "submit", "sequences", sequence_name, "predictions" + ), + exist_ok=True, + ) + submit = pred.astype(np.uint32) + submit = np.vectorize( + self.test_loader.dataset.learning_map_inv.__getitem__ + )(submit).astype(np.uint32) + submit.tofile( + os.path.join( + save_path, + "submit", + "sequences", + sequence_name, + "predictions", + f"{frame_name}.label", + ) + ) + elif self.cfg.data.test.type == "NuScenesDataset": + np.array(pred + 1).astype(np.uint8).tofile( + os.path.join( + save_path, + "submit", + "lidarseg", + "test", + "{}_lidarseg.bin".format(data_name), + ) + ) + + intersection, union, target = intersection_and_union( + pred, segment, self.cfg.data.num_classes, self.cfg.data.ignore_index + ) + intersection_meter.update(intersection) + union_meter.update(union) + target_meter.update(target) + record[data_name] = dict( + intersection=intersection, union=union, target=target + ) + + mask = union != 0 + iou_class = intersection / (union + 1e-10) + iou = np.mean(iou_class[mask]) + acc = sum(intersection) / (sum(target) + 1e-10) + + m_iou = np.mean(intersection_meter.sum / (union_meter.sum + 1e-10)) + m_acc = np.mean(intersection_meter.sum / (target_meter.sum + 1e-10)) + + batch_time.update(time.time() - end) + logger.info( + "Test: {} [{}/{}]-{} " + "Batch {batch_time.val:.3f} ({batch_time.avg:.3f}) " + "Accuracy {acc:.4f} ({m_acc:.4f}) " + "mIoU {iou:.4f} ({m_iou:.4f})".format( + data_name, + idx + 1, + len(self.test_loader), + segment.size, + batch_time=batch_time, + acc=acc, + m_acc=m_acc, + iou=iou, + m_iou=m_iou, + ) + ) + + logger.info("Syncing ...") + comm.synchronize() + record_sync = comm.gather(record, dst=0) + + if comm.is_main_process(): + record = {} + for _ in range(len(record_sync)): + r = record_sync.pop() + record.update(r) + del r + intersection = np.sum( + [meters["intersection"] for _, meters in record.items()], axis=0 + ) + union = np.sum([meters["union"] for _, meters in record.items()], axis=0) + target = np.sum([meters["target"] for _, meters in record.items()], axis=0) + + if self.cfg.data.test.type == "S3DISDataset": + torch.save( + dict(intersection=intersection, union=union, target=target), + os.path.join(save_path, f"{self.test_loader.dataset.split}.pth"), + ) + + iou_class = intersection / (union + 1e-10) + accuracy_class = intersection / (target + 1e-10) + mIoU = np.mean(iou_class) + mAcc = np.mean(accuracy_class) + allAcc = sum(intersection) / (sum(target) + 1e-10) + + logger.info( + "Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}".format( + mIoU, mAcc, allAcc + ) + ) + for i in range(self.cfg.data.num_classes): + logger.info( + "Class_{idx} - {name} Result: iou/accuracy {iou:.4f}/{accuracy:.4f}".format( + idx=i, + name=self.cfg.data.names[i], + iou=iou_class[i], + accuracy=accuracy_class[i], + ) + ) + logger.info("<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<") + + @staticmethod + def collate_fn(batch): + return batch + + +@TESTERS.register_module() +class ClsTester(TesterBase): + def test(self): + logger = get_root_logger() + logger.info(">>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>") + batch_time = AverageMeter() + intersection_meter = AverageMeter() + union_meter = AverageMeter() + target_meter = AverageMeter() + self.model.eval() + + for i, input_dict in enumerate(self.test_loader): + for key in input_dict.keys(): + if isinstance(input_dict[key], torch.Tensor): + input_dict[key] = input_dict[key].cuda(non_blocking=True) + end = time.time() + with torch.no_grad(): + output_dict = self.model(input_dict) + output = output_dict["cls_logits"] + pred = output.max(1)[1] + label = input_dict["category"] + intersection, union, target = intersection_and_union_gpu( + pred, label, self.cfg.data.num_classes, self.cfg.data.ignore_index + ) + if comm.get_world_size() > 1: + dist.all_reduce(intersection), dist.all_reduce(union), dist.all_reduce( + target + ) + intersection, union, target = ( + intersection.cpu().numpy(), + union.cpu().numpy(), + target.cpu().numpy(), + ) + intersection_meter.update(intersection), union_meter.update( + union + ), target_meter.update(target) + + accuracy = sum(intersection_meter.val) / (sum(target_meter.val) + 1e-10) + batch_time.update(time.time() - end) + + logger.info( + "Test: [{}/{}] " + "Batch {batch_time.val:.3f} ({batch_time.avg:.3f}) " + "Accuracy {accuracy:.4f} ".format( + i + 1, + len(self.test_loader), + batch_time=batch_time, + accuracy=accuracy, + ) + ) + + iou_class = intersection_meter.sum / (union_meter.sum + 1e-10) + accuracy_class = intersection_meter.sum / (target_meter.sum + 1e-10) + mIoU = np.mean(iou_class) + mAcc = np.mean(accuracy_class) + allAcc = sum(intersection_meter.sum) / (sum(target_meter.sum) + 1e-10) + logger.info( + "Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.".format( + mIoU, mAcc, allAcc + ) + ) + + for i in range(self.cfg.data.num_classes): + logger.info( + "Class_{idx} - {name} Result: iou/accuracy {iou:.4f}/{accuracy:.4f}".format( + idx=i, + name=self.cfg.data.names[i], + iou=iou_class[i], + accuracy=accuracy_class[i], + ) + ) + logger.info("<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<") + + @staticmethod + def collate_fn(batch): + return collate_fn(batch) + + +@TESTERS.register_module() +class ClsVotingTester(TesterBase): + def __init__( + self, + num_repeat=100, + metric="allAcc", + **kwargs, + ): + super().__init__(**kwargs) + self.num_repeat = num_repeat + self.metric = metric + self.best_idx = 0 + self.best_record = None + self.best_metric = 0 + + def test(self): + for i in range(self.num_repeat): + logger = get_root_logger() + logger.info(f">>>>>>>>>>>>>>>> Start Evaluation {i + 1} >>>>>>>>>>>>>>>>") + record = self.test_once() + if comm.is_main_process(): + if record[self.metric] > self.best_metric: + self.best_record = record + self.best_idx = i + self.best_metric = record[self.metric] + info = f"Current best record is Evaluation {i + 1}: " + for m in self.best_record.keys(): + info += f"{m}: {self.best_record[m]:.4f} " + logger.info(info) + + def test_once(self): + logger = get_root_logger() + batch_time = AverageMeter() + intersection_meter = AverageMeter() + target_meter = AverageMeter() + record = {} + self.model.eval() + + for idx, data_dict in enumerate(self.test_loader): + end = time.time() + data_dict = data_dict[0] # current assume batch size is 1 + voting_list = data_dict.pop("voting_list") + category = data_dict.pop("category") + data_name = data_dict.pop("name") + # pred = torch.zeros([1, self.cfg.data.num_classes]).cuda() + # for i in range(len(voting_list)): + # input_dict = voting_list[i] + # for key in input_dict.keys(): + # if isinstance(input_dict[key], torch.Tensor): + # input_dict[key] = input_dict[key].cuda(non_blocking=True) + # with torch.no_grad(): + # pred += F.softmax(self.model(input_dict)["cls_logits"], -1) + input_dict = collate_fn(voting_list) + for key in input_dict.keys(): + if isinstance(input_dict[key], torch.Tensor): + input_dict[key] = input_dict[key].cuda(non_blocking=True) + with torch.no_grad(): + pred = F.softmax(self.model(input_dict)["cls_logits"], -1).sum( + 0, keepdim=True + ) + pred = pred.max(1)[1].cpu().numpy() + intersection, union, target = intersection_and_union( + pred, category, self.cfg.data.num_classes, self.cfg.data.ignore_index + ) + intersection_meter.update(intersection) + target_meter.update(target) + record[data_name] = dict(intersection=intersection, target=target) + acc = sum(intersection) / (sum(target) + 1e-10) + m_acc = np.mean(intersection_meter.sum / (target_meter.sum + 1e-10)) + batch_time.update(time.time() - end) + logger.info( + "Test: {} [{}/{}] " + "Batch {batch_time.val:.3f} ({batch_time.avg:.3f}) " + "Accuracy {acc:.4f} ({m_acc:.4f}) ".format( + data_name, + idx + 1, + len(self.test_loader), + batch_time=batch_time, + acc=acc, + m_acc=m_acc, + ) + ) + + logger.info("Syncing ...") + comm.synchronize() + record_sync = comm.gather(record, dst=0) + + if comm.is_main_process(): + record = {} + for _ in range(len(record_sync)): + r = record_sync.pop() + record.update(r) + del r + intersection = np.sum( + [meters["intersection"] for _, meters in record.items()], axis=0 + ) + target = np.sum([meters["target"] for _, meters in record.items()], axis=0) + accuracy_class = intersection / (target + 1e-10) + mAcc = np.mean(accuracy_class) + allAcc = sum(intersection) / (sum(target) + 1e-10) + + logger.info("Val result: mAcc/allAcc {:.4f}/{:.4f}".format(mAcc, allAcc)) + for i in range(self.cfg.data.num_classes): + logger.info( + "Class_{idx} - {name} Result: iou/accuracy {accuracy:.4f}".format( + idx=i, + name=self.cfg.data.names[i], + accuracy=accuracy_class[i], + ) + ) + return dict(mAcc=mAcc, allAcc=allAcc) + + @staticmethod + def collate_fn(batch): + return batch + + +@TESTERS.register_module() +class PartSegTester(TesterBase): + def test(self): + test_dataset = self.test_loader.dataset + logger = get_root_logger() + logger.info(">>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>") + + batch_time = AverageMeter() + + num_categories = len(self.test_loader.dataset.categories) + iou_category, iou_count = np.zeros(num_categories), np.zeros(num_categories) + self.model.eval() + + save_path = os.path.join( + self.cfg.save_path, "result", "test_epoch{}".format(self.cfg.test_epoch) + ) + make_dirs(save_path) + + for idx in range(len(test_dataset)): + end = time.time() + data_name = test_dataset.get_data_name(idx) + + data_dict_list, label = test_dataset[idx] + pred = torch.zeros((label.size, self.cfg.data.num_classes)).cuda() + batch_num = int(np.ceil(len(data_dict_list) / self.cfg.batch_size_test)) + for i in range(batch_num): + s_i, e_i = i * self.cfg.batch_size_test, min( + (i + 1) * self.cfg.batch_size_test, len(data_dict_list) + ) + input_dict = collate_fn(data_dict_list[s_i:e_i]) + for key in input_dict.keys(): + if isinstance(input_dict[key], torch.Tensor): + input_dict[key] = input_dict[key].cuda(non_blocking=True) + with torch.no_grad(): + pred_part = self.model(input_dict)["cls_logits"] + pred_part = F.softmax(pred_part, -1) + if self.cfg.empty_cache: + torch.cuda.empty_cache() + pred_part = pred_part.reshape(-1, label.size, self.cfg.data.num_classes) + pred = pred + pred_part.total(dim=0) + logger.info( + "Test: {} {}/{}, Batch: {batch_idx}/{batch_num}".format( + data_name, + idx + 1, + len(test_dataset), + batch_idx=i, + batch_num=batch_num, + ) + ) + pred = pred.max(1)[1].data.cpu().numpy() + + category_index = data_dict_list[0]["cls_token"] + category = self.test_loader.dataset.categories[category_index] + parts_idx = self.test_loader.dataset.category2part[category] + parts_iou = np.zeros(len(parts_idx)) + for j, part in enumerate(parts_idx): + if (np.sum(label == part) == 0) and (np.sum(pred == part) == 0): + parts_iou[j] = 1.0 + else: + i = (label == part) & (pred == part) + u = (label == part) | (pred == part) + parts_iou[j] = np.sum(i) / (np.sum(u) + 1e-10) + iou_category[category_index] += parts_iou.mean() + iou_count[category_index] += 1 + + batch_time.update(time.time() - end) + logger.info( + "Test: {} [{}/{}] " + "Batch {batch_time.val:.3f} " + "({batch_time.avg:.3f}) ".format( + data_name, idx + 1, len(self.test_loader), batch_time=batch_time + ) + ) + + ins_mIoU = iou_category.sum() / (iou_count.sum() + 1e-10) + cat_mIoU = (iou_category / (iou_count + 1e-10)).mean() + logger.info( + "Val result: ins.mIoU/cat.mIoU {:.4f}/{:.4f}.".format(ins_mIoU, cat_mIoU) + ) + for i in range(num_categories): + logger.info( + "Class_{idx}-{name} Result: iou_cat/num_sample {iou_cat:.4f}/{iou_count:.4f}".format( + idx=i, + name=self.test_loader.dataset.categories[i], + iou_cat=iou_category[i] / (iou_count[i] + 1e-10), + iou_count=int(iou_count[i]), + ) + ) + logger.info("<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<") + + @staticmethod + def collate_fn(batch): + return collate_fn(batch) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/engines/train.py b/submodules/PointTransformerV3/Pointcept/pointcept/engines/train.py new file mode 100644 index 0000000000000000000000000000000000000000..11c543fdfe754c5473503a05f628447b7afed502 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/engines/train.py @@ -0,0 +1,318 @@ +""" +Trainer + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import sys +import weakref +import torch +import torch.nn as nn +import torch.utils.data +from functools import partial + +if sys.version_info >= (3, 10): + from collections.abc import Iterator +else: + from collections import Iterator +from tensorboardX import SummaryWriter + +from .defaults import create_ddp_model, worker_init_fn +from .hooks import HookBase, build_hooks +import pointcept.utils.comm as comm +from pointcept.datasets import build_dataset, point_collate_fn, collate_fn +from pointcept.models import build_model +from pointcept.utils.logger import get_root_logger +from pointcept.utils.optimizer import build_optimizer +from pointcept.utils.scheduler import build_scheduler +from pointcept.utils.events import EventStorage, ExceptionWriter +from pointcept.utils.registry import Registry + + +TRAINERS = Registry("trainers") + + +class TrainerBase: + def __init__(self) -> None: + self.hooks = [] + self.epoch = 0 + self.start_epoch = 0 + self.max_epoch = 0 + self.max_iter = 0 + self.comm_info = dict() + self.data_iterator: Iterator = enumerate([]) + self.storage: EventStorage + self.writer: SummaryWriter + + def register_hooks(self, hooks) -> None: + hooks = build_hooks(hooks) + for h in hooks: + assert isinstance(h, HookBase) + # To avoid circular reference, hooks and trainer cannot own each other. + # This normally does not matter, but will cause memory leak if the + # involved objects contain __del__: + # See http://engineering.hearsaysocial.com/2013/06/16/circular-references-in-python/ + h.trainer = weakref.proxy(self) + self.hooks.extend(hooks) + + def train(self): + with EventStorage() as self.storage: + # => before train + self.before_train() + for self.epoch in range(self.start_epoch, self.max_epoch): + # => before epoch + self.before_epoch() + # => run_epoch + for ( + self.comm_info["iter"], + self.comm_info["input_dict"], + ) in self.data_iterator: + # => before_step + self.before_step() + # => run_step + self.run_step() + # => after_step + self.after_step() + # => after epoch + self.after_epoch() + # => after train + self.after_train() + + def before_train(self): + for h in self.hooks: + h.before_train() + + def before_epoch(self): + for h in self.hooks: + h.before_epoch() + + def before_step(self): + for h in self.hooks: + h.before_step() + + def run_step(self): + raise NotImplementedError + + def after_step(self): + for h in self.hooks: + h.after_step() + + def after_epoch(self): + for h in self.hooks: + h.after_epoch() + self.storage.reset_histories() + + def after_train(self): + # Sync GPU before running train hooks + comm.synchronize() + for h in self.hooks: + h.after_train() + if comm.is_main_process(): + self.writer.close() + + +@TRAINERS.register_module("DefaultTrainer") +class Trainer(TrainerBase): + def __init__(self, cfg): + super(Trainer, self).__init__() + self.epoch = 0 + self.start_epoch = 0 + self.max_epoch = cfg.eval_epoch + self.best_metric_value = -torch.inf + self.logger = get_root_logger( + log_file=os.path.join(cfg.save_path, "train.log"), + file_mode="a" if cfg.resume else "w", + ) + self.logger.info("=> Loading config ...") + self.cfg = cfg + self.logger.info(f"Save path: {cfg.save_path}") + self.logger.info(f"Config:\n{cfg.pretty_text}") + self.logger.info("=> Building model ...") + self.model = self.build_model() + self.logger.info("=> Building writer ...") + self.writer = self.build_writer() + self.logger.info("=> Building train dataset & dataloader ...") + self.train_loader = self.build_train_loader() + self.logger.info("=> Building val dataset & dataloader ...") + self.val_loader = self.build_val_loader() + self.logger.info("=> Building optimize, scheduler, scaler(amp) ...") + self.optimizer = self.build_optimizer() + self.scheduler = self.build_scheduler() + self.scaler = self.build_scaler() + self.logger.info("=> Building hooks ...") + self.register_hooks(self.cfg.hooks) + + def train(self): + with EventStorage() as self.storage, ExceptionWriter(): + # => before train + self.before_train() + self.logger.info(">>>>>>>>>>>>>>>> Start Training >>>>>>>>>>>>>>>>") + for self.epoch in range(self.start_epoch, self.max_epoch): + # => before epoch + # TODO: optimize to iteration based + if comm.get_world_size() > 1: + self.train_loader.sampler.set_epoch(self.epoch) + self.model.train() + self.data_iterator = enumerate(self.train_loader) + self.before_epoch() + # => run_epoch + for ( + self.comm_info["iter"], + self.comm_info["input_dict"], + ) in self.data_iterator: + # => before_step + self.before_step() + # => run_step + self.run_step() + # => after_step + self.after_step() + # => after epoch + self.after_epoch() + # => after train + self.after_train() + + def run_step(self): + input_dict = self.comm_info["input_dict"] + for key in input_dict.keys(): + if isinstance(input_dict[key], torch.Tensor): + input_dict[key] = input_dict[key].cuda(non_blocking=True) + with torch.cuda.amp.autocast(enabled=self.cfg.enable_amp): + output_dict = self.model(input_dict) + loss = output_dict["loss"] + self.optimizer.zero_grad() + if self.cfg.enable_amp: + self.scaler.scale(loss).backward() + self.scaler.unscale_(self.optimizer) + if self.cfg.clip_grad is not None: + torch.nn.utils.clip_grad_norm_( + self.model.parameters(), self.cfg.clip_grad + ) + self.scaler.step(self.optimizer) + + # When enable amp, optimizer.step call are skipped if the loss scaling factor is too large. + # Fix torch warning scheduler step before optimizer step. + scaler = self.scaler.get_scale() + self.scaler.update() + if scaler <= self.scaler.get_scale(): + self.scheduler.step() + else: + loss.backward() + if self.cfg.clip_grad is not None: + torch.nn.utils.clip_grad_norm_( + self.model.parameters(), self.cfg.clip_grad + ) + self.optimizer.step() + self.scheduler.step() + if self.cfg.empty_cache: + torch.cuda.empty_cache() + self.comm_info["model_output_dict"] = output_dict + + def after_epoch(self): + for h in self.hooks: + h.after_epoch() + self.storage.reset_histories() + if self.cfg.empty_cache_per_epoch: + torch.cuda.empty_cache() + + def build_model(self): + model = build_model(self.cfg.model) + if self.cfg.sync_bn: + model = nn.SyncBatchNorm.convert_sync_batchnorm(model) + n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) + # logger.info(f"Model: \n{self.model}") + self.logger.info(f"Num params: {n_parameters}") + model = create_ddp_model( + model.cuda(), + broadcast_buffers=False, + find_unused_parameters=self.cfg.find_unused_parameters, + ) + return model + + def build_writer(self): + writer = SummaryWriter(self.cfg.save_path) if comm.is_main_process() else None + self.logger.info(f"Tensorboard writer logging dir: {self.cfg.save_path}") + return writer + + def build_train_loader(self): + train_data = build_dataset(self.cfg.data.train) + + if comm.get_world_size() > 1: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_data) + else: + train_sampler = None + + init_fn = ( + partial( + worker_init_fn, + num_workers=self.cfg.num_worker_per_gpu, + rank=comm.get_rank(), + seed=self.cfg.seed, + ) + if self.cfg.seed is not None + else None + ) + + train_loader = torch.utils.data.DataLoader( + train_data, + batch_size=self.cfg.batch_size_per_gpu, + shuffle=(train_sampler is None), + num_workers=self.cfg.num_worker_per_gpu, + sampler=train_sampler, + collate_fn=partial(point_collate_fn, mix_prob=self.cfg.mix_prob), + pin_memory=True, + worker_init_fn=init_fn, + drop_last=True, + persistent_workers=True, + ) + return train_loader + + def build_val_loader(self): + val_loader = None + if self.cfg.evaluate: + val_data = build_dataset(self.cfg.data.val) + if comm.get_world_size() > 1: + val_sampler = torch.utils.data.distributed.DistributedSampler(val_data) + else: + val_sampler = None + val_loader = torch.utils.data.DataLoader( + val_data, + batch_size=self.cfg.batch_size_val_per_gpu, + shuffle=False, + num_workers=self.cfg.num_worker_per_gpu, + pin_memory=True, + sampler=val_sampler, + collate_fn=collate_fn, + ) + return val_loader + + def build_optimizer(self): + return build_optimizer(self.cfg.optimizer, self.model, self.cfg.param_dicts) + + def build_scheduler(self): + assert hasattr(self, "optimizer") + assert hasattr(self, "train_loader") + self.cfg.scheduler.total_steps = len(self.train_loader) * self.cfg.eval_epoch + return build_scheduler(self.cfg.scheduler, self.optimizer) + + def build_scaler(self): + scaler = torch.cuda.amp.GradScaler() if self.cfg.enable_amp else None + return scaler + + +@TRAINERS.register_module("MultiDatasetTrainer") +class MultiDatasetTrainer(Trainer): + def build_train_loader(self): + from pointcept.datasets import MultiDatasetDataloader + + train_data = build_dataset(self.cfg.data.train) + train_loader = MultiDatasetDataloader( + train_data, + self.cfg.batch_size_per_gpu, + self.cfg.num_worker_per_gpu, + self.cfg.mix_prob, + self.cfg.seed, + ) + self.comm_info["iter_per_epoch"] = len(train_loader) + return train_loader diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..aea1d44143a64992d72c098bf9fecdbaf6d2f030 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/__init__.py @@ -0,0 +1,24 @@ +from .builder import build_model +from .default import DefaultSegmentor, DefaultClassifier + +# Backbones +from .sparse_unet import * +from .point_transformer import * +from .point_transformer_v2 import * +from .point_transformer_v3 import * +from .stratified_transformer import * +from .spvcnn import * +from .octformer import * +from .oacnns import * + +# from .swin3d import * + +# Semantic Segmentation +from .context_aware_classifier import * + +# Instance Segmentation +from .point_group import * + +# Pretraining +from .masked_scene_contrast import * +from .point_prompt_training import * diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/builder.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..bbda24465a405a5a2094f8d0c420a53c50fe79cc --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/builder.py @@ -0,0 +1,16 @@ +""" +Model Builder + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from pointcept.utils.registry import Registry + +MODELS = Registry("models") +MODULES = Registry("modules") + + +def build_model(cfg): + """Build models.""" + return MODELS.build(cfg) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/context_aware_classifier/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/context_aware_classifier/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8327900bbf92258fa242f4b99d5b235bc1ae44aa --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/context_aware_classifier/__init__.py @@ -0,0 +1 @@ +from .context_aware_classifier_v1m1_base import CACSegmentor diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/context_aware_classifier/context_aware_classifier_v1m1_base.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/context_aware_classifier/context_aware_classifier_v1m1_base.py new file mode 100644 index 0000000000000000000000000000000000000000..893f4c8fc6d68a1fb7dc24e1ceb3b3ace11ebec6 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/context_aware_classifier/context_aware_classifier_v1m1_base.py @@ -0,0 +1,275 @@ +""" +Context-aware Classifier for Semantic Segmentation + +Author: Zhuotao Tian, Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import torch +import torch.nn as nn +import torch.nn.functional as F +from pointcept.models.losses import build_criteria +from pointcept.models.utils.structure import Point +from pointcept.models.builder import MODELS, build_model + + +@MODELS.register_module("CAC-v1m1") +class CACSegmentor(nn.Module): + def __init__( + self, + num_classes, + backbone_out_channels, + backbone=None, + criteria=None, + cos_temp=15, + main_weight=1, + pre_weight=1, + pre_self_weight=1, + kl_weight=1, + conf_thresh=0, + detach_pre_logits=False, + ): + super().__init__() + self.num_classes = num_classes + self.cos_temp = cos_temp + self.main_weight = main_weight + self.pre_weight = pre_weight + self.pre_self_weight = pre_self_weight + self.kl_weight = kl_weight + self.conf_thresh = conf_thresh + self.detach_pre_logits = detach_pre_logits + + # backbone + self.backbone = build_model(backbone) + # heads + self.seg_head = nn.Linear(backbone_out_channels, num_classes) + self.proj = nn.Sequential( + nn.Linear(backbone_out_channels * 2, backbone_out_channels * 2, bias=False), + nn.ReLU(inplace=True), + nn.Linear(backbone_out_channels * 2, backbone_out_channels), + ) + self.apd_proj = nn.Sequential( + nn.Linear(backbone_out_channels * 2, backbone_out_channels * 2, bias=False), + nn.ReLU(inplace=True), + nn.Linear(backbone_out_channels * 2, backbone_out_channels), + ) + self.feat_proj_layer = nn.Sequential( + nn.Linear(backbone_out_channels, backbone_out_channels, bias=False), + nn.BatchNorm1d(backbone_out_channels), + nn.ReLU(inplace=True), + nn.Linear(backbone_out_channels, backbone_out_channels), + ) + # Criteria + self.criteria = build_criteria(criteria) + + @staticmethod + def get_pred(x, proto): + # x: [n,c]; proto: [cls, c] + x = F.normalize(x, 2, 1) + proto = F.normalize(proto, 2, 1) + pred = x @ proto.permute(1, 0) # [n,c] x [c, cls] -> [n, cls] + return pred + + def get_adaptive_perspective(self, feat, target, new_proto, proto): + raw_feat = feat.clone() + # target: [n] + # feat: [n,c] + # proto: [cls, c] + unique_y = list(target.unique()) + if -1 in unique_y: + unique_y.remove(-1) + target = target.unsqueeze(-1) # [n, 1] + + for tmp_y in unique_y: + tmp_mask = (target == tmp_y).float() + tmp_proto = (feat * tmp_mask).sum(0) / (tmp_mask.sum(0) + 1e-4) # c + onehot_vec = torch.zeros(new_proto.shape[0], 1).cuda() # cls, 1 + onehot_vec[tmp_y.long()] = 1 + new_proto = ( + new_proto * (1 - onehot_vec) + tmp_proto.unsqueeze(0) * onehot_vec + ) + + new_proto = torch.cat([new_proto, proto], -1) + new_proto = self.apd_proj(new_proto) + raw_feat = self.feat_proj_layer(raw_feat) + pred = self.get_pred(raw_feat, new_proto) + return pred + + def post_refine_proto_batch(self, feat, pred, proto, offset=None): + # x: [n, c]; pred: [n, cls]; proto: [cls, c] + pred_list = [] + x = feat + raw_x = x.clone() + if self.detach_pre_logits: + pred = pred.detach() + raw_pred = pred.clone() + + if offset is None: + raw_x = x.clone() + n, n_cls = pred.shape[:] + pred = pred.view(n, n_cls) + pred = F.softmax(pred, 1).permute(1, 0) # [n, cls] -> [cls, n] + if self.conf_thresh > 0: + max_pred = ( + (pred.max(0)[0] >= self.conf_thresh).float().unsqueeze(0) + ) # 1, n + pred = pred * max_pred + pred_proto = (pred / (pred.sum(-1).unsqueeze(-1) + 1e-7)) @ raw_x # cls, c + + pred_proto = torch.cat([pred_proto, proto], -1) # cls, 2c + pred_proto = self.proj(pred_proto) + raw_x = self.feat_proj_layer(raw_x) + new_pred = self.get_pred(raw_x, pred_proto) + else: + for i in range(len(offset)): + if i == 0: + start = 0 + end = offset[i] + else: + start, end = offset[i - 1], offset[i] + tmp_x = raw_x[start:end] + pred = raw_pred[start:end] + n, n_cls = pred.shape[:] + pred = pred.view(n, n_cls) + pred = F.softmax(pred, 1).permute(1, 0) # [n, cls] -> [cls, n] + if self.conf_thresh > 0: + max_pred = ( + (pred.max(0)[0] >= self.conf_thresh).float().unsqueeze(0) + ) # 1, n + pred = pred * max_pred + pred_proto = ( + pred / (pred.sum(-1).unsqueeze(-1) + 1e-7) + ) @ tmp_x # cls, c + + pred_proto = torch.cat([pred_proto, proto], -1) # cls, 2c + pred_proto = self.proj(pred_proto) + tmp_x = self.feat_proj_layer(tmp_x) + new_pred = self.get_pred(tmp_x, pred_proto) + pred_list.append(new_pred) + new_pred = torch.cat(pred_list, 0) + return new_pred + + @staticmethod + def get_distill_loss(pred, soft, target, smoothness=0.5, eps=0): + """ + knowledge distillation loss + """ + n, c = soft.shape[:] + soft = soft.detach() + target = target.unsqueeze(-1) # n, 1 + onehot = target.view(-1, 1) # n, 1 + ignore_mask = (onehot == -1).float() + sm_soft = F.softmax(soft / 1, 1) # n, c + + onehot = onehot * (1 - ignore_mask) + onehot = torch.zeros(n, c).cuda().scatter_(1, onehot.long(), 1) # n, c + smoothed_label = smoothness * sm_soft + (1 - smoothness) * onehot + if eps > 0: + smoothed_label = smoothed_label * (1 - eps) + (1 - smoothed_label) * eps / ( + smoothed_label.shape[1] - 1 + ) + + loss = torch.mul(-1 * F.log_softmax(pred, dim=1), smoothed_label) # b, n, h, w + loss = loss.sum(1) + + sm_soft = F.softmax(soft / 1, 1) # n, c + entropy_mask = -1 * (sm_soft * torch.log(sm_soft + 1e-4)).sum(1) + + # for class-wise entropy estimation + target = target.squeeze(-1) + unique_classes = list(target.unique()) + if -1 in unique_classes: + unique_classes.remove(-1) + valid_mask = (target != -1).float() + entropy_mask = entropy_mask * valid_mask + loss_list = [] + weight_list = [] + for tmp_y in unique_classes: + tmp_mask = (target == tmp_y).float().squeeze() + tmp_entropy_mask = entropy_mask * tmp_mask + class_weight = 1 + tmp_loss = (loss * tmp_entropy_mask).sum() / (tmp_entropy_mask.sum() + 1e-4) + loss_list.append(class_weight * tmp_loss) + weight_list.append(class_weight) + + if len(weight_list) > 0: + loss = sum(loss_list) / (sum(weight_list) + 1e-4) + else: + loss = torch.zeros(1).cuda().mean() + return loss + + def forward(self, data_dict): + offset = data_dict["offset"] + point = self.backbone(data_dict) + if isinstance(point, Point): + feat = point.feat + else: + feat = point + seg_logits = self.seg_head(feat) + + if self.training: + target = data_dict["segment"] + pre_logits = seg_logits.clone() + refine_logits = ( + self.post_refine_proto_batch( + feat=feat, + pred=seg_logits, + proto=self.seg_head.weight.squeeze(), + offset=offset, + ) + * self.cos_temp + ) + + cac_pred = ( + self.get_adaptive_perspective( + feat=feat, + target=target, + new_proto=self.seg_head.weight.detach().data.squeeze(), + proto=self.seg_head.weight.squeeze(), + ) + * self.cos_temp + ) + + seg_loss = self.criteria(refine_logits, target) * self.main_weight + pre_loss = self.criteria(cac_pred, target) * self.pre_weight + pre_self_loss = self.criteria(pre_logits, target) * self.pre_self_weight + kl_loss = ( + self.get_distill_loss( + pred=refine_logits, soft=cac_pred.detach(), target=target + ) + * self.kl_weight + ) + loss = seg_loss + pre_loss + pre_self_loss + kl_loss + return dict( + loss=loss, + seg_loss=seg_loss, + pre_loss=pre_loss, + pre_self_loss=pre_self_loss, + kl_loss=kl_loss, + ) + + elif "segment" in data_dict.keys(): + refine_logits = ( + self.post_refine_proto_batch( + feat=feat, + pred=seg_logits, + proto=self.seg_head.weight.squeeze(), + offset=offset, + ) + * self.cos_temp + ) + + loss = self.criteria(seg_logits, data_dict["segment"]) + return dict(loss=loss, seg_logits=refine_logits) + + else: + refine_logits = ( + self.post_refine_proto_batch( + feat=feat, + pred=seg_logits, + proto=self.seg_head.weight.squeeze(), + offset=offset, + ) + * self.cos_temp + ) + return dict(seg_logits=refine_logits) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/default.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/default.py new file mode 100644 index 0000000000000000000000000000000000000000..8dd600b9e71e70350c644755534c5ef2d7b82c58 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/default.py @@ -0,0 +1,125 @@ +import torch.nn as nn +import torch_scatter + +from pointcept.models.losses import build_criteria +from pointcept.models.utils.structure import Point +from .builder import MODELS, build_model + + +@MODELS.register_module() +class DefaultSegmentor(nn.Module): + def __init__(self, backbone=None, criteria=None): + super().__init__() + self.backbone = build_model(backbone) + self.criteria = build_criteria(criteria) + + def forward(self, input_dict): + if "condition" in input_dict.keys(): + # PPT (https://arxiv.org/abs/2308.09718) + # currently, only support one batch one condition + input_dict["condition"] = input_dict["condition"][0] + seg_logits = self.backbone(input_dict) + # train + if self.training: + loss = self.criteria(seg_logits, input_dict["segment"]) + return dict(loss=loss) + # eval + elif "segment" in input_dict.keys(): + loss = self.criteria(seg_logits, input_dict["segment"]) + return dict(loss=loss, seg_logits=seg_logits) + # test + else: + return dict(seg_logits=seg_logits) + + +@MODELS.register_module() +class DefaultSegmentorV2(nn.Module): + def __init__( + self, + num_classes, + backbone_out_channels, + backbone=None, + criteria=None, + ): + super().__init__() + self.seg_head = ( + nn.Linear(backbone_out_channels, num_classes) + if num_classes > 0 + else nn.Identity() + ) + self.backbone = build_model(backbone) + self.criteria = build_criteria(criteria) + + def forward(self, input_dict): + point = Point(input_dict) + point = self.backbone(point) + # Backbone added after v1.5.0 return Point instead of feat and use DefaultSegmentorV2 + # TODO: remove this part after make all backbone return Point only. + if isinstance(point, Point): + feat = point.feat + else: + feat = point + seg_logits = self.seg_head(feat) + # train + if self.training: + loss = self.criteria(seg_logits, input_dict["segment"]) + return dict(loss=loss) + # eval + elif "segment" in input_dict.keys(): + loss = self.criteria(seg_logits, input_dict["segment"]) + return dict(loss=loss, seg_logits=seg_logits) + # test + else: + return dict(seg_logits=seg_logits) + + +@MODELS.register_module() +class DefaultClassifier(nn.Module): + def __init__( + self, + backbone=None, + criteria=None, + num_classes=40, + backbone_embed_dim=256, + ): + super().__init__() + self.backbone = build_model(backbone) + self.criteria = build_criteria(criteria) + self.num_classes = num_classes + self.backbone_embed_dim = backbone_embed_dim + self.cls_head = nn.Sequential( + nn.Linear(backbone_embed_dim, 256), + nn.BatchNorm1d(256), + nn.ReLU(inplace=True), + nn.Dropout(p=0.5), + nn.Linear(256, 128), + nn.BatchNorm1d(128), + nn.ReLU(inplace=True), + nn.Dropout(p=0.5), + nn.Linear(128, num_classes), + ) + + def forward(self, input_dict): + point = Point(input_dict) + point = self.backbone(point) + # Backbone added after v1.5.0 return Point instead of feat + # And after v1.5.0 feature aggregation for classification operated in classifier + # TODO: remove this part after make all backbone return Point only. + if isinstance(point, Point): + point.feat = torch_scatter.segment_csr( + src=point.feat, + indptr=nn.functional.pad(point.offset, (1, 0)), + reduce="mean", + ) + feat = point.feat + else: + feat = point + cls_logits = self.cls_head(feat) + if self.training: + loss = self.criteria(cls_logits, input_dict["category"]) + return dict(loss=loss) + elif "category" in input_dict.keys(): + loss = self.criteria(cls_logits, input_dict["category"]) + return dict(loss=loss, cls_logits=cls_logits) + else: + return dict(cls_logits=cls_logits) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/losses/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/losses/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ee44096e13acdf7d6cf7ba9ff764b4b6af3aafe5 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/losses/__init__.py @@ -0,0 +1,4 @@ +from .builder import build_criteria + +from .misc import CrossEntropyLoss, SmoothCELoss, DiceLoss, FocalLoss, BinaryFocalLoss +from .lovasz import LovaszLoss diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/losses/builder.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/losses/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..ef642d9830622d847eb2b7e7013252a32c2b6368 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/losses/builder.py @@ -0,0 +1,31 @@ +""" +Criteria Builder + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from pointcept.utils.registry import Registry + +LOSSES = Registry("losses") + + +class Criteria(object): + def __init__(self, cfg=None): + self.cfg = cfg if cfg is not None else [] + self.criteria = [] + for loss_cfg in self.cfg: + self.criteria.append(LOSSES.build(cfg=loss_cfg)) + + def __call__(self, pred, target): + if len(self.criteria) == 0: + # loss computation occur in model + return pred + loss = 0 + for c in self.criteria: + loss += c(pred, target) + return loss + + +def build_criteria(cfg): + return Criteria(cfg) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/losses/lovasz.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/losses/lovasz.py new file mode 100644 index 0000000000000000000000000000000000000000..690c2ba507478853cf7ea827ac6e4a35d8f31709 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/losses/lovasz.py @@ -0,0 +1,257 @@ +""" +Lovasz Loss +refer https://arxiv.org/abs/1705.08790 + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from typing import Optional +from itertools import filterfalse +import torch +import torch.nn.functional as F +from torch.nn.modules.loss import _Loss + +from .builder import LOSSES + +BINARY_MODE: str = "binary" +MULTICLASS_MODE: str = "multiclass" +MULTILABEL_MODE: str = "multilabel" + + +def _lovasz_grad(gt_sorted): + """Compute gradient of the Lovasz extension w.r.t sorted errors + See Alg. 1 in paper + """ + p = len(gt_sorted) + gts = gt_sorted.sum() + intersection = gts - gt_sorted.float().cumsum(0) + union = gts + (1 - gt_sorted).float().cumsum(0) + jaccard = 1.0 - intersection / union + if p > 1: # cover 1-pixel case + jaccard[1:p] = jaccard[1:p] - jaccard[0:-1] + return jaccard + + +def _lovasz_hinge(logits, labels, per_image=True, ignore=None): + """ + Binary Lovasz hinge loss + logits: [B, H, W] Logits at each pixel (between -infinity and +infinity) + labels: [B, H, W] Tensor, binary ground truth masks (0 or 1) + per_image: compute the loss per image instead of per batch + ignore: void class id + """ + if per_image: + loss = mean( + _lovasz_hinge_flat( + *_flatten_binary_scores(log.unsqueeze(0), lab.unsqueeze(0), ignore) + ) + for log, lab in zip(logits, labels) + ) + else: + loss = _lovasz_hinge_flat(*_flatten_binary_scores(logits, labels, ignore)) + return loss + + +def _lovasz_hinge_flat(logits, labels): + """Binary Lovasz hinge loss + Args: + logits: [P] Logits at each prediction (between -infinity and +infinity) + labels: [P] Tensor, binary ground truth labels (0 or 1) + """ + if len(labels) == 0: + # only void pixels, the gradients should be 0 + return logits.sum() * 0.0 + signs = 2.0 * labels.float() - 1.0 + errors = 1.0 - logits * signs + errors_sorted, perm = torch.sort(errors, dim=0, descending=True) + perm = perm.data + gt_sorted = labels[perm] + grad = _lovasz_grad(gt_sorted) + loss = torch.dot(F.relu(errors_sorted), grad) + return loss + + +def _flatten_binary_scores(scores, labels, ignore=None): + """Flattens predictions in the batch (binary case) + Remove labels equal to 'ignore' + """ + scores = scores.view(-1) + labels = labels.view(-1) + if ignore is None: + return scores, labels + valid = labels != ignore + vscores = scores[valid] + vlabels = labels[valid] + return vscores, vlabels + + +def _lovasz_softmax( + probas, labels, classes="present", class_seen=None, per_image=False, ignore=None +): + """Multi-class Lovasz-Softmax loss + Args: + @param probas: [B, C, H, W] Class probabilities at each prediction (between 0 and 1). + Interpreted as binary (sigmoid) output with outputs of size [B, H, W]. + @param labels: [B, H, W] Tensor, ground truth labels (between 0 and C - 1) + @param classes: 'all' for all, 'present' for classes present in labels, or a list of classes to average. + @param per_image: compute the loss per image instead of per batch + @param ignore: void class labels + """ + if per_image: + loss = mean( + _lovasz_softmax_flat( + *_flatten_probas(prob.unsqueeze(0), lab.unsqueeze(0), ignore), + classes=classes + ) + for prob, lab in zip(probas, labels) + ) + else: + loss = _lovasz_softmax_flat( + *_flatten_probas(probas, labels, ignore), + classes=classes, + class_seen=class_seen + ) + return loss + + +def _lovasz_softmax_flat(probas, labels, classes="present", class_seen=None): + """Multi-class Lovasz-Softmax loss + Args: + @param probas: [P, C] Class probabilities at each prediction (between 0 and 1) + @param labels: [P] Tensor, ground truth labels (between 0 and C - 1) + @param classes: 'all' for all, 'present' for classes present in labels, or a list of classes to average. + """ + if probas.numel() == 0: + # only void pixels, the gradients should be 0 + return probas * 0.0 + C = probas.size(1) + losses = [] + class_to_sum = list(range(C)) if classes in ["all", "present"] else classes + # for c in class_to_sum: + for c in labels.unique(): + if class_seen is None: + fg = (labels == c).type_as(probas) # foreground for class c + if classes == "present" and fg.sum() == 0: + continue + if C == 1: + if len(classes) > 1: + raise ValueError("Sigmoid output possible only with 1 class") + class_pred = probas[:, 0] + else: + class_pred = probas[:, c] + errors = (fg - class_pred).abs() + errors_sorted, perm = torch.sort(errors, 0, descending=True) + perm = perm.data + fg_sorted = fg[perm] + losses.append(torch.dot(errors_sorted, _lovasz_grad(fg_sorted))) + else: + if c in class_seen: + fg = (labels == c).type_as(probas) # foreground for class c + if classes == "present" and fg.sum() == 0: + continue + if C == 1: + if len(classes) > 1: + raise ValueError("Sigmoid output possible only with 1 class") + class_pred = probas[:, 0] + else: + class_pred = probas[:, c] + errors = (fg - class_pred).abs() + errors_sorted, perm = torch.sort(errors, 0, descending=True) + perm = perm.data + fg_sorted = fg[perm] + losses.append(torch.dot(errors_sorted, _lovasz_grad(fg_sorted))) + return mean(losses) + + +def _flatten_probas(probas, labels, ignore=None): + """Flattens predictions in the batch""" + if probas.dim() == 3: + # assumes output of a sigmoid layer + B, H, W = probas.size() + probas = probas.view(B, 1, H, W) + + C = probas.size(1) + probas = torch.movedim(probas, 1, -1) # [B, C, Di, Dj, ...] -> [B, Di, Dj, ..., C] + probas = probas.contiguous().view(-1, C) # [P, C] + + labels = labels.view(-1) + if ignore is None: + return probas, labels + valid = labels != ignore + vprobas = probas[valid] + vlabels = labels[valid] + return vprobas, vlabels + + +def isnan(x): + return x != x + + +def mean(values, ignore_nan=False, empty=0): + """Nan-mean compatible with generators.""" + values = iter(values) + if ignore_nan: + values = filterfalse(isnan, values) + try: + n = 1 + acc = next(values) + except StopIteration: + if empty == "raise": + raise ValueError("Empty mean") + return empty + for n, v in enumerate(values, 2): + acc += v + if n == 1: + return acc + return acc / n + + +@LOSSES.register_module() +class LovaszLoss(_Loss): + def __init__( + self, + mode: str, + class_seen: Optional[int] = None, + per_image: bool = False, + ignore_index: Optional[int] = None, + loss_weight: float = 1.0, + ): + """Lovasz loss for segmentation task. + It supports binary, multiclass and multilabel cases + Args: + mode: Loss mode 'binary', 'multiclass' or 'multilabel' + ignore_index: Label that indicates ignored pixels (does not contribute to loss) + per_image: If True loss computed per each image and then averaged, else computed per whole batch + Shape + - **y_pred** - torch.Tensor of shape (N, C, H, W) + - **y_true** - torch.Tensor of shape (N, H, W) or (N, C, H, W) + Reference + https://github.com/BloodAxe/pytorch-toolbelt + """ + assert mode in {BINARY_MODE, MULTILABEL_MODE, MULTICLASS_MODE} + super().__init__() + + self.mode = mode + self.ignore_index = ignore_index + self.per_image = per_image + self.class_seen = class_seen + self.loss_weight = loss_weight + + def forward(self, y_pred, y_true): + if self.mode in {BINARY_MODE, MULTILABEL_MODE}: + loss = _lovasz_hinge( + y_pred, y_true, per_image=self.per_image, ignore=self.ignore_index + ) + elif self.mode == MULTICLASS_MODE: + y_pred = y_pred.softmax(dim=1) + loss = _lovasz_softmax( + y_pred, + y_true, + class_seen=self.class_seen, + per_image=self.per_image, + ignore=self.ignore_index, + ) + else: + raise ValueError("Wrong mode {}.".format(self.mode)) + return loss * self.loss_weight diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/losses/misc.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/losses/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..ec300a54b2d920d37882d25e8c7771bce01db97c --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/losses/misc.py @@ -0,0 +1,223 @@ +""" +Misc Losses + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import torch +import torch.nn as nn +import torch.nn.functional as F +from .builder import LOSSES + + +@LOSSES.register_module() +class CrossEntropyLoss(nn.Module): + def __init__( + self, + weight=None, + size_average=None, + reduce=None, + reduction="mean", + label_smoothing=0.0, + loss_weight=1.0, + ignore_index=-1, + ): + super(CrossEntropyLoss, self).__init__() + weight = torch.tensor(weight).cuda() if weight is not None else None + self.loss_weight = loss_weight + self.loss = nn.CrossEntropyLoss( + weight=weight, + size_average=size_average, + ignore_index=ignore_index, + reduce=reduce, + reduction=reduction, + label_smoothing=label_smoothing, + ) + + def forward(self, pred, target): + return self.loss(pred, target) * self.loss_weight + + +@LOSSES.register_module() +class SmoothCELoss(nn.Module): + def __init__(self, smoothing_ratio=0.1): + super(SmoothCELoss, self).__init__() + self.smoothing_ratio = smoothing_ratio + + def forward(self, pred, target): + eps = self.smoothing_ratio + n_class = pred.size(1) + one_hot = torch.zeros_like(pred).scatter(1, target.view(-1, 1), 1) + one_hot = one_hot * (1 - eps) + (1 - one_hot) * eps / (n_class - 1) + log_prb = F.log_softmax(pred, dim=1) + loss = -(one_hot * log_prb).total(dim=1) + loss = loss[torch.isfinite(loss)].mean() + return loss + + +@LOSSES.register_module() +class BinaryFocalLoss(nn.Module): + def __init__(self, gamma=2.0, alpha=0.5, logits=True, reduce=True, loss_weight=1.0): + """Binary Focal Loss + ` + """ + super(BinaryFocalLoss, self).__init__() + assert 0 < alpha < 1 + self.gamma = gamma + self.alpha = alpha + self.logits = logits + self.reduce = reduce + self.loss_weight = loss_weight + + def forward(self, pred, target, **kwargs): + """Forward function. + Args: + pred (torch.Tensor): The prediction with shape (N) + target (torch.Tensor): The ground truth. If containing class + indices, shape (N) where each value is 0≤targets[i]≤1, If containing class probabilities, + same shape as the input. + Returns: + torch.Tensor: The calculated loss + """ + if self.logits: + bce = F.binary_cross_entropy_with_logits(pred, target, reduction="none") + else: + bce = F.binary_cross_entropy(pred, target, reduction="none") + pt = torch.exp(-bce) + alpha = self.alpha * target + (1 - self.alpha) * (1 - target) + focal_loss = alpha * (1 - pt) ** self.gamma * bce + + if self.reduce: + focal_loss = torch.mean(focal_loss) + return focal_loss * self.loss_weight + + +@LOSSES.register_module() +class FocalLoss(nn.Module): + def __init__( + self, gamma=2.0, alpha=0.5, reduction="mean", loss_weight=1.0, ignore_index=-1 + ): + """Focal Loss + ` + """ + super(FocalLoss, self).__init__() + assert reduction in ( + "mean", + "sum", + ), "AssertionError: reduction should be 'mean' or 'sum'" + assert isinstance( + alpha, (float, list) + ), "AssertionError: alpha should be of type float" + assert isinstance(gamma, float), "AssertionError: gamma should be of type float" + assert isinstance( + loss_weight, float + ), "AssertionError: loss_weight should be of type float" + assert isinstance(ignore_index, int), "ignore_index must be of type int" + self.gamma = gamma + self.alpha = alpha + self.reduction = reduction + self.loss_weight = loss_weight + self.ignore_index = ignore_index + + def forward(self, pred, target, **kwargs): + """Forward function. + Args: + pred (torch.Tensor): The prediction with shape (N, C) where C = number of classes. + target (torch.Tensor): The ground truth. If containing class + indices, shape (N) where each value is 0≤targets[i]≤C−1, If containing class probabilities, + same shape as the input. + Returns: + torch.Tensor: The calculated loss + """ + # [B, C, d_1, d_2, ..., d_k] -> [C, B, d_1, d_2, ..., d_k] + pred = pred.transpose(0, 1) + # [C, B, d_1, d_2, ..., d_k] -> [C, N] + pred = pred.reshape(pred.size(0), -1) + # [C, N] -> [N, C] + pred = pred.transpose(0, 1).contiguous() + # (B, d_1, d_2, ..., d_k) --> (B * d_1 * d_2 * ... * d_k,) + target = target.view(-1).contiguous() + assert pred.size(0) == target.size( + 0 + ), "The shape of pred doesn't match the shape of target" + valid_mask = target != self.ignore_index + target = target[valid_mask] + pred = pred[valid_mask] + + if len(target) == 0: + return 0.0 + + num_classes = pred.size(1) + target = F.one_hot(target, num_classes=num_classes) + + alpha = self.alpha + if isinstance(alpha, list): + alpha = pred.new_tensor(alpha) + pred_sigmoid = pred.sigmoid() + target = target.type_as(pred) + one_minus_pt = (1 - pred_sigmoid) * target + pred_sigmoid * (1 - target) + focal_weight = (alpha * target + (1 - alpha) * (1 - target)) * one_minus_pt.pow( + self.gamma + ) + + loss = ( + F.binary_cross_entropy_with_logits(pred, target, reduction="none") + * focal_weight + ) + if self.reduction == "mean": + loss = loss.mean() + elif self.reduction == "sum": + loss = loss.total() + return self.loss_weight * loss + + +@LOSSES.register_module() +class DiceLoss(nn.Module): + def __init__(self, smooth=1, exponent=2, loss_weight=1.0, ignore_index=-1): + """DiceLoss. + This loss is proposed in `V-Net: Fully Convolutional Neural Networks for + Volumetric Medical Image Segmentation `_. + """ + super(DiceLoss, self).__init__() + self.smooth = smooth + self.exponent = exponent + self.loss_weight = loss_weight + self.ignore_index = ignore_index + + def forward(self, pred, target, **kwargs): + # [B, C, d_1, d_2, ..., d_k] -> [C, B, d_1, d_2, ..., d_k] + pred = pred.transpose(0, 1) + # [C, B, d_1, d_2, ..., d_k] -> [C, N] + pred = pred.reshape(pred.size(0), -1) + # [C, N] -> [N, C] + pred = pred.transpose(0, 1).contiguous() + # (B, d_1, d_2, ..., d_k) --> (B * d_1 * d_2 * ... * d_k,) + target = target.view(-1).contiguous() + assert pred.size(0) == target.size( + 0 + ), "The shape of pred doesn't match the shape of target" + valid_mask = target != self.ignore_index + target = target[valid_mask] + pred = pred[valid_mask] + + pred = F.softmax(pred, dim=1) + num_classes = pred.shape[1] + target = F.one_hot( + torch.clamp(target.long(), 0, num_classes - 1), num_classes=num_classes + ) + + total_loss = 0 + for i in range(num_classes): + if i != self.ignore_index: + num = torch.sum(torch.mul(pred[:, i], target[:, i])) * 2 + self.smooth + den = ( + torch.sum( + pred[:, i].pow(self.exponent) + target[:, i].pow(self.exponent) + ) + + self.smooth + ) + dice_loss = 1 - num / den + total_loss += dice_loss + loss = total_loss / num_classes + return self.loss_weight * loss diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/masked_scene_contrast/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/masked_scene_contrast/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..18733d94acc5c7ed0feb7c52c77cb040d53bee7e --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/masked_scene_contrast/__init__.py @@ -0,0 +1,2 @@ +from .masked_scene_contrast_v1m1_base import MaskedSceneContrast +from .masked_scene_contrast_v1m2_csc import MaskedSceneContrast diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/masked_scene_contrast/masked_scene_contrast_v1m1_base.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/masked_scene_contrast/masked_scene_contrast_v1m1_base.py new file mode 100644 index 0000000000000000000000000000000000000000..3b18bd563a36695eca4882a6620d4ddbe246ef80 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/masked_scene_contrast/masked_scene_contrast_v1m1_base.py @@ -0,0 +1,310 @@ +""" +Masked Scene Contrast +https://arxiv.org/abs/2303.14191 + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import random +from itertools import chain +import torch +import torch.nn as nn +import torch.distributed as dist +from torch_geometric.nn.pool import voxel_grid + +from timm.models.layers import trunc_normal_ +import pointops + +from pointcept.models.builder import MODELS, build_model +from pointcept.models.utils import offset2batch +from pointcept.utils.comm import get_world_size + + +@MODELS.register_module("MSC-v1m1") +class MaskedSceneContrast(nn.Module): + def __init__( + self, + backbone, + backbone_in_channels, + backbone_out_channels, + mask_grid_size=0.1, + mask_rate=0.4, + view1_mix_prob=0, + view2_mix_prob=0, + matching_max_k=8, + matching_max_radius=0.03, + matching_max_pair=8192, + nce_t=0.4, + contrast_weight=1, + reconstruct_weight=1, + reconstruct_color=True, + reconstruct_normal=True, + ): + super().__init__() + self.backbone = build_model(backbone) + self.mask_grid_size = mask_grid_size + self.mask_rate = mask_rate + self.view1_mix_prob = view1_mix_prob + self.view2_mix_prob = view2_mix_prob + self.matching_max_k = matching_max_k + self.matching_max_radius = matching_max_radius + self.matching_max_pair = matching_max_pair + self.nce_t = nce_t + self.contrast_weight = contrast_weight + self.reconstruct_weight = reconstruct_weight + self.reconstruct_color = reconstruct_color + self.reconstruct_normal = reconstruct_normal + + self.mask_token = nn.Parameter(torch.zeros(1, backbone_in_channels)) + trunc_normal_(self.mask_token, mean=0.0, std=0.02) + self.color_head = ( + nn.Linear(backbone_out_channels, 3) if reconstruct_color else None + ) + self.normal_head = ( + nn.Linear(backbone_out_channels, 3) if reconstruct_normal else None + ) + self.nce_criteria = torch.nn.CrossEntropyLoss(reduction="mean") + + @torch.no_grad() + def generate_cross_masks( + self, view1_origin_coord, view1_offset, view2_origin_coord, view2_offset + ): + # union origin coord + view1_batch = offset2batch(view1_offset) + view2_batch = offset2batch(view2_offset) + + view1_batch_count = view1_batch.bincount() + view2_batch_count = view2_batch.bincount() + view1_origin_coord_split = view1_origin_coord.split(list(view1_batch_count)) + view2_origin_coord_split = view2_origin_coord.split(list(view2_batch_count)) + union_origin_coord = torch.cat( + list( + chain.from_iterable( + zip(view1_origin_coord_split, view2_origin_coord_split) + ) + ) + ) + union_offset = torch.cat( + [view1_offset.unsqueeze(-1), view2_offset.unsqueeze(-1)], dim=-1 + ).sum(-1) + union_batch = offset2batch(union_offset) + + # grid partition + mask_patch_coord = union_origin_coord.div(self.mask_grid_size) + mask_patch_grid_coord = torch.floor(mask_patch_coord) + mask_patch_cluster = voxel_grid( + pos=mask_patch_grid_coord, size=1, batch=union_batch, start=0 + ) + unique, cluster, counts = torch.unique( + mask_patch_cluster, sorted=True, return_inverse=True, return_counts=True + ) + patch_num = unique.shape[0] + patch_max_point = counts.max().item() + patch2point_map = cluster.new_zeros(patch_num, patch_max_point) + patch2point_mask = torch.lt( + torch.arange(patch_max_point).cuda().unsqueeze(0), counts.unsqueeze(-1) + ) + sorted_cluster_value, sorted_cluster_indices = torch.sort(cluster) + patch2point_map[patch2point_mask] = sorted_cluster_indices + + # generate cross masks + assert self.mask_rate <= 0.5 + patch_mask = torch.zeros(patch_num, device=union_origin_coord.device).int() + rand_perm = torch.randperm(patch_num) + mask_patch_num = int(patch_num * self.mask_rate) + + # mask1 tag with 1, mask2 tag with 2 + patch_mask[rand_perm[0:mask_patch_num]] = 1 + patch_mask[rand_perm[mask_patch_num : mask_patch_num * 2]] = 2 + point_mask = torch.zeros( + union_origin_coord.shape[0], device=union_origin_coord.device + ).int() + point_mask[ + patch2point_map[patch_mask == 1][patch2point_mask[patch_mask == 1]] + ] = 1 + point_mask[ + patch2point_map[patch_mask == 2][patch2point_mask[patch_mask == 2]] + ] = 2 + + # separate mask to view1 and view2 + point_mask_split = point_mask.split( + list( + torch.cat( + [view1_batch_count.unsqueeze(-1), view2_batch_count.unsqueeze(-1)], + dim=-1, + ).flatten() + ) + ) + view1_point_mask = torch.cat(point_mask_split[0::2]) == 1 + view2_point_mask = torch.cat(point_mask_split[1::2]) == 2 + return view1_point_mask, view2_point_mask + + @torch.no_grad() + def match_contrastive_pair( + self, view1_coord, view1_offset, view2_coord, view2_offset, max_k, max_radius + ): + index, distance = pointops.knn_query( + max_k, + view2_coord.float(), + view2_offset.int(), + view1_coord.float(), + view1_offset.int(), + ) + index = torch.cat( + [ + torch.arange(index.shape[0], device=index.device, dtype=torch.long) + .view(-1, 1, 1) + .expand(-1, max_k, 1), + index.view(-1, max_k, 1), + ], + dim=-1, + )[distance.squeeze(-1) < max_radius] + unique, count = index[:, 0].unique(return_counts=True) + select = ( + torch.cumsum(count, dim=0) + - torch.randint(count.max(), count.shape, device=count.device) % count + - 1 + ) + index = index[select] + if index.shape[0] > self.matching_max_pair: + index = index[torch.randperm(index.shape[0])[: self.matching_max_pair]] + return index + + def compute_contrastive_loss( + self, view1_feat, view1_offset, view2_feat, view2_offset, match_index + ): + assert view1_offset.shape == view2_offset.shape + + view1_feat = view1_feat[match_index[:, 0]] + view2_feat = view2_feat[match_index[:, 1]] + view1_feat = view1_feat / ( + torch.norm(view1_feat, p=2, dim=1, keepdim=True) + 1e-7 + ) + view2_feat = view2_feat / ( + torch.norm(view2_feat, p=2, dim=1, keepdim=True) + 1e-7 + ) + sim = torch.mm(view1_feat, view2_feat.transpose(1, 0)) + + with torch.no_grad(): + pos_sim = torch.diagonal(sim).mean() + neg_sim = sim.mean(dim=-1).mean() - pos_sim / match_index.shape[0] + labels = torch.arange(sim.shape[0], device=view1_feat.device).long() + loss = self.nce_criteria(torch.div(sim, self.nce_t), labels) + + if get_world_size() > 1: + dist.all_reduce(loss) + dist.all_reduce(pos_sim) + dist.all_reduce(neg_sim) + return ( + loss / get_world_size(), + pos_sim / get_world_size(), + neg_sim / get_world_size(), + ) + + def forward(self, data_dict): + view1_origin_coord = data_dict["view1_origin_coord"] + view1_coord = data_dict["view1_coord"] + view1_feat = data_dict["view1_feat"] + view1_offset = data_dict["view1_offset"].int() + + view2_origin_coord = data_dict["view2_origin_coord"] + view2_coord = data_dict["view2_coord"] + view2_feat = data_dict["view2_feat"] + view2_offset = data_dict["view2_offset"].int() + + # mask generation by union original coord (without spatial augmentation) + view1_point_mask, view2_point_mask = self.generate_cross_masks( + view1_origin_coord, view1_offset, view2_origin_coord, view2_offset + ) + + view1_mask_tokens = self.mask_token.expand(view1_coord.shape[0], -1) + view1_weight = view1_point_mask.unsqueeze(-1).type_as(view1_mask_tokens) + view1_feat = view1_feat * (1 - view1_weight) + view1_mask_tokens * view1_weight + + view2_mask_tokens = self.mask_token.expand(view2_coord.shape[0], -1) + view2_weight = view2_point_mask.unsqueeze(-1).type_as(view2_mask_tokens) + view2_feat = view2_feat * (1 - view2_weight) + view2_mask_tokens * view2_weight + + view1_data_dict = dict( + origin_coord=view1_origin_coord, + coord=view1_coord, + feat=view1_feat, + offset=view1_offset, + ) + view2_data_dict = dict( + origin_coord=view2_origin_coord, + coord=view2_coord, + feat=view2_feat, + offset=view2_offset, + ) + + # SparseConv based method need grid coord + if "view1_grid_coord" in data_dict.keys(): + view1_data_dict["grid_coord"] = data_dict["view1_grid_coord"] + if "view2_grid_coord" in data_dict.keys(): + view2_data_dict["grid_coord"] = data_dict["view2_grid_coord"] + + # view mixing strategy + if random.random() < self.view1_mix_prob: + view1_data_dict["offset"] = torch.cat( + [view1_offset[1:-1:2], view1_offset[-1].unsqueeze(0)], dim=0 + ) + if random.random() < self.view2_mix_prob: + view2_data_dict["offset"] = torch.cat( + [view2_offset[1:-1:2], view2_offset[-1].unsqueeze(0)], dim=0 + ) + + view1_feat = self.backbone(view1_data_dict) + view2_feat = self.backbone(view2_data_dict) + match_index = self.match_contrastive_pair( + view1_origin_coord, + view1_offset, + view2_origin_coord, + view2_offset, + max_k=self.matching_max_k, + max_radius=self.matching_max_radius, + ) + nce_loss, pos_sim, neg_sim = self.compute_contrastive_loss( + view1_feat, view1_offset, view2_feat, view2_offset, match_index + ) + loss = nce_loss * self.contrast_weight + result_dict = dict(nce_loss=nce_loss, pos_sim=pos_sim, neg_sim=neg_sim) + + if self.color_head is not None: + assert "view1_color" in data_dict.keys() + assert "view2_color" in data_dict.keys() + view1_color = data_dict["view1_color"] + view2_color = data_dict["view2_color"] + view1_color_pred = self.color_head(view1_feat[view1_point_mask]) + view2_color_pred = self.color_head(view2_feat[view2_point_mask]) + color_loss = ( + torch.sum((view1_color_pred - view1_color[view1_point_mask]) ** 2) + + torch.sum((view2_color_pred - view2_color[view2_point_mask]) ** 2) + ) / (view1_color_pred.shape[0] + view2_color_pred.shape[0]) + loss = loss + color_loss * self.reconstruct_weight + result_dict["color_loss"] = color_loss + + if self.normal_head is not None: + assert "view1_normal" in data_dict.keys() + assert "view2_normal" in data_dict.keys() + view1_normal = data_dict["view1_normal"] + view2_normal = data_dict["view2_normal"] + view1_normal_pred = self.normal_head(view1_feat[view1_point_mask]) + view2_normal_pred = self.normal_head(view2_feat[view2_point_mask]) + + view1_normal_pred = view1_normal_pred / ( + torch.norm(view1_normal_pred, p=2, dim=1, keepdim=True) + 1e-10 + ) + view2_normal_pred = view2_normal_pred / ( + torch.norm(view2_normal_pred, p=2, dim=1, keepdim=True) + 1e-10 + ) + normal_loss = ( + torch.sum(view1_normal_pred * view1_normal[view1_point_mask]) + + torch.sum(view2_normal_pred * view2_normal[view2_point_mask]) + ) / (view1_normal_pred.shape[0] + view2_normal_pred.shape[0]) + loss = loss + normal_loss * self.reconstruct_weight + result_dict["normal_loss"] = normal_loss + + result_dict["loss"] = loss + return result_dict diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/masked_scene_contrast/masked_scene_contrast_v1m2_csc.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/masked_scene_contrast/masked_scene_contrast_v1m2_csc.py new file mode 100644 index 0000000000000000000000000000000000000000..139e26b89d88bbe8711fb9a162b5347d846fd399 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/masked_scene_contrast/masked_scene_contrast_v1m2_csc.py @@ -0,0 +1,377 @@ +""" +Masked Scene Contrast v1m2 +contrastive learning backend with CSC (https://arxiv.org/abs/2012.09165) + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com), Chengyao Wang (cywang22@cse.cuhk.edu.hk) +Please cite our work if the code is helpful to you. +""" + +import random +from itertools import chain +import torch +import torch.nn as nn +import torch.distributed as dist +from torch_geometric.nn.pool import voxel_grid + +from timm.models.layers import trunc_normal_ +import pointops + +from pointcept.models.builder import MODELS, build_model +from pointcept.models.utils import offset2batch +from pointcept.utils.comm import get_world_size + + +@MODELS.register_module("MSC-v1m2") +class MaskedSceneContrast(nn.Module): + def __init__( + self, + backbone, + backbone_in_channels, + backbone_out_channels, + mask_grid_size=0.1, + mask_rate=0.4, + view1_mix_prob=0, + view2_mix_prob=0, + matching_max_k=8, + matching_max_radius=0.03, + matching_max_pair=8192, + nce_t=0.4, + contrast_weight=1, + reconstruct_weight=1, + reconstruct_color=True, + reconstruct_normal=True, + partitions=4, + r1=0.125, + r2=2, + ): + super().__init__() + self.backbone = build_model(backbone) + self.mask_grid_size = mask_grid_size + self.mask_rate = mask_rate + self.view1_mix_prob = view1_mix_prob + self.view2_mix_prob = view2_mix_prob + self.matching_max_k = matching_max_k + self.matching_max_radius = matching_max_radius + self.matching_max_pair = matching_max_pair + self.nce_t = nce_t + self.contrast_weight = contrast_weight + self.reconstruct_weight = reconstruct_weight + self.reconstruct_color = reconstruct_color + self.reconstruct_normal = reconstruct_normal + + # csc partition + self.partitions = partitions + self.r1 = r1 + self.r2 = r2 + + self.mask_token = nn.Parameter(torch.zeros(1, backbone_in_channels)) + trunc_normal_(self.mask_token, mean=0.0, std=0.02) + self.color_head = ( + nn.Linear(backbone_out_channels, 3) if reconstruct_color else None + ) + self.normal_head = ( + nn.Linear(backbone_out_channels, 3) if reconstruct_normal else None + ) + self.nce_criteria = torch.nn.CrossEntropyLoss(reduction="mean") + + @torch.no_grad() + def generate_cross_masks( + self, view1_origin_coord, view1_offset, view2_origin_coord, view2_offset + ): + # union origin coord + view1_batch = offset2batch(view1_offset) + view2_batch = offset2batch(view2_offset) + + view1_batch_count = view1_batch.bincount() + view2_batch_count = view2_batch.bincount() + view1_origin_coord_split = view1_origin_coord.split(list(view1_batch_count)) + view2_origin_coord_split = view2_origin_coord.split(list(view2_batch_count)) + union_origin_coord = torch.cat( + list( + chain.from_iterable( + zip(view1_origin_coord_split, view2_origin_coord_split) + ) + ) + ) + union_offset = torch.cat( + [view1_offset.unsqueeze(-1), view2_offset.unsqueeze(-1)], dim=-1 + ).sum(-1) + union_batch = offset2batch(union_offset) + + # grid partition + mask_patch_coord = union_origin_coord.div(self.mask_grid_size) + mask_patch_grid_coord = torch.floor(mask_patch_coord) + mask_patch_cluster = voxel_grid( + pos=mask_patch_grid_coord, size=1, batch=union_batch, start=0 + ) + unique, cluster, counts = torch.unique( + mask_patch_cluster, sorted=True, return_inverse=True, return_counts=True + ) + patch_num = unique.shape[0] + patch_max_point = counts.max().item() + patch2point_map = cluster.new_zeros(patch_num, patch_max_point) + patch2point_mask = torch.lt( + torch.arange(patch_max_point).cuda().unsqueeze(0), counts.unsqueeze(-1) + ) + sorted_cluster_value, sorted_cluster_indices = torch.sort(cluster) + patch2point_map[patch2point_mask] = sorted_cluster_indices + + # generate cross masks + assert self.mask_rate <= 0.5 + patch_mask = torch.zeros(patch_num, device=union_origin_coord.device).int() + rand_perm = torch.randperm(patch_num) + mask_patch_num = int(patch_num * self.mask_rate) + + # mask1 tag with 1, mask2 tag with 2 + patch_mask[rand_perm[0:mask_patch_num]] = 1 + patch_mask[rand_perm[mask_patch_num : mask_patch_num * 2]] = 2 + point_mask = torch.zeros( + union_origin_coord.shape[0], device=union_origin_coord.device + ).int() + point_mask[ + patch2point_map[patch_mask == 1][patch2point_mask[patch_mask == 1]] + ] = 1 + point_mask[ + patch2point_map[patch_mask == 2][patch2point_mask[patch_mask == 2]] + ] = 2 + + # separate mask to view1 and view2 + point_mask_split = point_mask.split( + list( + torch.cat( + [view1_batch_count.unsqueeze(-1), view2_batch_count.unsqueeze(-1)], + dim=-1, + ).flatten() + ) + ) + view1_point_mask = torch.cat(point_mask_split[0::2]) == 1 + view2_point_mask = torch.cat(point_mask_split[1::2]) == 2 + return view1_point_mask, view2_point_mask + + @torch.no_grad() + def match_contrastive_pair( + self, view1_coord, view1_offset, view2_coord, view2_offset, max_k, max_radius + ): + index, distance = pointops.knn_query( + max_k, + view2_coord.float(), + view2_offset.int(), + view1_coord.float(), + view1_offset.int(), + ) + index = torch.cat( + [ + torch.arange(index.shape[0], device=index.device, dtype=torch.long) + .view(-1, 1, 1) + .expand(-1, max_k, 1), + index.view(-1, max_k, 1), + ], + dim=-1, + )[distance.squeeze(-1) < max_radius] + unique, count = index[:, 0].unique(return_counts=True) + select = ( + torch.cumsum(count, dim=0) + - torch.randint(count.max(), count.shape, device=count.device) % count + - 1 + ) + index = index[select] + if index.shape[0] > self.matching_max_pair: + index = index[torch.randperm(index.shape[0])[: self.matching_max_pair]] + return index + + def compute_partitions(self, coord1, coord2): + partition_matrix = torch.zeros((coord1.shape[0], coord2.shape[0])) + partition_matrix = partition_matrix.cuda() - 1e7 + + rel_trans = coord1.unsqueeze(0) - coord2.unsqueeze(1) + mask_up = rel_trans[:, :, 2] > 0.0 + mask_down = rel_trans[:, :, 2] < 0.0 + + distance_matrix = torch.sqrt(torch.sum(rel_trans.pow(2), 2).add(1e-7)) + + mask = (distance_matrix[:, :] > self.r1) & (distance_matrix[:, :] <= self.r2) + partition_matrix[mask & mask_up] = 0 + partition_matrix[mask & mask_down] = 1 + + mask = distance_matrix[:, :] > self.r2 + partition_matrix[mask & mask_up] = 2 + partition_matrix[mask & mask_down] = 3 + + return partition_matrix + + def compute_contrastive_loss( + self, + view1_feat, + view1_coord, + view1_offset, + view2_feat, + view2_coord, + view2_offset, + match_index, + ): + assert view1_offset.shape == view2_offset.shape + device = view1_feat.device + loss = torch.tensor(0.0, device=device) + pos_sim = torch.tensor(0.0, device=device) + neg_sim = torch.tensor(0.0, device=device) + large_num = 1e9 + + view1_feat = view1_feat[match_index[:, 0]] + view2_feat = view2_feat[match_index[:, 1]] + view1_feat = view1_feat / ( + torch.norm(view1_feat, p=2, dim=1, keepdim=True) + 1e-7 + ) + view2_feat = view2_feat / ( + torch.norm(view2_feat, p=2, dim=1, keepdim=True) + 1e-7 + ) + + view1_coord = view1_coord[match_index[:, 0]] + view2_coord = view2_coord[match_index[:, 1]] + + batch = offset2batch(view1_offset)[match_index[:, 0]] + for batch_id in batch.unique(): + batch_mask = batch == batch_id + sim = torch.mm(view1_feat[batch_mask], view2_feat[batch_mask].T) + + with torch.no_grad(): + pos_sim += torch.diagonal(sim).mean() + neg_sim += sim.mean(dim=-1).mean() - pos_sim / batch_mask.sum() + + labels = torch.arange(sim.shape[0], device=view1_feat.device).long() + part = self.compute_partitions( + view1_coord[batch_mask], view2_coord[batch_mask] + ) + for part_id in part.unique(): + part_mask = part == part_id + part_mask.fill_diagonal_(True) + loss += self.nce_criteria( + torch.div(sim, self.nce_t) - large_num * (~part_mask).float(), + labels, + ) + + loss /= len(view1_offset) * self.partitions + pos_sim /= len(view1_offset) + neg_sim /= len(view1_offset) + + if get_world_size() > 1: + dist.all_reduce(loss) + dist.all_reduce(pos_sim) + dist.all_reduce(neg_sim) + return ( + loss / get_world_size(), + pos_sim / get_world_size(), + neg_sim / get_world_size(), + ) + + def forward(self, data_dict): + view1_origin_coord = data_dict["view1_origin_coord"] + view1_coord = data_dict["view1_coord"] + view1_feat = data_dict["view1_feat"] + view1_offset = data_dict["view1_offset"].int() + + view2_origin_coord = data_dict["view2_origin_coord"] + view2_coord = data_dict["view2_coord"] + view2_feat = data_dict["view2_feat"] + view2_offset = data_dict["view2_offset"].int() + + # mask generation by union original coord (without spatial augmentation) + view1_point_mask, view2_point_mask = self.generate_cross_masks( + view1_origin_coord, view1_offset, view2_origin_coord, view2_offset + ) + + view1_mask_tokens = self.mask_token.expand(view1_coord.shape[0], -1) + view1_weight = view1_point_mask.unsqueeze(-1).type_as(view1_mask_tokens) + view1_feat = view1_feat * (1 - view1_weight) + view1_mask_tokens * view1_weight + + view2_mask_tokens = self.mask_token.expand(view2_coord.shape[0], -1) + view2_weight = view2_point_mask.unsqueeze(-1).type_as(view2_mask_tokens) + view2_feat = view2_feat * (1 - view2_weight) + view2_mask_tokens * view2_weight + + view1_data_dict = dict( + origin_coord=view1_origin_coord, + coord=view1_coord, + feat=view1_feat, + offset=view1_offset, + ) + view2_data_dict = dict( + origin_coord=view2_origin_coord, + coord=view2_coord, + feat=view2_feat, + offset=view2_offset, + ) + + # SparseConv based method need grid coord + if "view1_grid_coord" in data_dict.keys(): + view1_data_dict["grid_coord"] = data_dict["view1_grid_coord"] + if "view2_grid_coord" in data_dict.keys(): + view2_data_dict["grid_coord"] = data_dict["view2_grid_coord"] + + # view mixing strategy + if random.random() < self.view1_mix_prob: + view1_data_dict["offset"] = torch.cat( + [view1_offset[1:-1:2], view1_offset[-1].unsqueeze(0)], dim=0 + ) + if random.random() < self.view2_mix_prob: + view2_data_dict["offset"] = torch.cat( + [view2_offset[1:-1:2], view2_offset[-1].unsqueeze(0)], dim=0 + ) + + view1_feat = self.backbone(view1_data_dict) + view2_feat = self.backbone(view2_data_dict) + match_index = self.match_contrastive_pair( + view1_origin_coord, + view1_offset, + view2_origin_coord, + view2_offset, + max_k=self.matching_max_k, + max_radius=self.matching_max_radius, + ) + nce_loss, pos_sim, neg_sim = self.compute_contrastive_loss( + view1_feat, + view1_origin_coord, + view1_offset, + view2_feat, + view2_origin_coord, + view2_offset, + match_index, + ) + loss = nce_loss * self.contrast_weight + result_dict = dict(nce_loss=nce_loss, pos_sim=pos_sim, neg_sim=neg_sim) + + if self.color_head is not None: + assert "view1_color" in data_dict.keys() + assert "view2_color" in data_dict.keys() + view1_color = data_dict["view1_color"] + view2_color = data_dict["view2_color"] + view1_color_pred = self.color_head(view1_feat[view1_point_mask]) + view2_color_pred = self.color_head(view2_feat[view2_point_mask]) + color_loss = ( + torch.sum((view1_color_pred - view1_color[view1_point_mask]) ** 2) + + torch.sum((view2_color_pred - view2_color[view2_point_mask]) ** 2) + ) / (view1_color_pred.shape[0] + view2_color_pred.shape[0]) + loss = loss + color_loss * self.reconstruct_weight + result_dict["color_loss"] = color_loss + + if self.normal_head is not None: + assert "view1_normal" in data_dict.keys() + assert "view2_normal" in data_dict.keys() + view1_normal = data_dict["view1_normal"] + view2_normal = data_dict["view2_normal"] + view1_normal_pred = self.normal_head(view1_feat[view1_point_mask]) + view2_normal_pred = self.normal_head(view2_feat[view2_point_mask]) + + view1_normal_pred = view1_normal_pred / ( + torch.norm(view1_normal_pred, p=2, dim=1, keepdim=True) + 1e-10 + ) + view2_normal_pred = view2_normal_pred / ( + torch.norm(view2_normal_pred, p=2, dim=1, keepdim=True) + 1e-10 + ) + normal_loss = ( + torch.sum(view1_normal_pred * view1_normal[view1_point_mask]) + + torch.sum(view2_normal_pred * view2_normal[view2_point_mask]) + ) / (view1_normal_pred.shape[0] + view2_normal_pred.shape[0]) + loss = loss + normal_loss * self.reconstruct_weight + result_dict["normal_loss"] = normal_loss + + result_dict["loss"] = loss + return result_dict diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/modules.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/modules.py new file mode 100644 index 0000000000000000000000000000000000000000..8a737ae9e52620ffc4eb139a81e0bdd46cded0ed --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/modules.py @@ -0,0 +1,83 @@ +import sys +import torch.nn as nn +import spconv.pytorch as spconv +from collections import OrderedDict +from pointcept.models.utils.structure import Point + + +class PointModule(nn.Module): + r"""PointModule + placeholder, all module subclass from this will take Point in PointSequential. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class PointSequential(PointModule): + r"""A sequential container. + Modules will be added to it in the order they are passed in the constructor. + Alternatively, an ordered dict of modules can also be passed in. + """ + + def __init__(self, *args, **kwargs): + super().__init__() + if len(args) == 1 and isinstance(args[0], OrderedDict): + for key, module in args[0].items(): + self.add_module(key, module) + else: + for idx, module in enumerate(args): + self.add_module(str(idx), module) + for name, module in kwargs.items(): + if sys.version_info < (3, 6): + raise ValueError("kwargs only supported in py36+") + if name in self._modules: + raise ValueError("name exists.") + self.add_module(name, module) + + def __getitem__(self, idx): + if not (-len(self) <= idx < len(self)): + raise IndexError("index {} is out of range".format(idx)) + if idx < 0: + idx += len(self) + it = iter(self._modules.values()) + for i in range(idx): + next(it) + return next(it) + + def __len__(self): + return len(self._modules) + + def add(self, module, name=None): + if name is None: + name = str(len(self._modules)) + if name in self._modules: + raise KeyError("name exists") + self.add_module(name, module) + + def forward(self, input): + for k, module in self._modules.items(): + # Point module + if isinstance(module, PointModule): + input = module(input) + # Spconv module + elif spconv.modules.is_spconv_module(module): + if isinstance(input, Point): + input.sparse_conv_feat = module(input.sparse_conv_feat) + input.feat = input.sparse_conv_feat.features + else: + input = module(input) + # PyTorch module + else: + if isinstance(input, Point): + input.feat = module(input.feat) + if "sparse_conv_feat" in input.keys(): + input.sparse_conv_feat = input.sparse_conv_feat.replace_feature( + input.feat + ) + elif isinstance(input, spconv.SparseConvTensor): + if input.indices.shape[0] != 0: + input = input.replace_feature(module(input.features)) + else: + input = module(input) + return input diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/oacnns/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/oacnns/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..654b767080457f3814142159a121a2bef9c682b7 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/oacnns/__init__.py @@ -0,0 +1 @@ +from .oacnns_v1m1_base import OACNNs diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/oacnns/oacnns_v1m1_base.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/oacnns/oacnns_v1m1_base.py new file mode 100644 index 0000000000000000000000000000000000000000..bd8ee6d25ed152f04ab8b3905bbd8c2bdf127d06 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/oacnns/oacnns_v1m1_base.py @@ -0,0 +1,345 @@ +from functools import partial +import torch +import torch.nn as nn +from einops import rearrange +import spconv.pytorch as spconv +from timm.models.layers import trunc_normal_ +from ..builder import MODELS +from ..utils import offset2batch +from torch_geometric.nn.pool import voxel_grid +from torch_geometric.utils import scatter + + +class BasicBlock(nn.Module): + def __init__( + self, + in_channels, + embed_channels, + norm_fn=None, + indice_key=None, + depth=4, + groups=None, + grid_size=None, + bias=False, + ): + super().__init__() + assert embed_channels % groups == 0 + self.groups = groups + self.embed_channels = embed_channels + self.proj = nn.ModuleList() + self.grid_size = grid_size + self.weight = nn.ModuleList() + self.l_w = nn.ModuleList() + self.proj.append( + nn.Sequential( + nn.Linear(embed_channels, embed_channels, bias=False), + norm_fn(embed_channels), + nn.ReLU(), + ) + ) + for _ in range(depth - 1): + self.proj.append( + nn.Sequential( + nn.Linear(embed_channels, embed_channels, bias=False), + norm_fn(embed_channels), + nn.ReLU(), + ) + ) + self.l_w.append( + nn.Sequential( + nn.Linear(embed_channels, embed_channels, bias=False), + norm_fn(embed_channels), + nn.ReLU(), + ) + ) + self.weight.append(nn.Linear(embed_channels, embed_channels, bias=False)) + + self.adaptive = nn.Linear(embed_channels, depth - 1, bias=False) + self.fuse = nn.Sequential( + nn.Linear(embed_channels * 2, embed_channels, bias=False), + norm_fn(embed_channels), + nn.ReLU(), + ) + self.voxel_block = spconv.SparseSequential( + spconv.SubMConv3d( + embed_channels, + embed_channels, + kernel_size=3, + stride=1, + padding=1, + indice_key=indice_key, + bias=bias, + ), + norm_fn(embed_channels), + nn.ReLU(), + spconv.SubMConv3d( + embed_channels, + embed_channels, + kernel_size=3, + stride=1, + padding=1, + indice_key=indice_key, + bias=bias, + ), + norm_fn(embed_channels), + ) + self.act = nn.ReLU() + + def forward(self, x, clusters): + feat = x.features + feats = [] + for i, cluster in enumerate(clusters): + pw = self.l_w[i](feat) + pw = pw - scatter(pw, cluster, reduce="mean")[cluster] + pw = self.weight[i](pw) + pw = torch.exp(pw - pw.max()) + pw = pw / (scatter(pw, cluster, reduce="sum", dim=0)[cluster] + 1e-6) + pfeat = self.proj[i](feat) * pw + pfeat = scatter(pfeat, cluster, reduce="sum")[cluster] + feats.append(pfeat) + adp = self.adaptive(feat) + adp = torch.softmax(adp, dim=1) + feats = torch.stack(feats, dim=1) + feats = torch.einsum("l n, l n c -> l c", adp, feats) + feat = self.proj[-1](feat) + feat = torch.cat([feat, feats], dim=1) + feat = self.fuse(feat) + x.features + res = feat + x = x.replace_feature(feat) + x = self.voxel_block(x) + x = x.replace_feature(self.act(x.features + res)) + return x + + +class DonwBlock(nn.Module): + def __init__( + self, + in_channels, + embed_channels, + depth, + sp_indice_key, + point_grid_size, + num_ref=16, + groups=None, + norm_fn=None, + sub_indice_key=None, + ): + super().__init__() + self.num_ref = num_ref + self.depth = depth + self.point_grid_size = point_grid_size + self.down = spconv.SparseSequential( + spconv.SparseConv3d( + in_channels, + embed_channels, + kernel_size=2, + stride=2, + indice_key=sp_indice_key, + bias=False, + ), + norm_fn(embed_channels), + nn.ReLU(), + ) + self.blocks = nn.ModuleList() + for _ in range(depth): + self.blocks.append( + BasicBlock( + in_channels=embed_channels, + embed_channels=embed_channels, + depth=len(point_grid_size) + 1, + groups=groups, + grid_size=point_grid_size, + norm_fn=norm_fn, + indice_key=sub_indice_key, + ) + ) + + def forward(self, x): + x = self.down(x) + coord = x.indices[:, 1:].float() + batch = x.indices[:, 0] + clusters = [] + for grid_size in self.point_grid_size: + cluster = voxel_grid(pos=coord, size=grid_size, batch=batch) + _, cluster = torch.unique(cluster, return_inverse=True) + clusters.append(cluster) + for block in self.blocks: + x = block(x, clusters) + return x + + +class UpBlock(nn.Module): + def __init__( + self, + in_channels, + skip_channels, + embed_channels, + depth, + sp_indice_key, + norm_fn=None, + down_ratio=2, + sub_indice_key=None, + ): + super().__init__() + assert depth > 0 + self.up = spconv.SparseSequential( + spconv.SparseInverseConv3d( + in_channels, + embed_channels, + kernel_size=down_ratio, + indice_key=sp_indice_key, + bias=False, + ), + norm_fn(embed_channels), + nn.ReLU(), + ) + self.blocks = nn.ModuleList() + self.fuse = nn.Sequential( + nn.Linear(skip_channels + embed_channels, embed_channels), + norm_fn(embed_channels), + nn.ReLU(), + nn.Linear(embed_channels, embed_channels), + norm_fn(embed_channels), + nn.ReLU(), + ) + + def forward(self, x, skip_x): + x = self.up(x) + x = x.replace_feature( + self.fuse(torch.cat([x.features, skip_x.features], dim=1)) + x.features + ) + return x + + +@MODELS.register_module() +class OACNNs(nn.Module): + def __init__( + self, + in_channels, + num_classes, + embed_channels=64, + enc_num_ref=[16, 16, 16, 16], + enc_channels=[64, 64, 128, 256], + groups=[2, 4, 8, 16], + enc_depth=[2, 3, 6, 4], + down_ratio=[2, 2, 2, 2], + dec_channels=[96, 96, 128, 256], + point_grid_size=[[16, 32, 64], [8, 16, 24], [4, 8, 12], [2, 4, 6]], + dec_depth=[2, 2, 2, 2], + ): + super().__init__() + self.in_channels = in_channels + self.num_classes = num_classes + self.num_stages = len(enc_channels) + self.embed_channels = embed_channels + norm_fn = partial(nn.BatchNorm1d, eps=1e-3, momentum=0.01) + + self.stem = spconv.SparseSequential( + spconv.SubMConv3d( + in_channels, + embed_channels, + kernel_size=3, + padding=1, + indice_key="stem", + bias=False, + ), + norm_fn(embed_channels), + nn.ReLU(), + spconv.SubMConv3d( + embed_channels, + embed_channels, + kernel_size=3, + padding=1, + indice_key="stem", + bias=False, + ), + norm_fn(embed_channels), + nn.ReLU(), + spconv.SubMConv3d( + embed_channels, + embed_channels, + kernel_size=3, + padding=1, + indice_key="stem", + bias=False, + ), + norm_fn(embed_channels), + nn.ReLU(), + ) + + self.enc = nn.ModuleList() + self.dec = nn.ModuleList() + for i in range(self.num_stages): + self.enc.append( + DonwBlock( + in_channels=embed_channels if i == 0 else enc_channels[i - 1], + embed_channels=enc_channels[i], + depth=enc_depth[i], + norm_fn=norm_fn, + groups=groups[i], + point_grid_size=point_grid_size[i], + num_ref=enc_num_ref[i], + sp_indice_key=f"spconv{i}", + sub_indice_key=f"subm{i + 1}", + ) + ) + self.dec.append( + UpBlock( + in_channels=( + enc_channels[-1] + if i == self.num_stages - 1 + else dec_channels[i + 1] + ), + skip_channels=embed_channels if i == 0 else enc_channels[i - 1], + embed_channels=dec_channels[i], + depth=dec_depth[i], + norm_fn=norm_fn, + sp_indice_key=f"spconv{i}", + sub_indice_key=f"subm{i}", + ) + ) + + self.final = spconv.SubMConv3d(dec_channels[0], num_classes, kernel_size=1) + self.apply(self._init_weights) + + def forward(self, input_dict): + discrete_coord = input_dict["grid_coord"] + feat = input_dict["feat"] + offset = input_dict["offset"] + batch = offset2batch(offset) + x = spconv.SparseConvTensor( + features=feat, + indices=torch.cat([batch.unsqueeze(-1), discrete_coord], dim=1) + .int() + .contiguous(), + spatial_shape=torch.add( + torch.max(discrete_coord, dim=0).values, 1 + ).tolist(), + batch_size=batch[-1].tolist() + 1, + ) + + x = self.stem(x) + skips = [x] + for i in range(self.num_stages): + x = self.enc[i](x) + skips.append(x) + x = skips.pop(-1) + for i in reversed(range(self.num_stages)): + skip = skips.pop(-1) + x = self.dec[i](x, skip) + x = self.final(x) + return x.features + + @staticmethod + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, spconv.SubMConv3d): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm1d): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/octformer/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/octformer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..06bea370d10808b0bad2b68189e957765fc46081 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/octformer/__init__.py @@ -0,0 +1 @@ +from .octformer_v1m1_base import OctFormer diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/octformer/octformer_v1m1_base.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/octformer/octformer_v1m1_base.py new file mode 100644 index 0000000000000000000000000000000000000000..7c0faf700126b0d589867811e871025dc9782b76 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/octformer/octformer_v1m1_base.py @@ -0,0 +1,629 @@ +""" +Octree Transformer + +Modified from https://github.com/octree-nn/octformer + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from typing import Optional, List, Dict +import torch +import torch.nn as nn +from torch.utils.checkpoint import checkpoint + +try: + import ocnn + from ocnn.octree import Octree, Points +except ImportError: + from pointcept.utils.misc import DummyClass + + ocnn = None + Octree = DummyClass + Points = DummyClass + +try: + import dwconv +except ImportError: + dwconv = None + +from pointcept.models.builder import MODELS +from pointcept.models.utils import offset2batch + + +class OctreeT(Octree): + def __init__( + self, + octree: Octree, + patch_size: int = 24, + dilation: int = 4, + nempty: bool = True, + max_depth: Optional[int] = None, + start_depth: Optional[int] = None, + **kwargs + ): + super().__init__(octree.depth, octree.full_depth) + self.__dict__.update(octree.__dict__) + + self.patch_size = patch_size + self.dilation = dilation + self.nempty = nempty + self.max_depth = max_depth or self.depth + self.start_depth = start_depth or self.full_depth + self.invalid_mask_value = -1e3 + assert self.start_depth > 1 + + self.block_num = patch_size * dilation + self.nnum_t = self.nnum_nempty if nempty else self.nnum + self.nnum_a = ((self.nnum_t / self.block_num).ceil() * self.block_num).int() + + num = self.max_depth + 1 + self.batch_idx = [None] * num + self.patch_mask = [None] * num + self.dilate_mask = [None] * num + self.rel_pos = [None] * num + self.dilate_pos = [None] * num + self.build_t() + + def build_t(self): + for d in range(self.start_depth, self.max_depth + 1): + self.build_batch_idx(d) + self.build_attn_mask(d) + self.build_rel_pos(d) + + def build_batch_idx(self, depth: int): + batch = self.batch_id(depth, self.nempty) + self.batch_idx[depth] = self.patch_partition(batch, depth, self.batch_size) + + def build_attn_mask(self, depth: int): + batch = self.batch_idx[depth] + mask = batch.view(-1, self.patch_size) + self.patch_mask[depth] = self._calc_attn_mask(mask) + + mask = batch.view(-1, self.patch_size, self.dilation) + mask = mask.transpose(1, 2).reshape(-1, self.patch_size) + self.dilate_mask[depth] = self._calc_attn_mask(mask) + + def _calc_attn_mask(self, mask: torch.Tensor): + attn_mask = mask.unsqueeze(2) - mask.unsqueeze(1) + attn_mask = attn_mask.masked_fill(attn_mask != 0, self.invalid_mask_value) + return attn_mask + + def build_rel_pos(self, depth: int): + key = self.key(depth, self.nempty) + key = self.patch_partition(key, depth) + x, y, z, _ = ocnn.octree.key2xyz(key, depth) + xyz = torch.stack([x, y, z], dim=1) + + xyz = xyz.view(-1, self.patch_size, 3) + self.rel_pos[depth] = xyz.unsqueeze(2) - xyz.unsqueeze(1) + + xyz = xyz.view(-1, self.patch_size, self.dilation, 3) + xyz = xyz.transpose(1, 2).reshape(-1, self.patch_size, 3) + self.dilate_pos[depth] = xyz.unsqueeze(2) - xyz.unsqueeze(1) + + def patch_partition(self, data: torch.Tensor, depth: int, fill_value=0): + num = self.nnum_a[depth] - self.nnum_t[depth] + tail = data.new_full((num,) + data.shape[1:], fill_value) + return torch.cat([data, tail], dim=0) + + def patch_reverse(self, data: torch.Tensor, depth: int): + return data[: self.nnum_t[depth]] + + +class MLP(torch.nn.Module): + def __init__( + self, + in_features: int, + hidden_features: Optional[int] = None, + out_features: Optional[int] = None, + activation=torch.nn.GELU, + drop: float = 0.0, + **kwargs + ): + super().__init__() + self.in_features = in_features + self.out_features = out_features or in_features + self.hidden_features = hidden_features or in_features + + self.fc1 = torch.nn.Linear(self.in_features, self.hidden_features) + self.act = activation() + self.fc2 = torch.nn.Linear(self.hidden_features, self.out_features) + self.drop = torch.nn.Dropout(drop, inplace=True) + + def forward(self, data: torch.Tensor): + data = self.fc1(data) + data = self.act(data) + data = self.drop(data) + data = self.fc2(data) + data = self.drop(data) + return data + + +class OctreeDWConvBn(torch.nn.Module): + def __init__( + self, + in_channels: int, + kernel_size: List[int] = [3], + stride: int = 1, + nempty: bool = False, + ): + super().__init__() + self.conv = dwconv.OctreeDWConv( + in_channels, kernel_size, nempty, use_bias=False + ) + self.bn = torch.nn.BatchNorm1d(in_channels) + + def forward(self, data: torch.Tensor, octree: Octree, depth: int): + out = self.conv(data, octree, depth) + out = self.bn(out) + return out + + +class RPE(torch.nn.Module): + def __init__(self, patch_size: int, num_heads: int, dilation: int = 1): + super().__init__() + self.patch_size = patch_size + self.num_heads = num_heads + self.dilation = dilation + self.pos_bnd = self.get_pos_bnd(patch_size) + self.rpe_num = 2 * self.pos_bnd + 1 + self.rpe_table = torch.nn.Parameter(torch.zeros(3 * self.rpe_num, num_heads)) + torch.nn.init.trunc_normal_(self.rpe_table, std=0.02) + + def get_pos_bnd(self, patch_size: int): + return int(0.8 * patch_size * self.dilation**0.5) + + def xyz2idx(self, xyz: torch.Tensor): + mul = torch.arange(3, device=xyz.device) * self.rpe_num + xyz = xyz.clamp(-self.pos_bnd, self.pos_bnd) + idx = xyz + (self.pos_bnd + mul) + return idx + + def forward(self, xyz): + idx = self.xyz2idx(xyz) + out = self.rpe_table.index_select(0, idx.reshape(-1)) + out = out.view(idx.shape + (-1,)).sum(3) + out = out.permute(0, 3, 1, 2) # (N, K, K, H) -> (N, H, K, K) + return out + + def extra_repr(self) -> str: + return "num_heads={}, pos_bnd={}, dilation={}".format( + self.num_heads, self.pos_bnd, self.dilation + ) # noqa + + +class OctreeAttention(torch.nn.Module): + def __init__( + self, + dim: int, + patch_size: int, + num_heads: int, + qkv_bias: bool = True, + qk_scale: Optional[float] = None, + attn_drop: float = 0.0, + proj_drop: float = 0.0, + dilation: int = 1, + use_rpe: bool = True, + ): + super().__init__() + self.dim = dim + self.patch_size = patch_size + self.num_heads = num_heads + self.dilation = dilation + self.use_rpe = use_rpe + self.scale = qk_scale or (dim // num_heads) ** -0.5 + + self.qkv = torch.nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = torch.nn.Dropout(attn_drop) + self.proj = torch.nn.Linear(dim, dim) + self.proj_drop = torch.nn.Dropout(proj_drop) + self.softmax = torch.nn.Softmax(dim=-1) + self.rpe = RPE(patch_size, num_heads, dilation) if use_rpe else None + + def forward(self, data: torch.Tensor, octree: OctreeT, depth: int): + H = self.num_heads + K = self.patch_size + C = self.dim + D = self.dilation + + # patch partition + data = octree.patch_partition(data, depth) + if D > 1: # dilation + rel_pos = octree.dilate_pos[depth] + mask = octree.dilate_mask[depth] + data = data.view(-1, K, D, C).transpose(1, 2).reshape(-1, C) + else: + rel_pos = octree.rel_pos[depth] + mask = octree.patch_mask[depth] + data = data.view(-1, K, C) + + # qkv + qkv = self.qkv(data).reshape(-1, K, 3, H, C // H).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # (N, H, K, C') + q = q * self.scale + + # attn + attn = q @ k.transpose(-2, -1) # (N, H, K, K) + attn = self.apply_rpe(attn, rel_pos) # (N, H, K, K) + attn = attn + mask.unsqueeze(1) + attn = self.softmax(attn) + attn = self.attn_drop(attn) + data = (attn @ v).transpose(1, 2).reshape(-1, C) + + # patch reverse + if D > 1: # dilation + data = data.view(-1, D, K, C).transpose(1, 2).reshape(-1, C) + data = octree.patch_reverse(data, depth) + + # ffn + data = self.proj(data) + data = self.proj_drop(data) + return data + + def apply_rpe(self, attn, rel_pos): + if self.use_rpe: + attn = attn + self.rpe(rel_pos) + return attn + + def extra_repr(self) -> str: + return "dim={}, patch_size={}, num_heads={}, dilation={}".format( + self.dim, self.patch_size, self.num_heads, self.dilation + ) # noqa + + +class OctFormerBlock(torch.nn.Module): + def __init__( + self, + dim: int, + num_heads: int, + patch_size: int = 32, + dilation: int = 0, + mlp_ratio: float = 4.0, + qkv_bias: bool = True, + qk_scale: Optional[float] = None, + attn_drop: float = 0.0, + proj_drop: float = 0.0, + drop_path: float = 0.0, + nempty: bool = True, + activation: torch.nn.Module = torch.nn.GELU, + **kwargs + ): + super().__init__() + self.norm1 = torch.nn.LayerNorm(dim) + self.attention = OctreeAttention( + dim, + patch_size, + num_heads, + qkv_bias, + qk_scale, + attn_drop, + proj_drop, + dilation, + ) + self.norm2 = torch.nn.LayerNorm(dim) + self.mlp = MLP(dim, int(dim * mlp_ratio), dim, activation, proj_drop) + self.drop_path = ocnn.nn.OctreeDropPath(drop_path, nempty) + self.cpe = OctreeDWConvBn(dim, nempty=nempty) + + def forward(self, data: torch.Tensor, octree: OctreeT, depth: int): + data = self.cpe(data, octree, depth) + data + attn = self.attention(self.norm1(data), octree, depth) + data = data + self.drop_path(attn, octree, depth) + ffn = self.mlp(self.norm2(data)) + data = data + self.drop_path(ffn, octree, depth) + return data + + +class OctFormerStage(torch.nn.Module): + def __init__( + self, + dim: int, + num_heads: int, + patch_size: int = 32, + dilation: int = 0, + mlp_ratio: float = 4.0, + qkv_bias: bool = True, + qk_scale: Optional[float] = None, + attn_drop: float = 0.0, + proj_drop: float = 0.0, + drop_path: float = 0.0, + nempty: bool = True, + activation: torch.nn.Module = torch.nn.GELU, + interval: int = 6, + use_checkpoint: bool = True, + num_blocks: int = 2, + octformer_block=OctFormerBlock, + **kwargs + ): + super().__init__() + self.num_blocks = num_blocks + self.use_checkpoint = use_checkpoint + self.interval = interval # normalization interval + self.num_norms = (num_blocks - 1) // self.interval + + self.blocks = torch.nn.ModuleList( + [ + octformer_block( + dim=dim, + num_heads=num_heads, + patch_size=patch_size, + dilation=1 if (i % 2 == 0) else dilation, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=proj_drop, + drop_path=( + drop_path[i] if isinstance(drop_path, list) else drop_path + ), + nempty=nempty, + activation=activation, + ) + for i in range(num_blocks) + ] + ) + # self.norms = torch.nn.ModuleList([ + # torch.nn.BatchNorm1d(dim) for _ in range(self.num_norms)]) + + def forward(self, data: torch.Tensor, octree: OctreeT, depth: int): + for i in range(self.num_blocks): + if self.use_checkpoint and self.training: + data = checkpoint(self.blocks[i], data, octree, depth) + else: + data = self.blocks[i](data, octree, depth) + # if i % self.interval == 0 and i != 0: + # data = self.norms[(i - 1) // self.interval](data) + return data + + +class OctFormerDecoder(torch.nn.Module): + def __init__( + self, channels: List[int], fpn_channel: int, nempty: bool, head_up: int = 1 + ): + super().__init__() + self.head_up = head_up + self.num_stages = len(channels) + self.conv1x1 = torch.nn.ModuleList( + [ + torch.nn.Linear(channels[i], fpn_channel) + for i in range(self.num_stages - 1, -1, -1) + ] + ) + self.upsample = ocnn.nn.OctreeUpsample("nearest", nempty) + self.conv3x3 = torch.nn.ModuleList( + [ + ocnn.modules.OctreeConvBnRelu( + fpn_channel, fpn_channel, kernel_size=[3], stride=1, nempty=nempty + ) + for _ in range(self.num_stages) + ] + ) + self.up_conv = torch.nn.ModuleList( + [ + ocnn.modules.OctreeDeconvBnRelu( + fpn_channel, fpn_channel, kernel_size=[3], stride=2, nempty=nempty + ) + for _ in range(self.head_up) + ] + ) + + def forward(self, features: Dict[int, torch.Tensor], octree: Octree): + depth = min(features.keys()) + depth_max = max(features.keys()) + assert self.num_stages == len(features) + + feature = self.conv1x1[0](features[depth]) + conv_out = self.conv3x3[0](feature, octree, depth) + out = self.upsample(conv_out, octree, depth, depth_max) + for i in range(1, self.num_stages): + depth_i = depth + i + feature = self.upsample(feature, octree, depth_i - 1) + feature = self.conv1x1[i](features[depth_i]) + feature + conv_out = self.conv3x3[i](feature, octree, depth_i) + out = out + self.upsample(conv_out, octree, depth_i, depth_max) + for i in range(self.head_up): + out = self.up_conv[i](out, octree, depth_max + i) + return out + + +class PatchEmbed(torch.nn.Module): + def __init__( + self, + in_channels: int = 3, + dim: int = 96, + num_down: int = 2, + nempty: bool = True, + **kwargs + ): + super().__init__() + self.num_stages = num_down + self.delta_depth = -num_down + channels = [int(dim * 2**i) for i in range(-self.num_stages, 1)] + + self.convs = torch.nn.ModuleList( + [ + ocnn.modules.OctreeConvBnRelu( + in_channels if i == 0 else channels[i], + channels[i], + kernel_size=[3], + stride=1, + nempty=nempty, + ) + for i in range(self.num_stages) + ] + ) + self.downsamples = torch.nn.ModuleList( + [ + ocnn.modules.OctreeConvBnRelu( + channels[i], + channels[i + 1], + kernel_size=[2], + stride=2, + nempty=nempty, + ) + for i in range(self.num_stages) + ] + ) + self.proj = ocnn.modules.OctreeConvBnRelu( + channels[-1], dim, kernel_size=[3], stride=1, nempty=nempty + ) + + def forward(self, data: torch.Tensor, octree: Octree, depth: int): + # TODO: reduce to single input + for i in range(self.num_stages): + depth_i = depth - i + data = self.convs[i](data, octree, depth_i) + data = self.downsamples[i](data, octree, depth_i) + data = self.proj(data, octree, depth_i - 1) + return data + + +class Downsample(torch.nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: List[int] = (2,), + nempty: bool = True, + ): + super().__init__() + self.norm = torch.nn.BatchNorm1d(out_channels) + self.conv = ocnn.nn.OctreeConv( + in_channels, + out_channels, + kernel_size, + stride=2, + nempty=nempty, + use_bias=True, + ) + + def forward(self, data: torch.Tensor, octree: Octree, depth: int): + data = self.conv(data, octree, depth) + data = self.norm(data) + return data + + +@MODELS.register_module("OctFormer-v1m1") +class OctFormer(torch.nn.Module): + def __init__( + self, + in_channels, + num_classes, + fpn_channels=168, + channels=(96, 192, 384, 384), + num_blocks=(2, 2, 18, 2), + num_heads=(6, 12, 24, 24), + patch_size=26, + stem_down=2, + head_up=2, + dilation=4, + drop_path=0.5, + nempty=True, + octree_scale_factor=10.24, + octree_depth=11, + octree_full_depth=2, + ): + super().__init__() + assert ocnn is not None, "Please follow `README.md` to install ocnn.`" + assert dwconv is not None, "Please follow `README.md` to install dwconv.`" + + self.patch_size = patch_size + self.dilation = dilation + self.nempty = nempty + self.num_stages = len(num_blocks) + self.stem_down = stem_down + self.octree_scale_factor = octree_scale_factor + self.octree_depth = octree_depth + self.octree_full_depth = octree_full_depth + drop_ratio = torch.linspace(0, drop_path, sum(num_blocks)).tolist() + + self.patch_embed = PatchEmbed(in_channels, channels[0], stem_down, nempty) + self.layers = torch.nn.ModuleList( + [ + OctFormerStage( + dim=channels[i], + num_heads=num_heads[i], + patch_size=patch_size, + drop_path=drop_ratio[ + sum(num_blocks[:i]) : sum(num_blocks[: i + 1]) + ], + dilation=dilation, + nempty=nempty, + num_blocks=num_blocks[i], + ) + for i in range(self.num_stages) + ] + ) + self.downsamples = torch.nn.ModuleList( + [ + Downsample(channels[i], channels[i + 1], kernel_size=[2], nempty=nempty) + for i in range(self.num_stages - 1) + ] + ) + self.decoder = OctFormerDecoder( + channels=channels, fpn_channel=fpn_channels, nempty=nempty, head_up=head_up + ) + self.interp = ocnn.nn.OctreeInterp("nearest", nempty) + self.seg_head = ( + nn.Sequential( + nn.Linear(fpn_channels, fpn_channels), + torch.nn.BatchNorm1d(fpn_channels), + nn.ReLU(inplace=True), + nn.Linear(fpn_channels, num_classes), + ) + if num_classes > 0 + else nn.Identity() + ) + + def points2octree(self, points): + octree = ocnn.octree.Octree(self.octree_depth, self.octree_full_depth) + octree.build_octree(points) + return octree + + def forward(self, data_dict): + coord = data_dict["coord"] + normal = data_dict["normal"] + feat = data_dict["feat"] + offset = data_dict["offset"] + batch = offset2batch(offset) + + point = Points( + points=coord / self.octree_scale_factor, + normals=normal, + features=feat, + batch_id=batch.unsqueeze(-1), + batch_size=len(offset), + ) + octree = ocnn.octree.Octree( + depth=self.octree_depth, + full_depth=self.octree_full_depth, + batch_size=len(offset), + device=coord.device, + ) + octree.build_octree(point) + octree.construct_all_neigh() + + feat = self.patch_embed(octree.features[octree.depth], octree, octree.depth) + depth = octree.depth - self.stem_down # current octree depth + octree = OctreeT( + octree, + self.patch_size, + self.dilation, + self.nempty, + max_depth=depth, + start_depth=depth - self.num_stages + 1, + ) + features = {} + for i in range(self.num_stages): + depth_i = depth - i + feat = self.layers[i](feat, octree, depth_i) + features[depth_i] = feat + if i < self.num_stages - 1: + feat = self.downsamples[i](feat, octree, depth_i) + out = self.decoder(features, octree) + # interp representation to points before Octreeization + query_pts = torch.cat([point.points, point.batch_id], dim=1).contiguous() + out = self.interp(out, octree, octree.depth, query_pts) + out = self.seg_head(out) + return out diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_group/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_group/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5d9f35f2a05f4a88043a45f2da64faf21f38f520 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_group/__init__.py @@ -0,0 +1 @@ +from .point_group_v1m1_base import PointGroup diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_group/point_group_v1m1_base.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_group/point_group_v1m1_base.py new file mode 100644 index 0000000000000000000000000000000000000000..2c36d3fe009fa33136d92605f1c810cff8892959 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_group/point_group_v1m1_base.py @@ -0,0 +1,174 @@ +""" +PointGroup for instance segmentation + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com), Chengyao Wang +Please cite our work if the code is helpful to you. +""" + +from functools import partial +import torch +import torch.nn as nn +import torch.nn.functional as F + +try: + from pointgroup_ops import ballquery_batch_p, bfs_cluster +except ImportError: + ballquery_batch_p, bfs_cluster = None, None + +from pointcept.models.utils import offset2batch, batch2offset + +from pointcept.models.builder import MODELS, build_model + + +@MODELS.register_module("PG-v1m1") +class PointGroup(nn.Module): + def __init__( + self, + backbone, + backbone_out_channels=64, + semantic_num_classes=20, + semantic_ignore_index=-1, + segment_ignore_index=(-1, 0, 1), + instance_ignore_index=-1, + cluster_thresh=1.5, + cluster_closed_points=300, + cluster_propose_points=100, + cluster_min_points=50, + voxel_size=0.02, + ): + super().__init__() + norm_fn = partial(nn.BatchNorm1d, eps=1e-3, momentum=0.01) + self.semantic_num_classes = semantic_num_classes + self.segment_ignore_index = segment_ignore_index + self.semantic_ignore_index = semantic_ignore_index + self.instance_ignore_index = instance_ignore_index + self.cluster_thresh = cluster_thresh + self.cluster_closed_points = cluster_closed_points + self.cluster_propose_points = cluster_propose_points + self.cluster_min_points = cluster_min_points + self.voxel_size = voxel_size + self.backbone = build_model(backbone) + self.bias_head = nn.Sequential( + nn.Linear(backbone_out_channels, backbone_out_channels), + norm_fn(backbone_out_channels), + nn.ReLU(), + nn.Linear(backbone_out_channels, 3), + ) + self.seg_head = nn.Linear(backbone_out_channels, semantic_num_classes) + self.ce_criteria = torch.nn.CrossEntropyLoss(ignore_index=semantic_ignore_index) + + def forward(self, data_dict): + coord = data_dict["coord"] + segment = data_dict["segment"] + instance = data_dict["instance"] + instance_centroid = data_dict["instance_centroid"] + offset = data_dict["offset"] + + feat = self.backbone(data_dict) + bias_pred = self.bias_head(feat) + logit_pred = self.seg_head(feat) + + # compute loss + seg_loss = self.ce_criteria(logit_pred, segment) + + mask = (instance != self.instance_ignore_index).float() + bias_gt = instance_centroid - coord + bias_dist = torch.sum(torch.abs(bias_pred - bias_gt), dim=-1) + bias_l1_loss = torch.sum(bias_dist * mask) / (torch.sum(mask) + 1e-8) + + bias_pred_norm = bias_pred / ( + torch.norm(bias_pred, p=2, dim=1, keepdim=True) + 1e-8 + ) + bias_gt_norm = bias_gt / (torch.norm(bias_gt, p=2, dim=1, keepdim=True) + 1e-8) + cosine_similarity = -(bias_pred_norm * bias_gt_norm).sum(-1) + bias_cosine_loss = torch.sum(cosine_similarity * mask) / ( + torch.sum(mask) + 1e-8 + ) + + loss = seg_loss + bias_l1_loss + bias_cosine_loss + return_dict = dict( + loss=loss, + seg_loss=seg_loss, + bias_l1_loss=bias_l1_loss, + bias_cosine_loss=bias_cosine_loss, + ) + + if not self.training: + center_pred = coord + bias_pred + center_pred /= self.voxel_size + logit_pred = F.softmax(logit_pred, dim=-1) + segment_pred = torch.max(logit_pred, 1)[1] # [n] + # cluster + mask = ( + ~torch.concat( + [ + (segment_pred == index).unsqueeze(-1) + for index in self.segment_ignore_index + ], + dim=1, + ) + .sum(-1) + .bool() + ) + + if mask.sum() == 0: + proposals_idx = torch.zeros(0).int() + proposals_offset = torch.zeros(1).int() + else: + center_pred_ = center_pred[mask] + segment_pred_ = segment_pred[mask] + + batch_ = offset2batch(offset)[mask] + offset_ = nn.ConstantPad1d((1, 0), 0)(batch2offset(batch_)) + idx, start_len = ballquery_batch_p( + center_pred_, + batch_.int(), + offset_.int(), + self.cluster_thresh, + self.cluster_closed_points, + ) + proposals_idx, proposals_offset = bfs_cluster( + segment_pred_.int().cpu(), + idx.cpu(), + start_len.cpu(), + self.cluster_min_points, + ) + proposals_idx[:, 1] = ( + mask.nonzero().view(-1)[proposals_idx[:, 1].long()].int() + ) + + # get proposal + proposals_pred = torch.zeros( + (proposals_offset.shape[0] - 1, center_pred.shape[0]), dtype=torch.int + ) + proposals_pred[proposals_idx[:, 0].long(), proposals_idx[:, 1].long()] = 1 + instance_pred = segment_pred[ + proposals_idx[:, 1][proposals_offset[:-1].long()].long() + ] + proposals_point_num = proposals_pred.sum(1) + proposals_mask = proposals_point_num > self.cluster_propose_points + proposals_pred = proposals_pred[proposals_mask] + instance_pred = instance_pred[proposals_mask] + + pred_scores = [] + pred_classes = [] + pred_masks = proposals_pred.detach().cpu() + for proposal_id in range(len(proposals_pred)): + segment_ = proposals_pred[proposal_id] + confidence_ = logit_pred[ + segment_.bool(), instance_pred[proposal_id] + ].mean() + object_ = instance_pred[proposal_id] + pred_scores.append(confidence_) + pred_classes.append(object_) + if len(pred_scores) > 0: + pred_scores = torch.stack(pred_scores).cpu() + pred_classes = torch.stack(pred_classes).cpu() + else: + pred_scores = torch.tensor([]) + pred_classes = torch.tensor([]) + + return_dict["pred_scores"] = pred_scores + return_dict["pred_masks"] = pred_masks + return_dict["pred_classes"] = pred_classes + return return_dict diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_group/utils.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_group/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d095b5bc89291ec30418c0aaf0eb9f672fe26ccc --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_group/utils.py @@ -0,0 +1,176 @@ +import torch +from torch.autograd import Function +import pointgroup_ops + + +class BallQueryBatchP(Function): + @staticmethod + def forward(ctx, coords, batch_idxs, batch_offsets, radius, meanActive): + """ + :param ctx: + :param coords: (n, 3) float + :param batch_idxs: (n) int + :param batch_offsets: (B+1) int + :param radius: float + :param meanActive: int + :return: idx (nActive), int + :return: start_len (n, 2), int + """ + + n = coords.size(0) + + assert coords.is_contiguous() and coords.is_cuda + assert batch_idxs.is_contiguous() and batch_idxs.is_cuda + assert batch_offsets.is_contiguous() and batch_offsets.is_cuda + + while True: + idx = torch.cuda.IntTensor(n * meanActive).zero_() + start_len = torch.cuda.IntTensor(n, 2).zero_() + nActive = pointgroup_ops.ballquery_batch_p( + coords, batch_idxs, batch_offsets, idx, start_len, n, meanActive, radius + ) + if nActive <= n * meanActive: + break + meanActive = int(nActive // n + 1) + idx = idx[:nActive] + + return idx, start_len + + @staticmethod + def backward(ctx, a=None, b=None): + return None, None, None + + +ballquery_batch_p = BallQueryBatchP.apply + + +class Clustering: + def __init__( + self, + ignored_labels, + class_mapping, + thresh=0.03, + closed_points=300, + min_points=50, + propose_points=100, + score_func=torch.max, + ) -> None: + self.ignored_labels = ignored_labels + self.thresh = thresh + self.closed_points = closed_points + self.min_points = min_points + self.class_mapping = class_mapping + self.propose_points = propose_points + self.score_func = score_func + + def cluster(self, vertices, scores): + labels = torch.max(scores, 1)[1] # (N) long, cuda + proposals_idx, proposals_offset = self.cluster_(vertices, labels) + + ## debug + # import ipdb; ipdb.set_trace() + # colors = np.array(create_color_palette())[labels.cpu()] + # write_triangle_mesh(vertices, colors, None, 'semantics.ply') + + # scatter + proposals_pred = torch.zeros( + (proposals_offset.shape[0] - 1, vertices.shape[0]), dtype=torch.int + ) # (nProposal, N), int, cuda + proposals_pred[proposals_idx[:, 0].long(), proposals_idx[:, 1].long()] = 1 + labels = labels[proposals_idx[:, 1][proposals_offset[:-1].long()].long()] + + proposals_pointnum = proposals_pred.sum(1) + npoint_mask = proposals_pointnum > self.propose_points + + proposals_pred = proposals_pred[npoint_mask] + labels = labels[npoint_mask] + return proposals_pred, labels + + def cluster_(self, vertices, labels): + """ + :param batch_idxs: (N), int, cuda + :labels: 0-19 + """ + batch_idxs = torch.zeros_like(labels) + + mask_non_ignored = torch.ones_like(labels).bool() + for ignored_label in self.ignored_labels: + mask_non_ignored = mask_non_ignored & ( + self.class_mapping[labels] != ignored_label + ) + object_idxs = mask_non_ignored.nonzero().view(-1) + + vertices_ = vertices[object_idxs].float() + labels_ = labels[object_idxs].int() + + if vertices_.numel() == 0: + return torch.zeros((0, 2)).int(), torch.zeros(1).int() + + batch_idxs_ = batch_idxs[object_idxs].int() + batch_offsets_ = torch.FloatTensor([0, object_idxs.shape[0]]).int().cuda() + + idx, start_len = ballquery_batch_p( + vertices_, batch_idxs_, batch_offsets_, self.thresh, self.closed_points + ) + proposals_idx, proposals_offset = bfs_cluster( + labels_.cpu(), idx.cpu(), start_len.cpu(), self.min_points + ) + proposals_idx[:, 1] = object_idxs[proposals_idx[:, 1].long()].int() + + return proposals_idx, proposals_offset + + def get_instances(self, vertices, scores): + proposals_pred, labels = self.cluster(vertices, scores) + instances = {} + for proposal_id in range(len(proposals_pred)): + clusters_i = proposals_pred[proposal_id] + score = scores[clusters_i.bool(), labels[proposal_id]] + score = self.score_func(score) + instances[proposal_id] = {} + instances[proposal_id]["conf"] = score.cpu().numpy() + instances[proposal_id]["label_id"] = self.class_mapping.cpu()[ + labels[proposal_id] + ] + instances[proposal_id]["pred_mask"] = clusters_i.cpu().numpy() + return instances + + +class BFSCluster(Function): + @staticmethod + def forward(ctx, semantic_label, ball_query_idxs, start_len, threshold): + """ + :param ctx: + :param semantic_label: (N), int + :param ball_query_idxs: (nActive), int + :param start_len: (N, 2), int + :return: cluster_idxs: int (sumNPoint, 2), dim 0 for cluster_id, dim 1 for corresponding point idxs in N + :return: cluster_offsets: int (nCluster + 1) + """ + + N = start_len.size(0) + + assert semantic_label.is_contiguous() + assert ball_query_idxs.is_contiguous() + assert start_len.is_contiguous() + + cluster_idxs = semantic_label.new() + cluster_offsets = semantic_label.new() + + pointgroup_ops.bfs_cluster( + semantic_label, + ball_query_idxs, + start_len, + cluster_idxs, + cluster_offsets, + N, + threshold, + ) + + return cluster_idxs, cluster_offsets + + @staticmethod + def backward(ctx, a=None): + return None + + +bfs_cluster = BFSCluster.apply diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_prompt_training/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_prompt_training/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f4c980b70b8c49dfc51625623071f21d6405d856 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_prompt_training/__init__.py @@ -0,0 +1,4 @@ +from .point_prompt_training_v1m1_language_guided import * +from .point_prompt_training_v1m2_decoupled import * + +from .prompt_driven_normalization import PDNorm diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_prompt_training/point_prompt_training_v1m1_language_guided.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_prompt_training/point_prompt_training_v1m1_language_guided.py new file mode 100644 index 0000000000000000000000000000000000000000..443fc482ba02c319905a387c58fa7ec5032f6a07 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_prompt_training/point_prompt_training_v1m1_language_guided.py @@ -0,0 +1,118 @@ +""" +Point Prompt Training + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from functools import partial +from collections import OrderedDict + +import torch +import torch.nn as nn +from pointcept.models.utils.structure import Point +from pointcept.models.builder import MODELS +from pointcept.models.losses import build_criteria + + +@MODELS.register_module("PPT-v1m1") +class PointPromptTraining(nn.Module): + """ + PointPromptTraining provides Data-driven Context and enables multi-dataset training with + Language-driven Categorical Alignment. PDNorm is supported by SpUNet-v1m3 to adapt the + backbone to a specific dataset with a given dataset condition and context. + """ + + def __init__( + self, + backbone=None, + criteria=None, + backbone_out_channels=96, + context_channels=256, + conditions=("Structured3D", "ScanNet", "S3DIS"), + template="[x]", + clip_model="ViT-B/16", + # fmt: off + class_name=( + "wall", "floor", "cabinet", "bed", "chair", "sofa", "table", "door", + "window", "bookshelf", "bookcase", "picture", "counter", "desk", "shelves", "curtain", + "dresser", "pillow", "mirror", "ceiling", "refrigerator", "television", "shower curtain", "nightstand", + "toilet", "sink", "lamp", "bathtub", "garbagebin", "board", "beam", "column", + "clutter", "otherstructure", "otherfurniture", "otherprop", + ), + valid_index=( + (0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 33, 34, 35), + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15, 20, 22, 24, 25, 27, 34), + (0, 1, 4, 5, 6, 7, 8, 10, 19, 29, 30, 31, 32), + ), + # fmt: on + backbone_mode=False, + ): + super().__init__() + assert len(conditions) == len(valid_index) + assert backbone.type in ["SpUNet-v1m3", "PT-v2m3", "PT-v3m1"] + self.backbone = MODELS.build(backbone) + self.criteria = build_criteria(criteria) + self.conditions = conditions + self.valid_index = valid_index + self.embedding_table = nn.Embedding(len(conditions), context_channels) + self.backbone_mode = backbone_mode + if not self.backbone_mode: + import clip + + clip_model, _ = clip.load( + clip_model, device="cpu", download_root="./.cache/clip" + ) + clip_model.requires_grad_(False) + class_prompt = [template.replace("[x]", name) for name in class_name] + class_token = clip.tokenize(class_prompt) + class_embedding = clip_model.encode_text(class_token) + class_embedding = class_embedding / class_embedding.norm( + dim=-1, keepdim=True + ) + self.register_buffer("class_embedding", class_embedding) + self.proj_head = nn.Linear( + backbone_out_channels, clip_model.text_projection.shape[1] + ) + self.logit_scale = clip_model.logit_scale + + def forward(self, data_dict): + condition = data_dict["condition"][0] + assert condition in self.conditions + context = self.embedding_table( + torch.tensor( + [self.conditions.index(condition)], device=data_dict["coord"].device + ) + ) + data_dict["context"] = context + point = self.backbone(data_dict) + # Backbone added after v1.5.0 return Point instead of feat and use DefaultSegmentorV2 + # TODO: remove this part after make all backbone return Point only. + if isinstance(point, Point): + feat = point.feat + else: + feat = point + if self.backbone_mode: + # PPT serve as a multi-dataset backbone when enable backbone mode + return feat + feat = self.proj_head(feat) + feat = feat / feat.norm(dim=-1, keepdim=True) + sim = ( + feat + @ self.class_embedding[ + self.valid_index[self.conditions.index(condition)], : + ].t() + ) + logit_scale = self.logit_scale.exp() + seg_logits = logit_scale * sim + # train + if self.training: + loss = self.criteria(seg_logits, data_dict["segment"]) + return dict(loss=loss) + # eval + elif "segment" in data_dict.keys(): + loss = self.criteria(seg_logits, data_dict["segment"]) + return dict(loss=loss, seg_logits=seg_logits) + # test + else: + return dict(seg_logits=seg_logits) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_prompt_training/point_prompt_training_v1m2_decoupled.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_prompt_training/point_prompt_training_v1m2_decoupled.py new file mode 100644 index 0000000000000000000000000000000000000000..9ad9c6bf1fcb5e17b139d6c06b44b589fcee816f --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_prompt_training/point_prompt_training_v1m2_decoupled.py @@ -0,0 +1,79 @@ +""" +Point Prompt Training with decoupled segmentation head + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from functools import partial +from collections import OrderedDict + +import torch +import torch.nn as nn +from pointcept.models.utils.structure import Point +from pointcept.models.builder import MODELS +from pointcept.models.losses import build_criteria + + +@MODELS.register_module("PPT-v1m2") +class PointPromptTraining(nn.Module): + """ + PointPromptTraining v1m2 provides Data-driven Context and enables multi-dataset training with + Decoupled Segmentation Head. PDNorm is supported by SpUNet-v1m3 to adapt the + backbone to a specific dataset with a given dataset condition and context. + """ + + def __init__( + self, + backbone=None, + criteria=None, + backbone_out_channels=96, + context_channels=256, + conditions=("Structured3D", "ScanNet", "S3DIS"), + num_classes=(25, 20, 13), + backbone_mode=False, + ): + super().__init__() + assert len(conditions) == len(num_classes) + assert backbone.type in ["SpUNet-v1m3", "PT-v2m3", "PT-v3m1"] + self.backbone = MODELS.build(backbone) + self.criteria = build_criteria(criteria) + self.conditions = conditions + self.embedding_table = nn.Embedding(len(conditions), context_channels) + self.backbone_mode = backbone_mode + self.seg_heads = nn.ModuleList( + [nn.Linear(backbone_out_channels, num_cls) for num_cls in num_classes] + ) + + def forward(self, data_dict): + condition = data_dict["condition"][0] + assert condition in self.conditions + context = self.embedding_table( + torch.tensor( + [self.conditions.index(condition)], device=data_dict["coord"].device + ) + ) + data_dict["context"] = context + point = self.backbone(data_dict) + # Backbone added after v1.5.0 return Point instead of feat and use DefaultSegmentorV2 + # TODO: remove this part after make all backbone return Point only. + if isinstance(point, Point): + feat = point.feat + else: + feat = point + if self.backbone_mode: + # PPT serve as a multi-dataset backbone when enable backbone mode + return feat + seg_head = self.seg_heads[self.conditions.index(condition)] + seg_logits = seg_head(feat) + # train + if self.training: + loss = self.criteria(seg_logits, data_dict["segment"]) + return dict(loss=loss) + # eval + elif "segment" in data_dict.keys(): + loss = self.criteria(seg_logits, data_dict["segment"]) + return dict(loss=loss, seg_logits=seg_logits) + # test + else: + return dict(seg_logits=seg_logits) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_prompt_training/prompt_driven_normalization.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_prompt_training/prompt_driven_normalization.py new file mode 100644 index 0000000000000000000000000000000000000000..5d7d0d0c01dd4ccf939afeff870b5d72cab403a3 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_prompt_training/prompt_driven_normalization.py @@ -0,0 +1,47 @@ +import torch.nn as nn + +from pointcept.models.modules import PointModule, PointSequential +from pointcept.models.builder import MODULES + + +@MODULES.register_module() +class PDNorm(PointModule): + def __init__( + self, + num_features, + norm_layer, + context_channels=256, + conditions=("ScanNet", "S3DIS", "Structured3D"), + decouple=True, + adaptive=False, + ): + super().__init__() + self.conditions = conditions + self.decouple = decouple + self.adaptive = adaptive + if self.decouple: + self.norm = nn.ModuleList([norm_layer(num_features) for _ in conditions]) + else: + self.norm = norm_layer + if self.adaptive: + self.modulation = nn.Sequential( + nn.SiLU(), nn.Linear(context_channels, 2 * num_features, bias=True) + ) + + def forward(self, point): + assert {"feat", "condition"}.issubset(point.keys()) + if isinstance(point.condition, str): + condition = point.condition + else: + condition = point.condition[0] + if self.decouple: + assert condition in self.conditions + norm = self.norm[self.conditions.index(condition)] + else: + norm = self.norm + point.feat = norm(point.feat) + if self.adaptive: + assert "context" in point.keys() + shift, scale = self.modulation(point.context).chunk(2, dim=1) + point.feat = point.feat * (1.0 + scale) + shift + return point diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d6493a312bfcf559642e6a2cc77d96c3770f0dd1 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/__init__.py @@ -0,0 +1,3 @@ +from .point_transformer_seg import * +from .point_transformer_partseg import * +from .point_transformer_cls import * diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/point_transformer_cls.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/point_transformer_cls.py new file mode 100644 index 0000000000000000000000000000000000000000..8e12746fef73e9b3ee75b72942fbc8dc96e6e1bf --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/point_transformer_cls.py @@ -0,0 +1,131 @@ +""" +Point Transformer V1 for Object Classification + +Might be a bit different from the original paper + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import torch +import torch.nn as nn + +from .point_transformer_seg import TransitionDown, Bottleneck +from pointcept.models.builder import MODELS + + +class PointTransformerCls(nn.Module): + def __init__(self, block, blocks, in_channels=6, num_classes=40): + super().__init__() + self.in_channels = in_channels + self.in_planes, planes = in_channels, [32, 64, 128, 256, 512] + fpn_planes, fpnhead_planes, share_planes = 128, 64, 8 + stride, nsample = [1, 4, 4, 4, 4], [8, 16, 16, 16, 16] + self.enc1 = self._make_enc( + block, + planes[0], + blocks[0], + share_planes, + stride=stride[0], + nsample=nsample[0], + ) # N/1 + self.enc2 = self._make_enc( + block, + planes[1], + blocks[1], + share_planes, + stride=stride[1], + nsample=nsample[1], + ) # N/4 + self.enc3 = self._make_enc( + block, + planes[2], + blocks[2], + share_planes, + stride=stride[2], + nsample=nsample[2], + ) # N/16 + self.enc4 = self._make_enc( + block, + planes[3], + blocks[3], + share_planes, + stride=stride[3], + nsample=nsample[3], + ) # N/64 + self.enc5 = self._make_enc( + block, + planes[4], + blocks[4], + share_planes, + stride=stride[4], + nsample=nsample[4], + ) # N/256 + self.cls = nn.Sequential( + nn.Linear(planes[4], 256), + nn.BatchNorm1d(256), + nn.ReLU(inplace=True), + nn.Dropout(p=0.5), + nn.Linear(256, 128), + nn.BatchNorm1d(128), + nn.ReLU(inplace=True), + nn.Dropout(p=0.5), + nn.Linear(128, num_classes), + ) + + def _make_enc(self, block, planes, blocks, share_planes=8, stride=1, nsample=16): + layers = [ + TransitionDown(self.in_planes, planes * block.expansion, stride, nsample) + ] + self.in_planes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block(self.in_planes, self.in_planes, share_planes, nsample=nsample) + ) + return nn.Sequential(*layers) + + def forward(self, data_dict): + p0 = data_dict["coord"] + x0 = data_dict["feat"] + o0 = data_dict["offset"].int() + x0 = p0 if self.in_channels == 3 else torch.cat((p0, x0), 1) + p1, x1, o1 = self.enc1([p0, x0, o0]) + p2, x2, o2 = self.enc2([p1, x1, o1]) + p3, x3, o3 = self.enc3([p2, x2, o2]) + p4, x4, o4 = self.enc4([p3, x3, o3]) + p5, x5, o5 = self.enc5([p4, x4, o4]) + x = [] + for i in range(o5.shape[0]): + if i == 0: + s_i, e_i, cnt = 0, o5[0], o5[0] + else: + s_i, e_i, cnt = o5[i - 1], o5[i], o5[i] - o5[i - 1] + x_b = x5[s_i:e_i, :].sum(0, True) / cnt + x.append(x_b) + x = torch.cat(x, 0) + x = self.cls(x) + return x + + +@MODELS.register_module("PointTransformer-Cls26") +class PointTransformerCls26(PointTransformerCls): + def __init__(self, **kwargs): + super(PointTransformerCls26, self).__init__( + Bottleneck, [1, 1, 1, 1, 1], **kwargs + ) + + +@MODELS.register_module("PointTransformer-Cls38") +class PointTransformerCls38(PointTransformerCls): + def __init__(self, **kwargs): + super(PointTransformerCls38, self).__init__( + Bottleneck, [1, 2, 2, 2, 2], **kwargs + ) + + +@MODELS.register_module("PointTransformer-Cls50") +class PointTransformerCls50(PointTransformerCls): + def __init__(self, **kwargs): + super(PointTransformerCls50, self).__init__( + Bottleneck, [1, 2, 3, 5, 2], **kwargs + ) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/point_transformer_partseg.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/point_transformer_partseg.py new file mode 100644 index 0000000000000000000000000000000000000000..3326a9f7d6fd62a9394e434135615339a3c679f8 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/point_transformer_partseg.py @@ -0,0 +1,374 @@ +""" +Point Transformer V1 for Part Segmentation + +Might be a bit different from the original paper + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import torch +import torch.nn as nn +import einops +import pointops + +from pointcept.models.builder import MODELS +from .utils import LayerNorm1d + + +class PointTransformerLayer(nn.Module): + def __init__(self, in_planes, out_planes, share_planes=8, nsample=16): + super().__init__() + self.mid_planes = mid_planes = out_planes // 1 + self.out_planes = out_planes + self.share_planes = share_planes + self.nsample = nsample + self.linear_q = nn.Linear(in_planes, mid_planes) + self.linear_k = nn.Linear(in_planes, mid_planes) + self.linear_v = nn.Linear(in_planes, out_planes) + self.linear_p = nn.Sequential( + nn.Linear(3, 3), + LayerNorm1d(3), + nn.ReLU(inplace=True), + nn.Linear(3, out_planes), + ) + self.linear_w = nn.Sequential( + LayerNorm1d(mid_planes), + nn.ReLU(inplace=True), + nn.Linear(mid_planes, out_planes // share_planes), + LayerNorm1d(out_planes // share_planes), + nn.ReLU(inplace=True), + nn.Linear(out_planes // share_planes, out_planes // share_planes), + ) + self.softmax = nn.Softmax(dim=1) + + def forward(self, pxo) -> torch.Tensor: + p, x, o = pxo # (n, 3), (n, c), (b) + x_q, x_k, x_v = self.linear_q(x), self.linear_k(x), self.linear_v(x) + x_k, idx = pointops.knn_query_and_group( + x_k, p, o, new_xyz=p, new_offset=o, nsample=self.nsample, with_xyz=True + ) + x_v, _ = pointops.knn_query_and_group( + x_v, + p, + o, + new_xyz=p, + new_offset=o, + idx=idx, + nsample=self.nsample, + with_xyz=False, + ) + p_r, x_k = x_k[:, :, 0:3], x_k[:, :, 3:] + p_r = self.linear_p(p_r) + r_qk = ( + x_k + - x_q.unsqueeze(1) + + einops.reduce( + p_r, "n ns (i j) -> n ns j", reduction="sum", j=self.mid_planes + ) + ) + w = self.linear_w(r_qk) # (n, nsample, c) + w = self.softmax(w) + x = torch.einsum( + "n t s i, n t i -> n s i", + einops.rearrange(x_v + p_r, "n ns (s i) -> n ns s i", s=self.share_planes), + w, + ) + x = einops.rearrange(x, "n s i -> n (s i)") + return x + + +class TransitionDown(nn.Module): + def __init__(self, in_planes, out_planes, stride=1, nsample=16): + super().__init__() + self.stride, self.nsample = stride, nsample + if stride != 1: + self.linear = nn.Linear(3 + in_planes, out_planes, bias=False) + self.pool = nn.MaxPool1d(nsample) + else: + self.linear = nn.Linear(in_planes, out_planes, bias=False) + self.bn = nn.BatchNorm1d(out_planes) + self.relu = nn.ReLU(inplace=True) + + def forward(self, pxo): + p, x, o = pxo # (n, 3), (n, c), (b) + if self.stride != 1: + n_o, count = [o[0].item() // self.stride], o[0].item() // self.stride + for i in range(1, o.shape[0]): + count += (o[i].item() - o[i - 1].item()) // self.stride + n_o.append(count) + n_o = torch.cuda.IntTensor(n_o) + idx = pointops.farthest_point_sampling(p, o, n_o) # (m) + n_p = p[idx.long(), :] # (m, 3) + x, _ = pointops.knn_query_and_group( + x, + p, + offset=o, + new_xyz=n_p, + new_offset=n_o, + nsample=self.nsample, + with_xyz=True, + ) + x = self.relu( + self.bn(self.linear(x).transpose(1, 2).contiguous()) + ) # (m, c, nsample) + x = self.pool(x).squeeze(-1) # (m, c) + p, o = n_p, n_o + else: + x = self.relu(self.bn(self.linear(x))) # (n, c) + return [p, x, o] + + +class TransitionUp(nn.Module): + def __init__(self, in_planes, out_planes=None, num_shape_class=None): + super().__init__() + if out_planes is None: + self.num_shape_class = num_shape_class + if num_shape_class is not None: + self.linear1 = nn.Sequential( + nn.Linear(2 * in_planes + 1024, in_planes), + nn.BatchNorm1d(in_planes), + nn.ReLU(inplace=True), + ) + else: + self.linear1 = nn.Sequential( + nn.Linear(2 * in_planes, in_planes), + nn.BatchNorm1d(in_planes), + nn.ReLU(inplace=True), + ) + + self.linear2 = nn.Sequential( + nn.Linear(in_planes, in_planes), nn.ReLU(inplace=True) + ) + if num_shape_class is not None: + self.linear3 = nn.Sequential( + nn.Linear(num_shape_class, 1024), nn.ReLU(inplace=True) + ) + else: + self.linear1 = nn.Sequential( + nn.Linear(out_planes, out_planes), + nn.BatchNorm1d(out_planes), + nn.ReLU(inplace=True), + ) + self.linear2 = nn.Sequential( + nn.Linear(in_planes, out_planes), + nn.BatchNorm1d(out_planes), + nn.ReLU(inplace=True), + ) + + def forward(self, pxo1, pxo2=None, y=None): + if pxo2 is None: + _, x, o = pxo1 # (n, 3), (n, c), (b) + x_tmp = [] + for i in range(o.shape[0]): + if i == 0: + s_i, e_i, cnt = 0, o[0], o[0] + else: + s_i, e_i, cnt = o[i - 1], o[i], o[i] - o[i - 1] + x_b = x[s_i:e_i, :] + y_b = y[i].unsqueeze(-1).unsqueeze(-1).long() + y_onehot = torch.zeros(1, self.num_shape_class).cuda() # (1, l) + y_onehot.scatter_(1, y_b, 1) # (1, l) + x_b = torch.cat( + ( + x_b, + self.linear2(x_b.sum(0, True) / cnt).repeat(cnt, 1), + self.linear3(y_onehot).repeat(cnt, 1), + ), + dim=1, + ) + x_tmp.append(x_b) + x = torch.cat(x_tmp, 0) + x = self.linear1(x) + else: + p1, x1, o1 = pxo1 + p2, x2, o2 = pxo2 + x = self.linear1(x1) + pointops.interpolation( + p2, p1, self.linear2(x2), o2, o1 + ) + return x + + +class Bottleneck(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, share_planes=8, nsample=16): + super(Bottleneck, self).__init__() + self.linear1 = nn.Linear(in_planes, planes, bias=False) + self.bn1 = nn.BatchNorm1d(planes) + self.transformer = PointTransformerLayer(planes, planes, share_planes, nsample) + self.bn2 = nn.BatchNorm1d(planes) + self.linear3 = nn.Linear(planes, planes * self.expansion, bias=False) + self.bn3 = nn.BatchNorm1d(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + + def forward(self, pxo): + p, x, o = pxo # (n, 3), (n, c), (b) + identity = x + x = self.relu(self.bn1(self.linear1(x))) + x = self.relu(self.bn2(self.transformer([p, x, o]))) + x = self.bn3(self.linear3(x)) + x += identity + x = self.relu(x) + return [p, x, o] + + +class PointTransformerSeg(nn.Module): + def __init__( + self, block, blocks, in_channels=6, num_classes=50, num_shape_classes=None + ): + super().__init__() + self.in_channels = in_channels + self.num_classes = num_classes + self.num_shape_classes = num_shape_classes + self.in_planes, planes = in_channels, [32, 64, 128, 256, 512] + fpn_planes, fpnhead_planes, share_planes = 128, 64, 8 + stride, nsample = [1, 4, 4, 4, 4], [8, 16, 16, 16, 16] + self.enc1 = self._make_enc( + block, + planes[0], + blocks[0], + share_planes, + stride=stride[0], + nsample=nsample[0], + ) # N/1 + self.enc2 = self._make_enc( + block, + planes[1], + blocks[1], + share_planes, + stride=stride[1], + nsample=nsample[1], + ) # N/4 + self.enc3 = self._make_enc( + block, + planes[2], + blocks[2], + share_planes, + stride=stride[2], + nsample=nsample[2], + ) # N/16 + self.enc4 = self._make_enc( + block, + planes[3], + blocks[3], + share_planes, + stride=stride[3], + nsample=nsample[3], + ) # N/64 + self.enc5 = self._make_enc( + block, + planes[4], + blocks[4], + share_planes, + stride=stride[4], + nsample=nsample[4], + ) # N/256 + self.dec5 = self._make_dec( + block, + planes[4], + 1, + share_planes, + num_shape_classes=num_shape_classes, + nsample=nsample[4], + is_head=True, + ) # transform p5 + self.dec4 = self._make_dec( + block, planes[3], 1, share_planes, nsample=nsample[3] + ) # fusion p5 and p4 + self.dec3 = self._make_dec( + block, planes[2], 1, share_planes, nsample=nsample[2] + ) # fusion p4 and p3 + self.dec2 = self._make_dec( + block, planes[1], 1, share_planes, nsample=nsample[1] + ) # fusion p3 and p2 + self.dec1 = self._make_dec( + block, planes[0], 1, share_planes, nsample=nsample[0] + ) # fusion p2 and p1 + self.cls = nn.Sequential( + nn.Linear(planes[0], planes[0]), + nn.BatchNorm1d(planes[0]), + nn.ReLU(inplace=True), + nn.Linear(planes[0], num_classes), + ) + + def _make_enc(self, block, planes, blocks, share_planes=8, stride=1, nsample=16): + layers = [ + TransitionDown(self.in_planes, planes * block.expansion, stride, nsample) + ] + self.in_planes = planes * block.expansion + for _ in range(blocks): + layers.append( + block(self.in_planes, self.in_planes, share_planes, nsample=nsample) + ) + return nn.Sequential(*layers) + + def _make_dec( + self, + block, + planes, + blocks, + share_planes=8, + num_shape_classes=None, + nsample=16, + is_head=False, + ): + layers = [ + TransitionUp( + self.in_planes, + None if is_head else planes * block.expansion, + num_shape_classes, + ) + ] + self.in_planes = planes * block.expansion + for _ in range(blocks): + layers.append( + block(self.in_planes, self.in_planes, share_planes, nsample=nsample) + ) + return nn.Sequential(*layers) + + def forward(self, data_dict): + p0 = data_dict["coord"] + x0 = data_dict["feat"] + o0 = data_dict["offset"].int() + if self.num_shape_classes is not None: + y = data_dict["cls_token"] + p1, x1, o1 = self.enc1([p0, x0, o0]) + p2, x2, o2 = self.enc2([p1, x1, o1]) + p3, x3, o3 = self.enc3([p2, x2, o2]) + p4, x4, o4 = self.enc4([p3, x3, o3]) + p5, x5, o5 = self.enc5([p4, x4, o4]) + if self.num_shape_classes is not None: + x5 = self.dec5[1:]([p5, self.dec5[0]([p5, x5, o5], y=y), o5])[1] + else: + x5 = self.dec5[1:]([p5, self.dec5[0]([p5, x5, o5]), o5])[1] + x4 = self.dec4[1:]([p4, self.dec4[0]([p4, x4, o4], [p5, x5, o5]), o4])[1] + x3 = self.dec3[1:]([p3, self.dec3[0]([p3, x3, o3], [p4, x4, o4]), o3])[1] + x2 = self.dec2[1:]([p2, self.dec2[0]([p2, x2, o2], [p3, x3, o3]), o2])[1] + x1 = self.dec1[1:]([p1, self.dec1[0]([p1, x1, o1], [p2, x2, o2]), o1])[1] + x = self.cls(x1) + return x + + +@MODELS.register_module("PointTransformer-PartSeg26") +class PointTransformerSeg26(PointTransformerSeg): + def __init__(self, **kwargs): + super(PointTransformerSeg26, self).__init__( + Bottleneck, [1, 1, 1, 1, 1], **kwargs + ) + + +@MODELS.register_module("PointTransformer-PartSeg38") +class PointTransformerSeg38(PointTransformerSeg): + def __init__(self, **kwargs): + super(PointTransformerSeg38, self).__init__( + Bottleneck, [1, 2, 2, 2, 2], **kwargs + ) + + +@MODELS.register_module("PointTransformer-PartSeg50") +class PointTransformerSeg50(PointTransformerSeg): + def __init__(self, **kwargs): + super(PointTransformerSeg50, self).__init__( + Bottleneck, [1, 2, 3, 5, 2], **kwargs + ) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/point_transformer_seg.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/point_transformer_seg.py new file mode 100644 index 0000000000000000000000000000000000000000..248cacad1ade65e48fa4686560fb40617a0ea449 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/point_transformer_seg.py @@ -0,0 +1,327 @@ +""" +Point Transformer V1 for Semantic Segmentation + +Might be a bit different from the original paper + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import torch +import torch.nn as nn +import einops +import pointops + +from pointcept.models.builder import MODELS +from .utils import LayerNorm1d + + +class PointTransformerLayer(nn.Module): + def __init__(self, in_planes, out_planes, share_planes=8, nsample=16): + super().__init__() + self.mid_planes = mid_planes = out_planes // 1 + self.out_planes = out_planes + self.share_planes = share_planes + self.nsample = nsample + self.linear_q = nn.Linear(in_planes, mid_planes) + self.linear_k = nn.Linear(in_planes, mid_planes) + self.linear_v = nn.Linear(in_planes, out_planes) + self.linear_p = nn.Sequential( + nn.Linear(3, 3), + LayerNorm1d(3), + nn.ReLU(inplace=True), + nn.Linear(3, out_planes), + ) + self.linear_w = nn.Sequential( + LayerNorm1d(mid_planes), + nn.ReLU(inplace=True), + nn.Linear(mid_planes, out_planes // share_planes), + LayerNorm1d(out_planes // share_planes), + nn.ReLU(inplace=True), + nn.Linear(out_planes // share_planes, out_planes // share_planes), + ) + self.softmax = nn.Softmax(dim=1) + + def forward(self, pxo) -> torch.Tensor: + p, x, o = pxo # (n, 3), (n, c), (b) + x_q, x_k, x_v = self.linear_q(x), self.linear_k(x), self.linear_v(x) + x_k, idx = pointops.knn_query_and_group( + x_k, p, o, new_xyz=p, new_offset=o, nsample=self.nsample, with_xyz=True + ) + x_v, _ = pointops.knn_query_and_group( + x_v, + p, + o, + new_xyz=p, + new_offset=o, + idx=idx, + nsample=self.nsample, + with_xyz=False, + ) + p_r, x_k = x_k[:, :, 0:3], x_k[:, :, 3:] + p_r = self.linear_p(p_r) + r_qk = ( + x_k + - x_q.unsqueeze(1) + + einops.reduce( + p_r, "n ns (i j) -> n ns j", reduction="sum", j=self.mid_planes + ) + ) + w = self.linear_w(r_qk) # (n, nsample, c) + w = self.softmax(w) + x = torch.einsum( + "n t s i, n t i -> n s i", + einops.rearrange(x_v + p_r, "n ns (s i) -> n ns s i", s=self.share_planes), + w, + ) + x = einops.rearrange(x, "n s i -> n (s i)") + return x + + +class TransitionDown(nn.Module): + def __init__(self, in_planes, out_planes, stride=1, nsample=16): + super().__init__() + self.stride, self.nsample = stride, nsample + if stride != 1: + self.linear = nn.Linear(3 + in_planes, out_planes, bias=False) + self.pool = nn.MaxPool1d(nsample) + else: + self.linear = nn.Linear(in_planes, out_planes, bias=False) + self.bn = nn.BatchNorm1d(out_planes) + self.relu = nn.ReLU(inplace=True) + + def forward(self, pxo): + p, x, o = pxo # (n, 3), (n, c), (b) + if self.stride != 1: + n_o, count = [o[0].item() // self.stride], o[0].item() // self.stride + for i in range(1, o.shape[0]): + count += (o[i].item() - o[i - 1].item()) // self.stride + n_o.append(count) + n_o = torch.cuda.IntTensor(n_o) + idx = pointops.farthest_point_sampling(p, o, n_o) # (m) + n_p = p[idx.long(), :] # (m, 3) + x, _ = pointops.knn_query_and_group( + x, + p, + offset=o, + new_xyz=n_p, + new_offset=n_o, + nsample=self.nsample, + with_xyz=True, + ) + x = self.relu( + self.bn(self.linear(x).transpose(1, 2).contiguous()) + ) # (m, c, nsample) + x = self.pool(x).squeeze(-1) # (m, c) + p, o = n_p, n_o + else: + x = self.relu(self.bn(self.linear(x))) # (n, c) + return [p, x, o] + + +class TransitionUp(nn.Module): + def __init__(self, in_planes, out_planes=None): + super().__init__() + if out_planes is None: + self.linear1 = nn.Sequential( + nn.Linear(2 * in_planes, in_planes), + nn.BatchNorm1d(in_planes), + nn.ReLU(inplace=True), + ) + self.linear2 = nn.Sequential( + nn.Linear(in_planes, in_planes), nn.ReLU(inplace=True) + ) + else: + self.linear1 = nn.Sequential( + nn.Linear(out_planes, out_planes), + nn.BatchNorm1d(out_planes), + nn.ReLU(inplace=True), + ) + self.linear2 = nn.Sequential( + nn.Linear(in_planes, out_planes), + nn.BatchNorm1d(out_planes), + nn.ReLU(inplace=True), + ) + + def forward(self, pxo1, pxo2=None): + if pxo2 is None: + _, x, o = pxo1 # (n, 3), (n, c), (b) + x_tmp = [] + for i in range(o.shape[0]): + if i == 0: + s_i, e_i, cnt = 0, o[0], o[0] + else: + s_i, e_i, cnt = o[i - 1], o[i], o[i] - o[i - 1] + x_b = x[s_i:e_i, :] + x_b = torch.cat( + (x_b, self.linear2(x_b.sum(0, True) / cnt).repeat(cnt, 1)), 1 + ) + x_tmp.append(x_b) + x = torch.cat(x_tmp, 0) + x = self.linear1(x) + else: + p1, x1, o1 = pxo1 + p2, x2, o2 = pxo2 + x = self.linear1(x1) + pointops.interpolation( + p2, p1, self.linear2(x2), o2, o1 + ) + return x + + +class Bottleneck(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, share_planes=8, nsample=16): + super(Bottleneck, self).__init__() + self.linear1 = nn.Linear(in_planes, planes, bias=False) + self.bn1 = nn.BatchNorm1d(planes) + self.transformer = PointTransformerLayer(planes, planes, share_planes, nsample) + self.bn2 = nn.BatchNorm1d(planes) + self.linear3 = nn.Linear(planes, planes * self.expansion, bias=False) + self.bn3 = nn.BatchNorm1d(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + + def forward(self, pxo): + p, x, o = pxo # (n, 3), (n, c), (b) + identity = x + x = self.relu(self.bn1(self.linear1(x))) + x = self.relu(self.bn2(self.transformer([p, x, o]))) + x = self.bn3(self.linear3(x)) + x += identity + x = self.relu(x) + return [p, x, o] + + +class PointTransformerSeg(nn.Module): + def __init__(self, block, blocks, in_channels=6, num_classes=13): + super().__init__() + self.in_channels = in_channels + self.in_planes, planes = in_channels, [32, 64, 128, 256, 512] + fpn_planes, fpnhead_planes, share_planes = 128, 64, 8 + stride, nsample = [1, 4, 4, 4, 4], [8, 16, 16, 16, 16] + self.enc1 = self._make_enc( + block, + planes[0], + blocks[0], + share_planes, + stride=stride[0], + nsample=nsample[0], + ) # N/1 + self.enc2 = self._make_enc( + block, + planes[1], + blocks[1], + share_planes, + stride=stride[1], + nsample=nsample[1], + ) # N/4 + self.enc3 = self._make_enc( + block, + planes[2], + blocks[2], + share_planes, + stride=stride[2], + nsample=nsample[2], + ) # N/16 + self.enc4 = self._make_enc( + block, + planes[3], + blocks[3], + share_planes, + stride=stride[3], + nsample=nsample[3], + ) # N/64 + self.enc5 = self._make_enc( + block, + planes[4], + blocks[4], + share_planes, + stride=stride[4], + nsample=nsample[4], + ) # N/256 + self.dec5 = self._make_dec( + block, planes[4], 1, share_planes, nsample=nsample[4], is_head=True + ) # transform p5 + self.dec4 = self._make_dec( + block, planes[3], 1, share_planes, nsample=nsample[3] + ) # fusion p5 and p4 + self.dec3 = self._make_dec( + block, planes[2], 1, share_planes, nsample=nsample[2] + ) # fusion p4 and p3 + self.dec2 = self._make_dec( + block, planes[1], 1, share_planes, nsample=nsample[1] + ) # fusion p3 and p2 + self.dec1 = self._make_dec( + block, planes[0], 1, share_planes, nsample=nsample[0] + ) # fusion p2 and p1 + self.cls = nn.Sequential( + nn.Linear(planes[0], planes[0]), + nn.BatchNorm1d(planes[0]), + nn.ReLU(inplace=True), + nn.Linear(planes[0], num_classes), + ) + + def _make_enc(self, block, planes, blocks, share_planes=8, stride=1, nsample=16): + layers = [ + TransitionDown(self.in_planes, planes * block.expansion, stride, nsample) + ] + self.in_planes = planes * block.expansion + for _ in range(blocks): + layers.append( + block(self.in_planes, self.in_planes, share_planes, nsample=nsample) + ) + return nn.Sequential(*layers) + + def _make_dec( + self, block, planes, blocks, share_planes=8, nsample=16, is_head=False + ): + layers = [ + TransitionUp(self.in_planes, None if is_head else planes * block.expansion) + ] + self.in_planes = planes * block.expansion + for _ in range(blocks): + layers.append( + block(self.in_planes, self.in_planes, share_planes, nsample=nsample) + ) + return nn.Sequential(*layers) + + def forward(self, data_dict): + p0 = data_dict["coord"] + x0 = data_dict["feat"] + o0 = data_dict["offset"].int() + p1, x1, o1 = self.enc1([p0, x0, o0]) + p2, x2, o2 = self.enc2([p1, x1, o1]) + p3, x3, o3 = self.enc3([p2, x2, o2]) + p4, x4, o4 = self.enc4([p3, x3, o3]) + p5, x5, o5 = self.enc5([p4, x4, o4]) + x5 = self.dec5[1:]([p5, self.dec5[0]([p5, x5, o5]), o5])[1] + x4 = self.dec4[1:]([p4, self.dec4[0]([p4, x4, o4], [p5, x5, o5]), o4])[1] + x3 = self.dec3[1:]([p3, self.dec3[0]([p3, x3, o3], [p4, x4, o4]), o3])[1] + x2 = self.dec2[1:]([p2, self.dec2[0]([p2, x2, o2], [p3, x3, o3]), o2])[1] + x1 = self.dec1[1:]([p1, self.dec1[0]([p1, x1, o1], [p2, x2, o2]), o1])[1] + x = self.cls(x1) + return x + + +@MODELS.register_module("PointTransformer-Seg26") +class PointTransformerSeg26(PointTransformerSeg): + def __init__(self, **kwargs): + super(PointTransformerSeg26, self).__init__( + Bottleneck, [1, 1, 1, 1, 1], **kwargs + ) + + +@MODELS.register_module("PointTransformer-Seg38") +class PointTransformerSeg38(PointTransformerSeg): + def __init__(self, **kwargs): + super(PointTransformerSeg38, self).__init__( + Bottleneck, [1, 2, 2, 2, 2], **kwargs + ) + + +@MODELS.register_module("PointTransformer-Seg50") +class PointTransformerSeg50(PointTransformerSeg): + def __init__(self, **kwargs): + super(PointTransformerSeg50, self).__init__( + Bottleneck, [1, 2, 3, 5, 2], **kwargs + ) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/utils.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c5687701835bb1f8a8936ea5ae5d52285567dc77 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer/utils.py @@ -0,0 +1,14 @@ +import torch +import torch.nn as nn + +torch.nn.LayerNorm + + +class LayerNorm1d(nn.BatchNorm1d): + def forward(self, input: torch.Tensor) -> torch.Tensor: + return ( + super() + .forward(input.transpose(1, 2).contiguous()) + .transpose(1, 2) + .contiguous() + ) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v2/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e9689fa2518b599bc6f94e6f8d0ea461859b8909 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v2/__init__.py @@ -0,0 +1,10 @@ +""" +Point Transformer V2 + +Copyright (c) Xiaoyang Wu (xiaoyang.wu@connect.hku.hk). All Rights Reserved. +Please cite our work if you use any part of the code. +""" + +from .point_transformer_v2m1_origin import * +from .point_transformer_v2m2_base import * +from .point_transformer_v2m3_pdnorm import * diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v2/point_transformer_v2m1_origin.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v2/point_transformer_v2m1_origin.py new file mode 100644 index 0000000000000000000000000000000000000000..b325d9eb7d5e1507ce62d5cbf60bb000cf83acbc --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v2/point_transformer_v2m1_origin.py @@ -0,0 +1,614 @@ +""" +Point Transformer V2 mode 1 + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from copy import deepcopy +import math +import torch +import torch.nn as nn +from torch.utils.checkpoint import checkpoint +from torch_geometric.nn.pool import voxel_grid +from torch_scatter import segment_csr + +import einops +from timm.models.layers import DropPath +import pointops + +from pointcept.models.builder import MODELS +from pointcept.models.utils import offset2batch, batch2offset + + +class GroupedLinear(nn.Module): + __constants__ = ["in_features", "out_features", "groups"] + in_features: int + out_features: int + groups: int + weight: torch.Tensor + + def __init__( + self, in_features: int, out_features: int, groups: int, device=None, dtype=None + ) -> None: + factory_kwargs = {"device": device, "dtype": dtype} + super(GroupedLinear, self).__init__() + self.in_features = in_features + self.out_features = out_features + self.groups = groups + assert in_features & groups == 0 + assert out_features % groups == 0 + # for convenient, currently only support out_features == groups, one output + assert out_features == groups + self.weight = nn.Parameter(torch.empty((1, in_features), **factory_kwargs)) + self.reset_parameters() + + def reset_parameters(self) -> None: + nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + return ( + (input * self.weight) + .reshape( + list(input.shape[:-1]) + [self.groups, input.shape[-1] // self.groups] + ) + .sum(-1) + ) + + def extra_repr(self) -> str: + return "in_features={}, out_features={}, bias={}".format( + self.in_features, self.out_features, self.bias is not None + ) + + +class PointBatchNorm(nn.Module): + """ + Batch Normalization for Point Clouds data in shape of [B*N, C], [B*N, L, C] + """ + + def __init__(self, embed_channels): + super().__init__() + self.norm = nn.BatchNorm1d(embed_channels) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + if input.dim() == 3: + return ( + self.norm(input.transpose(1, 2).contiguous()) + .transpose(1, 2) + .contiguous() + ) + elif input.dim() == 2: + return self.norm(input) + else: + raise NotImplementedError + + +class GroupedVectorAttention(nn.Module): + def __init__( + self, + embed_channels, + groups, + attn_drop_rate=0.0, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + ): + super(GroupedVectorAttention, self).__init__() + self.embed_channels = embed_channels + self.groups = groups + assert embed_channels % groups == 0 + self.attn_drop_rate = attn_drop_rate + self.qkv_bias = qkv_bias + self.pe_multiplier = pe_multiplier + self.pe_bias = pe_bias + + self.linear_q = nn.Sequential( + nn.Linear(embed_channels, embed_channels, bias=qkv_bias), + PointBatchNorm(embed_channels), + nn.ReLU(inplace=True), + ) + self.linear_k = nn.Sequential( + nn.Linear(embed_channels, embed_channels, bias=qkv_bias), + PointBatchNorm(embed_channels), + nn.ReLU(inplace=True), + ) + + self.linear_v = nn.Linear(embed_channels, embed_channels, bias=qkv_bias) + + if self.pe_multiplier: + self.linear_p_multiplier = nn.Sequential( + nn.Linear(3, embed_channels), + PointBatchNorm(embed_channels), + nn.ReLU(inplace=True), + nn.Linear(embed_channels, embed_channels), + ) + if self.pe_bias: + self.linear_p_bias = nn.Sequential( + nn.Linear(3, embed_channels), + PointBatchNorm(embed_channels), + nn.ReLU(inplace=True), + nn.Linear(embed_channels, embed_channels), + ) + self.weight_encoding = nn.Sequential( + GroupedLinear(embed_channels, groups, groups), + PointBatchNorm(groups), + nn.ReLU(inplace=True), + nn.Linear(groups, groups), + ) + self.softmax = nn.Softmax(dim=1) + self.attn_drop = nn.Dropout(attn_drop_rate) + + def forward(self, feat, coord, reference_index): + query, key, value = ( + self.linear_q(feat), + self.linear_k(feat), + self.linear_v(feat), + ) + key = pointops.grouping(reference_index, key, coord, with_xyz=True) + value = pointops.grouping(reference_index, value, coord, with_xyz=False) + pos, key = key[:, :, 0:3], key[:, :, 3:] + relation_qk = key - query.unsqueeze(1) + if self.pe_multiplier: + pem = self.linear_p_multiplier(pos) + relation_qk = relation_qk * pem + if self.pe_bias: + peb = self.linear_p_bias(pos) + relation_qk = relation_qk + peb + value = value + peb + + weight = self.weight_encoding(relation_qk) + weight = self.attn_drop(self.softmax(weight)) + + mask = torch.sign(reference_index + 1) + weight = torch.einsum("n s g, n s -> n s g", weight, mask) + value = einops.rearrange(value, "n ns (g i) -> n ns g i", g=self.groups) + feat = torch.einsum("n s g i, n s g -> n g i", value, weight) + feat = einops.rearrange(feat, "n g i -> n (g i)") + return feat + + +class Block(nn.Module): + def __init__( + self, + embed_channels, + groups, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.0, + enable_checkpoint=False, + ): + super(Block, self).__init__() + self.attn = GroupedVectorAttention( + embed_channels=embed_channels, + groups=groups, + qkv_bias=qkv_bias, + attn_drop_rate=attn_drop_rate, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + ) + self.fc1 = nn.Linear(embed_channels, embed_channels, bias=False) + self.fc3 = nn.Linear(embed_channels, embed_channels, bias=False) + self.norm1 = PointBatchNorm(embed_channels) + self.norm2 = PointBatchNorm(embed_channels) + self.norm3 = PointBatchNorm(embed_channels) + self.act = nn.ReLU(inplace=True) + self.enable_checkpoint = enable_checkpoint + self.drop_path = ( + DropPath(drop_path_rate) if drop_path_rate > 0.0 else nn.Identity() + ) + + def forward(self, points, reference_index): + coord, feat, offset = points + identity = feat + feat = self.act(self.norm1(self.fc1(feat))) + feat = ( + self.attn(feat, coord, reference_index) + if not self.enable_checkpoint + else checkpoint(self.attn, feat, coord, reference_index) + ) + feat = self.act(self.norm2(feat)) + feat = self.norm3(self.fc3(feat)) + feat = identity + self.drop_path(feat) + feat = self.act(feat) + return [coord, feat, offset] + + +class BlockSequence(nn.Module): + def __init__( + self, + depth, + embed_channels, + groups, + neighbours=16, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.0, + enable_checkpoint=False, + ): + super(BlockSequence, self).__init__() + + if isinstance(drop_path_rate, list): + drop_path_rates = drop_path_rate + assert len(drop_path_rates) == depth + elif isinstance(drop_path_rate, float): + drop_path_rates = [deepcopy(drop_path_rate) for _ in range(depth)] + else: + drop_path_rates = [0.0 for _ in range(depth)] + + self.neighbours = neighbours + self.blocks = nn.ModuleList() + for i in range(depth): + block = Block( + embed_channels=embed_channels, + groups=groups, + qkv_bias=qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rates[i], + enable_checkpoint=enable_checkpoint, + ) + self.blocks.append(block) + + def forward(self, points): + coord, feat, offset = points + # reference index query of neighbourhood attention + # for windows attention, modify reference index query method + reference_index, _ = pointops.knn_query(self.neighbours, coord, offset) + for block in self.blocks: + points = block(points, reference_index) + return points + + +class GridPool(nn.Module): + """ + Partition-based Pooling (Grid Pooling) + """ + + def __init__(self, in_channels, out_channels, grid_size, bias=False): + super(GridPool, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.grid_size = grid_size + + self.fc = nn.Linear(in_channels, out_channels, bias=bias) + self.norm = PointBatchNorm(out_channels) + self.act = nn.ReLU(inplace=True) + + def forward(self, points, start=None): + coord, feat, offset = points + batch = offset2batch(offset) + feat = self.act(self.norm(self.fc(feat))) + start = ( + segment_csr( + coord, + torch.cat([batch.new_zeros(1), torch.cumsum(batch.bincount(), dim=0)]), + reduce="min", + ) + if start is None + else start + ) + cluster = voxel_grid( + pos=coord - start[batch], size=self.grid_size, batch=batch, start=0 + ) + unique, cluster, counts = torch.unique( + cluster, sorted=True, return_inverse=True, return_counts=True + ) + _, sorted_cluster_indices = torch.sort(cluster) + idx_ptr = torch.cat([counts.new_zeros(1), torch.cumsum(counts, dim=0)]) + coord = segment_csr(coord[sorted_cluster_indices], idx_ptr, reduce="mean") + feat = segment_csr(feat[sorted_cluster_indices], idx_ptr, reduce="max") + batch = batch[idx_ptr[:-1]] + offset = batch2offset(batch) + return [coord, feat, offset], cluster + + +class UnpoolWithSkip(nn.Module): + """ + Map Unpooling with skip connection + """ + + def __init__( + self, + in_channels, + skip_channels, + out_channels, + bias=True, + skip=True, + backend="map", + ): + super(UnpoolWithSkip, self).__init__() + self.in_channels = in_channels + self.skip_channels = skip_channels + self.out_channels = out_channels + self.skip = skip + self.backend = backend + assert self.backend in ["map", "interp"] + + self.proj = nn.Sequential( + nn.Linear(in_channels, out_channels, bias=bias), + PointBatchNorm(out_channels), + nn.ReLU(inplace=True), + ) + self.proj_skip = nn.Sequential( + nn.Linear(skip_channels, out_channels, bias=bias), + PointBatchNorm(out_channels), + nn.ReLU(inplace=True), + ) + + def forward(self, points, skip_points, cluster=None): + coord, feat, offset = points + skip_coord, skip_feat, skip_offset = skip_points + if self.backend == "map" and cluster is not None: + feat = self.proj(feat)[cluster] + else: + feat = pointops.interpolation( + coord, skip_coord, self.proj(feat), offset, skip_offset + ) + if self.skip: + feat = feat + self.proj_skip(skip_feat) + return [skip_coord, feat, skip_offset] + + +class Encoder(nn.Module): + def __init__( + self, + depth, + in_channels, + embed_channels, + groups, + grid_size=None, + neighbours=16, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=None, + drop_path_rate=None, + enable_checkpoint=False, + ): + super(Encoder, self).__init__() + + self.down = GridPool( + in_channels=in_channels, + out_channels=embed_channels, + grid_size=grid_size, + ) + + self.blocks = BlockSequence( + depth=depth, + embed_channels=embed_channels, + groups=groups, + neighbours=neighbours, + qkv_bias=qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate if attn_drop_rate is not None else 0.0, + drop_path_rate=drop_path_rate if drop_path_rate is not None else 0.0, + enable_checkpoint=enable_checkpoint, + ) + + def forward(self, points): + points, cluster = self.down(points) + return self.blocks(points), cluster + + +class Decoder(nn.Module): + def __init__( + self, + in_channels, + skip_channels, + embed_channels, + groups, + depth, + neighbours=16, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=None, + drop_path_rate=None, + enable_checkpoint=False, + unpool_backend="map", + ): + super(Decoder, self).__init__() + + self.up = UnpoolWithSkip( + in_channels=in_channels, + out_channels=embed_channels, + skip_channels=skip_channels, + backend=unpool_backend, + ) + + self.blocks = BlockSequence( + depth=depth, + embed_channels=embed_channels, + groups=groups, + neighbours=neighbours, + qkv_bias=qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate if attn_drop_rate is not None else 0.0, + drop_path_rate=drop_path_rate if drop_path_rate is not None else 0.0, + enable_checkpoint=enable_checkpoint, + ) + + def forward(self, points, skip_points, cluster): + points = self.up(points, skip_points, cluster) + return self.blocks(points) + + +class GVAPatchEmbed(nn.Module): + def __init__( + self, + depth, + in_channels, + embed_channels, + groups, + neighbours=16, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.0, + enable_checkpoint=False, + ): + super(GVAPatchEmbed, self).__init__() + self.in_channels = in_channels + self.embed_channels = embed_channels + self.proj = nn.Sequential( + nn.Linear(in_channels, embed_channels, bias=False), + PointBatchNorm(embed_channels), + nn.ReLU(inplace=True), + ) + self.blocks = BlockSequence( + depth=depth, + embed_channels=embed_channels, + groups=groups, + neighbours=neighbours, + qkv_bias=qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rate, + enable_checkpoint=enable_checkpoint, + ) + + def forward(self, points): + coord, feat, offset = points + feat = self.proj(feat) + return self.blocks([coord, feat, offset]) + + +@MODELS.register_module("PT-v2m1") +class PointTransformerV2(nn.Module): + def __init__( + self, + in_channels, + num_classes, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.12, 0.24, 0.48), + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0, + enable_checkpoint=False, + unpool_backend="map", + ): + super(PointTransformerV2, self).__init__() + self.in_channels = in_channels + self.num_classes = num_classes + self.num_stages = len(enc_depths) + assert self.num_stages == len(dec_depths) + assert self.num_stages == len(enc_channels) + assert self.num_stages == len(dec_channels) + assert self.num_stages == len(enc_groups) + assert self.num_stages == len(dec_groups) + assert self.num_stages == len(enc_neighbours) + assert self.num_stages == len(dec_neighbours) + assert self.num_stages == len(grid_sizes) + self.patch_embed = GVAPatchEmbed( + in_channels=in_channels, + embed_channels=patch_embed_channels, + groups=patch_embed_groups, + depth=patch_embed_depth, + neighbours=patch_embed_neighbours, + qkv_bias=attn_qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + enable_checkpoint=enable_checkpoint, + ) + + enc_dp_rates = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(enc_depths)) + ] + dec_dp_rates = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(dec_depths)) + ] + enc_channels = [patch_embed_channels] + list(enc_channels) + dec_channels = list(dec_channels) + [enc_channels[-1]] + self.enc_stages = nn.ModuleList() + self.dec_stages = nn.ModuleList() + for i in range(self.num_stages): + enc = Encoder( + depth=enc_depths[i], + in_channels=enc_channels[i], + embed_channels=enc_channels[i + 1], + groups=enc_groups[i], + grid_size=grid_sizes[i], + neighbours=enc_neighbours[i], + qkv_bias=attn_qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + drop_path_rate=enc_dp_rates[ + sum(enc_depths[:i]) : sum(enc_depths[: i + 1]) + ], + enable_checkpoint=enable_checkpoint, + ) + dec = Decoder( + depth=dec_depths[i], + in_channels=dec_channels[i + 1], + skip_channels=enc_channels[i], + embed_channels=dec_channels[i], + groups=dec_groups[i], + neighbours=dec_neighbours[i], + qkv_bias=attn_qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + drop_path_rate=dec_dp_rates[ + sum(dec_depths[:i]) : sum(dec_depths[: i + 1]) + ], + enable_checkpoint=enable_checkpoint, + unpool_backend=unpool_backend, + ) + self.enc_stages.append(enc) + self.dec_stages.append(dec) + self.seg_head = ( + nn.Sequential( + nn.Linear(dec_channels[0], dec_channels[0]), + PointBatchNorm(dec_channels[0]), + nn.ReLU(inplace=True), + nn.Linear(dec_channels[0], num_classes), + ) + if num_classes > 0 + else nn.Identity() + ) + + def forward(self, data_dict): + coord = data_dict["coord"] + feat = data_dict["feat"] + offset = data_dict["offset"].int() + + # a batch of point cloud is a list of coord, feat and offset + points = [coord, feat, offset] + points = self.patch_embed(points) + skips = [[points]] + for i in range(self.num_stages): + points, cluster = self.enc_stages[i](points) + skips[-1].append(cluster) # record grid cluster of pooling + skips.append([points]) # record points info of current stage + + points = skips.pop(-1)[0] # unpooling points info in the last enc stage + for i in reversed(range(self.num_stages)): + skip_points, cluster = skips.pop(-1) + points = self.dec_stages[i](points, skip_points, cluster) + coord, feat, offset = points + seg_logits = self.seg_head(feat) + return seg_logits diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v2/point_transformer_v2m2_base.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v2/point_transformer_v2m2_base.py new file mode 100644 index 0000000000000000000000000000000000000000..dec45ff92504cf184505d0cad96d30346613df46 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v2/point_transformer_v2m2_base.py @@ -0,0 +1,576 @@ +""" +Point Transformer V2 Mode 2 (recommend) + +Disable Grouped Linear + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from copy import deepcopy +import math +import torch +import torch.nn as nn +from torch.utils.checkpoint import checkpoint +from torch_geometric.nn.pool import voxel_grid +from torch_scatter import segment_csr + +import einops +from timm.models.layers import DropPath +import pointops + +from pointcept.models.builder import MODELS +from pointcept.models.utils import offset2batch, batch2offset + + +class PointBatchNorm(nn.Module): + """ + Batch Normalization for Point Clouds data in shape of [B*N, C], [B*N, L, C] + """ + + def __init__(self, embed_channels): + super().__init__() + self.norm = nn.BatchNorm1d(embed_channels) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + if input.dim() == 3: + return ( + self.norm(input.transpose(1, 2).contiguous()) + .transpose(1, 2) + .contiguous() + ) + elif input.dim() == 2: + return self.norm(input) + else: + raise NotImplementedError + + +class GroupedVectorAttention(nn.Module): + def __init__( + self, + embed_channels, + groups, + attn_drop_rate=0.0, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + ): + super(GroupedVectorAttention, self).__init__() + self.embed_channels = embed_channels + self.groups = groups + assert embed_channels % groups == 0 + self.attn_drop_rate = attn_drop_rate + self.qkv_bias = qkv_bias + self.pe_multiplier = pe_multiplier + self.pe_bias = pe_bias + + self.linear_q = nn.Sequential( + nn.Linear(embed_channels, embed_channels, bias=qkv_bias), + PointBatchNorm(embed_channels), + nn.ReLU(inplace=True), + ) + self.linear_k = nn.Sequential( + nn.Linear(embed_channels, embed_channels, bias=qkv_bias), + PointBatchNorm(embed_channels), + nn.ReLU(inplace=True), + ) + + self.linear_v = nn.Linear(embed_channels, embed_channels, bias=qkv_bias) + + if self.pe_multiplier: + self.linear_p_multiplier = nn.Sequential( + nn.Linear(3, embed_channels), + PointBatchNorm(embed_channels), + nn.ReLU(inplace=True), + nn.Linear(embed_channels, embed_channels), + ) + if self.pe_bias: + self.linear_p_bias = nn.Sequential( + nn.Linear(3, embed_channels), + PointBatchNorm(embed_channels), + nn.ReLU(inplace=True), + nn.Linear(embed_channels, embed_channels), + ) + self.weight_encoding = nn.Sequential( + nn.Linear(embed_channels, groups), + PointBatchNorm(groups), + nn.ReLU(inplace=True), + nn.Linear(groups, groups), + ) + self.softmax = nn.Softmax(dim=1) + self.attn_drop = nn.Dropout(attn_drop_rate) + + def forward(self, feat, coord, reference_index): + query, key, value = ( + self.linear_q(feat), + self.linear_k(feat), + self.linear_v(feat), + ) + key = pointops.grouping(reference_index, key, coord, with_xyz=True) + value = pointops.grouping(reference_index, value, coord, with_xyz=False) + pos, key = key[:, :, 0:3], key[:, :, 3:] + relation_qk = key - query.unsqueeze(1) + if self.pe_multiplier: + pem = self.linear_p_multiplier(pos) + relation_qk = relation_qk * pem + if self.pe_bias: + peb = self.linear_p_bias(pos) + relation_qk = relation_qk + peb + value = value + peb + + weight = self.weight_encoding(relation_qk) + weight = self.attn_drop(self.softmax(weight)) + + mask = torch.sign(reference_index + 1) + weight = torch.einsum("n s g, n s -> n s g", weight, mask) + value = einops.rearrange(value, "n ns (g i) -> n ns g i", g=self.groups) + feat = torch.einsum("n s g i, n s g -> n g i", value, weight) + feat = einops.rearrange(feat, "n g i -> n (g i)") + return feat + + +class Block(nn.Module): + def __init__( + self, + embed_channels, + groups, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.0, + enable_checkpoint=False, + ): + super(Block, self).__init__() + self.attn = GroupedVectorAttention( + embed_channels=embed_channels, + groups=groups, + qkv_bias=qkv_bias, + attn_drop_rate=attn_drop_rate, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + ) + self.fc1 = nn.Linear(embed_channels, embed_channels, bias=False) + self.fc3 = nn.Linear(embed_channels, embed_channels, bias=False) + self.norm1 = PointBatchNorm(embed_channels) + self.norm2 = PointBatchNorm(embed_channels) + self.norm3 = PointBatchNorm(embed_channels) + self.act = nn.ReLU(inplace=True) + self.enable_checkpoint = enable_checkpoint + self.drop_path = ( + DropPath(drop_path_rate) if drop_path_rate > 0.0 else nn.Identity() + ) + + def forward(self, points, reference_index): + coord, feat, offset = points + identity = feat + feat = self.act(self.norm1(self.fc1(feat))) + feat = ( + self.attn(feat, coord, reference_index) + if not self.enable_checkpoint + else checkpoint(self.attn, feat, coord, reference_index) + ) + feat = self.act(self.norm2(feat)) + feat = self.norm3(self.fc3(feat)) + feat = identity + self.drop_path(feat) + feat = self.act(feat) + return [coord, feat, offset] + + +class BlockSequence(nn.Module): + def __init__( + self, + depth, + embed_channels, + groups, + neighbours=16, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.0, + enable_checkpoint=False, + ): + super(BlockSequence, self).__init__() + + if isinstance(drop_path_rate, list): + drop_path_rates = drop_path_rate + assert len(drop_path_rates) == depth + elif isinstance(drop_path_rate, float): + drop_path_rates = [deepcopy(drop_path_rate) for _ in range(depth)] + else: + drop_path_rates = [0.0 for _ in range(depth)] + + self.neighbours = neighbours + self.blocks = nn.ModuleList() + for i in range(depth): + block = Block( + embed_channels=embed_channels, + groups=groups, + qkv_bias=qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rates[i], + enable_checkpoint=enable_checkpoint, + ) + self.blocks.append(block) + + def forward(self, points): + coord, feat, offset = points + # reference index query of neighbourhood attention + # for windows attention, modify reference index query method + reference_index, _ = pointops.knn_query(self.neighbours, coord, offset) + for block in self.blocks: + points = block(points, reference_index) + return points + + +class GridPool(nn.Module): + """ + Partition-based Pooling (Grid Pooling) + """ + + def __init__(self, in_channels, out_channels, grid_size, bias=False): + super(GridPool, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.grid_size = grid_size + + self.fc = nn.Linear(in_channels, out_channels, bias=bias) + self.norm = PointBatchNorm(out_channels) + self.act = nn.ReLU(inplace=True) + + def forward(self, points, start=None): + coord, feat, offset = points + batch = offset2batch(offset) + feat = self.act(self.norm(self.fc(feat))) + start = ( + segment_csr( + coord, + torch.cat([batch.new_zeros(1), torch.cumsum(batch.bincount(), dim=0)]), + reduce="min", + ) + if start is None + else start + ) + cluster = voxel_grid( + pos=coord - start[batch], size=self.grid_size, batch=batch, start=0 + ) + unique, cluster, counts = torch.unique( + cluster, sorted=True, return_inverse=True, return_counts=True + ) + _, sorted_cluster_indices = torch.sort(cluster) + idx_ptr = torch.cat([counts.new_zeros(1), torch.cumsum(counts, dim=0)]) + coord = segment_csr(coord[sorted_cluster_indices], idx_ptr, reduce="mean") + feat = segment_csr(feat[sorted_cluster_indices], idx_ptr, reduce="max") + batch = batch[idx_ptr[:-1]] + offset = batch2offset(batch) + return [coord, feat, offset], cluster + + +class UnpoolWithSkip(nn.Module): + """ + Map Unpooling with skip connection + """ + + def __init__( + self, + in_channels, + skip_channels, + out_channels, + bias=True, + skip=True, + backend="map", + ): + super(UnpoolWithSkip, self).__init__() + self.in_channels = in_channels + self.skip_channels = skip_channels + self.out_channels = out_channels + self.skip = skip + self.backend = backend + assert self.backend in ["map", "interp"] + + self.proj = nn.Sequential( + nn.Linear(in_channels, out_channels, bias=bias), + PointBatchNorm(out_channels), + nn.ReLU(inplace=True), + ) + self.proj_skip = nn.Sequential( + nn.Linear(skip_channels, out_channels, bias=bias), + PointBatchNorm(out_channels), + nn.ReLU(inplace=True), + ) + + def forward(self, points, skip_points, cluster=None): + coord, feat, offset = points + skip_coord, skip_feat, skip_offset = skip_points + if self.backend == "map" and cluster is not None: + feat = self.proj(feat)[cluster] + else: + feat = pointops.interpolation( + coord, skip_coord, self.proj(feat), offset, skip_offset + ) + if self.skip: + feat = feat + self.proj_skip(skip_feat) + return [skip_coord, feat, skip_offset] + + +class Encoder(nn.Module): + def __init__( + self, + depth, + in_channels, + embed_channels, + groups, + grid_size=None, + neighbours=16, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=None, + drop_path_rate=None, + enable_checkpoint=False, + ): + super(Encoder, self).__init__() + + self.down = GridPool( + in_channels=in_channels, + out_channels=embed_channels, + grid_size=grid_size, + ) + + self.blocks = BlockSequence( + depth=depth, + embed_channels=embed_channels, + groups=groups, + neighbours=neighbours, + qkv_bias=qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate if attn_drop_rate is not None else 0.0, + drop_path_rate=drop_path_rate if drop_path_rate is not None else 0.0, + enable_checkpoint=enable_checkpoint, + ) + + def forward(self, points): + points, cluster = self.down(points) + return self.blocks(points), cluster + + +class Decoder(nn.Module): + def __init__( + self, + in_channels, + skip_channels, + embed_channels, + groups, + depth, + neighbours=16, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=None, + drop_path_rate=None, + enable_checkpoint=False, + unpool_backend="map", + ): + super(Decoder, self).__init__() + + self.up = UnpoolWithSkip( + in_channels=in_channels, + out_channels=embed_channels, + skip_channels=skip_channels, + backend=unpool_backend, + ) + + self.blocks = BlockSequence( + depth=depth, + embed_channels=embed_channels, + groups=groups, + neighbours=neighbours, + qkv_bias=qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate if attn_drop_rate is not None else 0.0, + drop_path_rate=drop_path_rate if drop_path_rate is not None else 0.0, + enable_checkpoint=enable_checkpoint, + ) + + def forward(self, points, skip_points, cluster): + points = self.up(points, skip_points, cluster) + return self.blocks(points) + + +class GVAPatchEmbed(nn.Module): + def __init__( + self, + depth, + in_channels, + embed_channels, + groups, + neighbours=16, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.0, + enable_checkpoint=False, + ): + super(GVAPatchEmbed, self).__init__() + self.in_channels = in_channels + self.embed_channels = embed_channels + self.proj = nn.Sequential( + nn.Linear(in_channels, embed_channels, bias=False), + PointBatchNorm(embed_channels), + nn.ReLU(inplace=True), + ) + self.blocks = BlockSequence( + depth=depth, + embed_channels=embed_channels, + groups=groups, + neighbours=neighbours, + qkv_bias=qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rate, + enable_checkpoint=enable_checkpoint, + ) + + def forward(self, points): + coord, feat, offset = points + feat = self.proj(feat) + return self.blocks([coord, feat, offset]) + + +@MODELS.register_module("PT-v2m2") +class PointTransformerV2(nn.Module): + def __init__( + self, + in_channels, + num_classes, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.12, 0.24, 0.48), + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0, + enable_checkpoint=False, + unpool_backend="map", + ): + super(PointTransformerV2, self).__init__() + self.in_channels = in_channels + self.num_classes = num_classes + self.num_stages = len(enc_depths) + assert self.num_stages == len(dec_depths) + assert self.num_stages == len(enc_channels) + assert self.num_stages == len(dec_channels) + assert self.num_stages == len(enc_groups) + assert self.num_stages == len(dec_groups) + assert self.num_stages == len(enc_neighbours) + assert self.num_stages == len(dec_neighbours) + assert self.num_stages == len(grid_sizes) + self.patch_embed = GVAPatchEmbed( + in_channels=in_channels, + embed_channels=patch_embed_channels, + groups=patch_embed_groups, + depth=patch_embed_depth, + neighbours=patch_embed_neighbours, + qkv_bias=attn_qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + enable_checkpoint=enable_checkpoint, + ) + + enc_dp_rates = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(enc_depths)) + ] + dec_dp_rates = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(dec_depths)) + ] + enc_channels = [patch_embed_channels] + list(enc_channels) + dec_channels = list(dec_channels) + [enc_channels[-1]] + self.enc_stages = nn.ModuleList() + self.dec_stages = nn.ModuleList() + for i in range(self.num_stages): + enc = Encoder( + depth=enc_depths[i], + in_channels=enc_channels[i], + embed_channels=enc_channels[i + 1], + groups=enc_groups[i], + grid_size=grid_sizes[i], + neighbours=enc_neighbours[i], + qkv_bias=attn_qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + drop_path_rate=enc_dp_rates[ + sum(enc_depths[:i]) : sum(enc_depths[: i + 1]) + ], + enable_checkpoint=enable_checkpoint, + ) + dec = Decoder( + depth=dec_depths[i], + in_channels=dec_channels[i + 1], + skip_channels=enc_channels[i], + embed_channels=dec_channels[i], + groups=dec_groups[i], + neighbours=dec_neighbours[i], + qkv_bias=attn_qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + drop_path_rate=dec_dp_rates[ + sum(dec_depths[:i]) : sum(dec_depths[: i + 1]) + ], + enable_checkpoint=enable_checkpoint, + unpool_backend=unpool_backend, + ) + self.enc_stages.append(enc) + self.dec_stages.append(dec) + self.seg_head = ( + nn.Sequential( + nn.Linear(dec_channels[0], dec_channels[0]), + PointBatchNorm(dec_channels[0]), + nn.ReLU(inplace=True), + nn.Linear(dec_channels[0], num_classes), + ) + if num_classes > 0 + else nn.Identity() + ) + + def forward(self, data_dict): + coord = data_dict["coord"] + feat = data_dict["feat"] + offset = data_dict["offset"].int() + + # a batch of point cloud is a list of coord, feat and offset + points = [coord, feat, offset] + points = self.patch_embed(points) + skips = [[points]] + for i in range(self.num_stages): + points, cluster = self.enc_stages[i](points) + skips[-1].append(cluster) # record grid cluster of pooling + skips.append([points]) # record points info of current stage + + points = skips.pop(-1)[0] # unpooling points info in the last enc stage + for i in reversed(range(self.num_stages)): + skip_points, cluster = skips.pop(-1) + points = self.dec_stages[i](points, skip_points, cluster) + coord, feat, offset = points + seg_logits = self.seg_head(feat) + return seg_logits diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v2/point_transformer_v2m3_pdnorm.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v2/point_transformer_v2m3_pdnorm.py new file mode 100644 index 0000000000000000000000000000000000000000..b944f19f2b13a73ae01bad4c094c51e4213896c4 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v2/point_transformer_v2m3_pdnorm.py @@ -0,0 +1,659 @@ +""" +Point Transformer V2M3 + +Enable Prompt-Driven Normalization for Point Prompt Training + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from functools import partial +from copy import deepcopy +import math +import torch +import torch.nn as nn +from torch.utils.checkpoint import checkpoint +from torch_geometric.nn.pool import voxel_grid +from torch_scatter import segment_csr + +import einops +from timm.models.layers import DropPath +import pointops + +from pointcept.models.builder import MODELS +from pointcept.models.utils import offset2batch, batch2offset + + +class PDBatchNorm(torch.nn.Module): + def __init__( + self, + num_features, + context_channels=256, + eps=1e-3, + momentum=0.01, + conditions=("ScanNet", "S3DIS", "Structured3D"), + decouple=True, + adaptive=False, + affine=True, + ): + super().__init__() + self.conditions = conditions + self.decouple = decouple + self.adaptive = adaptive + self.affine = affine + if self.decouple: + self.bns = nn.ModuleList( + [ + nn.BatchNorm1d( + num_features=num_features, + eps=eps, + momentum=momentum, + affine=affine, + ) + for _ in conditions + ] + ) + else: + self.bn = nn.BatchNorm1d( + num_features=num_features, eps=eps, momentum=momentum, affine=affine + ) + if self.adaptive: + self.modulation = nn.Sequential( + nn.SiLU(), nn.Linear(context_channels, 2 * num_features, bias=True) + ) + + def forward(self, feat, condition=None, context=None): + if self.decouple: + assert condition in self.conditions + bn = self.bns[self.conditions.index(condition)] + else: + bn = self.bn + feat = bn(feat) + if self.adaptive: + assert context is not None + shift, scale = self.modulation(context).chunk(2, dim=1) + feat = feat * (1.0 + scale) + shift + return feat + + +class PointBatchNorm(nn.Module): + """ + Batch Normalization for Point Clouds data in shape of [B*N, C], [B*N, L, C] + """ + + def __init__(self, embed_channels): + super().__init__() + self.norm = nn.BatchNorm1d(embed_channels) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + if input.dim() == 3: + return ( + self.norm(input.transpose(1, 2).contiguous()) + .transpose(1, 2) + .contiguous() + ) + elif input.dim() == 2: + return self.norm(input) + else: + raise NotImplementedError + + +class GroupedVectorAttention(nn.Module): + def __init__( + self, + embed_channels, + groups, + attn_drop_rate=0.0, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + ): + super(GroupedVectorAttention, self).__init__() + self.embed_channels = embed_channels + self.groups = groups + assert embed_channels % groups == 0 + self.attn_drop_rate = attn_drop_rate + self.qkv_bias = qkv_bias + self.pe_multiplier = pe_multiplier + self.pe_bias = pe_bias + + self.linear_q = nn.Sequential( + nn.Linear(embed_channels, embed_channels, bias=qkv_bias), + PointBatchNorm(embed_channels), + nn.ReLU(inplace=True), + ) + self.linear_k = nn.Sequential( + nn.Linear(embed_channels, embed_channels, bias=qkv_bias), + PointBatchNorm(embed_channels), + nn.ReLU(inplace=True), + ) + + self.linear_v = nn.Linear(embed_channels, embed_channels, bias=qkv_bias) + + if self.pe_multiplier: + self.linear_p_multiplier = nn.Sequential( + nn.Linear(3, embed_channels), + PointBatchNorm(embed_channels), + nn.ReLU(inplace=True), + nn.Linear(embed_channels, embed_channels), + ) + if self.pe_bias: + self.linear_p_bias = nn.Sequential( + nn.Linear(3, embed_channels), + PointBatchNorm(embed_channels), + nn.ReLU(inplace=True), + nn.Linear(embed_channels, embed_channels), + ) + self.weight_encoding = nn.Sequential( + nn.Linear(embed_channels, groups), + PointBatchNorm(groups), + nn.ReLU(inplace=True), + nn.Linear(groups, groups), + ) + self.softmax = nn.Softmax(dim=1) + self.attn_drop = nn.Dropout(attn_drop_rate) + + def forward(self, feat, coord, reference_index): + query, key, value = ( + self.linear_q(feat), + self.linear_k(feat), + self.linear_v(feat), + ) + key = pointops.grouping(reference_index, key, coord, with_xyz=True) + value = pointops.grouping(reference_index, value, coord, with_xyz=False) + pos, key = key[:, :, 0:3], key[:, :, 3:] + relation_qk = key - query.unsqueeze(1) + if self.pe_multiplier: + pem = self.linear_p_multiplier(pos) + relation_qk = relation_qk * pem + if self.pe_bias: + peb = self.linear_p_bias(pos) + relation_qk = relation_qk + peb + value = value + peb + + weight = self.weight_encoding(relation_qk) + weight = self.attn_drop(self.softmax(weight)) + + mask = torch.sign(reference_index + 1) + weight = torch.einsum("n s g, n s -> n s g", weight, mask) + value = einops.rearrange(value, "n ns (g i) -> n ns g i", g=self.groups) + feat = torch.einsum("n s g i, n s g -> n g i", value, weight) + feat = einops.rearrange(feat, "n g i -> n (g i)") + return feat + + +class Block(nn.Module): + def __init__( + self, + embed_channels, + groups, + norm_fn=None, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.0, + enable_checkpoint=False, + ): + super(Block, self).__init__() + self.attn = GroupedVectorAttention( + embed_channels=embed_channels, + groups=groups, + qkv_bias=qkv_bias, + attn_drop_rate=attn_drop_rate, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + ) + + assert norm_fn is not None + + self.fc1 = nn.Linear(embed_channels, embed_channels, bias=False) + self.fc3 = nn.Linear(embed_channels, embed_channels, bias=False) + self.norm1 = norm_fn(embed_channels) + self.norm2 = norm_fn(embed_channels) + self.norm3 = norm_fn(embed_channels) + self.act = nn.ReLU(inplace=True) + self.enable_checkpoint = enable_checkpoint + self.drop_path = ( + DropPath(drop_path_rate) if drop_path_rate > 0.0 else nn.Identity() + ) + + def forward(self, points, reference_index): + coord, feat, offset, condition, context = points + identity = feat + feat = self.act(self.norm1(self.fc1(feat), condition, context)) + feat = ( + self.attn(feat, coord, reference_index) + if not self.enable_checkpoint + else checkpoint(self.attn, feat, coord, reference_index) + ) + feat = self.act(self.norm2(feat, condition, context)) + feat = self.norm3(self.fc3(feat), condition, context) + feat = identity + self.drop_path(feat) + feat = self.act(feat) + return [coord, feat, offset, condition, context] + + +class BlockSequence(nn.Module): + def __init__( + self, + depth, + embed_channels, + groups, + neighbours=16, + norm_fn=None, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.0, + enable_checkpoint=False, + ): + super(BlockSequence, self).__init__() + + if isinstance(drop_path_rate, list): + drop_path_rates = drop_path_rate + assert len(drop_path_rates) == depth + elif isinstance(drop_path_rate, float): + drop_path_rates = [deepcopy(drop_path_rate) for _ in range(depth)] + else: + drop_path_rates = [0.0 for _ in range(depth)] + + self.neighbours = neighbours + self.blocks = nn.ModuleList() + for i in range(depth): + block = Block( + embed_channels=embed_channels, + groups=groups, + norm_fn=norm_fn, + qkv_bias=qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rates[i], + enable_checkpoint=enable_checkpoint, + ) + self.blocks.append(block) + + def forward(self, points): + coord, feat, offset, condition, context = points + # reference index query of neighbourhood attention + # for windows attention, modify reference index query method + reference_index, _ = pointops.knn_query(self.neighbours, coord, offset) + for block in self.blocks: + points = block(points, reference_index) + return points + + +class GridPool(nn.Module): + """ + Partition-based Pooling (Grid Pooling) + """ + + def __init__(self, in_channels, out_channels, grid_size, norm_fn, bias=False): + super(GridPool, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.grid_size = grid_size + + self.fc = nn.Linear(in_channels, out_channels, bias=bias) + self.norm = norm_fn(out_channels) + self.act = nn.ReLU(inplace=True) + + def forward(self, points, start=None): + coord, feat, offset, condition, context = points + batch = offset2batch(offset) + feat = self.act(self.norm(self.fc(feat), condition, context)) + start = ( + segment_csr( + coord, + torch.cat([batch.new_zeros(1), torch.cumsum(batch.bincount(), dim=0)]), + reduce="min", + ) + if start is None + else start + ) + cluster = voxel_grid( + pos=coord - start[batch], size=self.grid_size, batch=batch, start=0 + ) + unique, cluster, counts = torch.unique( + cluster, sorted=True, return_inverse=True, return_counts=True + ) + _, sorted_cluster_indices = torch.sort(cluster) + idx_ptr = torch.cat([counts.new_zeros(1), torch.cumsum(counts, dim=0)]) + coord = segment_csr(coord[sorted_cluster_indices], idx_ptr, reduce="mean") + feat = segment_csr(feat[sorted_cluster_indices], idx_ptr, reduce="max") + batch = batch[idx_ptr[:-1]] + offset = batch2offset(batch) + return [coord, feat, offset, condition, context], cluster + + +class UnpoolWithSkip(nn.Module): + """ + Map Unpooling with skip connection + """ + + def __init__( + self, + in_channels, + skip_channels, + out_channels, + norm_fn, + bias=True, + skip=True, + backend="map", + ): + super(UnpoolWithSkip, self).__init__() + self.in_channels = in_channels + self.skip_channels = skip_channels + self.out_channels = out_channels + self.skip = skip + self.backend = backend + assert self.backend in ["map", "interp"] + + self.proj_linear = nn.Linear(in_channels, out_channels, bias=bias) + self.proj_norm = norm_fn(out_channels) + self.proj_act = nn.ReLU(inplace=True) + + self.proj_skip_linear = nn.Linear(skip_channels, out_channels, bias=bias) + self.proj_skip_norm = norm_fn(out_channels) + self.proj_skip_act = nn.ReLU(inplace=True) + + def forward(self, points, skip_points, cluster=None): + coord, feat, offset, condition, context = points + skip_coord, skip_feat, skip_offset, _, _ = skip_points + feat = self.proj_act(self.proj_norm(self.proj_linear(feat), condition, context)) + if self.backend == "map" and cluster is not None: + feat = feat[cluster] + else: + feat = pointops.interpolation(coord, skip_coord, feat, offset, skip_offset) + if self.skip: + feat = feat + self.proj_skip_act( + self.proj_skip_norm( + self.proj_skip_linear(skip_feat), condition, context + ) + ) + return [skip_coord, feat, skip_offset, condition, context] + + +class Encoder(nn.Module): + def __init__( + self, + depth, + in_channels, + embed_channels, + groups, + norm_fn, + grid_size=None, + neighbours=16, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=None, + drop_path_rate=None, + enable_checkpoint=False, + ): + super(Encoder, self).__init__() + + self.down = GridPool( + in_channels=in_channels, + out_channels=embed_channels, + grid_size=grid_size, + norm_fn=norm_fn, + ) + + self.blocks = BlockSequence( + depth=depth, + embed_channels=embed_channels, + groups=groups, + neighbours=neighbours, + norm_fn=norm_fn, + qkv_bias=qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate if attn_drop_rate is not None else 0.0, + drop_path_rate=drop_path_rate if drop_path_rate is not None else 0.0, + enable_checkpoint=enable_checkpoint, + ) + + def forward(self, points): + points, cluster = self.down(points) + return self.blocks(points), cluster + + +class Decoder(nn.Module): + def __init__( + self, + in_channels, + skip_channels, + embed_channels, + groups, + depth, + norm_fn, + neighbours=16, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=None, + drop_path_rate=None, + enable_checkpoint=False, + unpool_backend="map", + ): + super(Decoder, self).__init__() + + self.up = UnpoolWithSkip( + in_channels=in_channels, + out_channels=embed_channels, + skip_channels=skip_channels, + backend=unpool_backend, + norm_fn=norm_fn, + ) + + self.blocks = BlockSequence( + depth=depth, + embed_channels=embed_channels, + groups=groups, + neighbours=neighbours, + norm_fn=norm_fn, + qkv_bias=qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate if attn_drop_rate is not None else 0.0, + drop_path_rate=drop_path_rate if drop_path_rate is not None else 0.0, + enable_checkpoint=enable_checkpoint, + ) + + def forward(self, points, skip_points, cluster): + points = self.up(points, skip_points, cluster) + return self.blocks(points) + + +class GVAPatchEmbed(nn.Module): + def __init__( + self, + depth, + in_channels, + embed_channels, + groups, + norm_fn, + neighbours=16, + qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0.0, + enable_checkpoint=False, + ): + super(GVAPatchEmbed, self).__init__() + self.in_channels = in_channels + self.embed_channels = embed_channels + self.proj_linear = nn.Linear(in_channels, embed_channels, bias=False) + self.proj_norm = norm_fn(embed_channels) + self.proj_act = nn.ReLU(inplace=True) + self.blocks = BlockSequence( + depth=depth, + embed_channels=embed_channels, + groups=groups, + neighbours=neighbours, + norm_fn=norm_fn, + qkv_bias=qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rate, + enable_checkpoint=enable_checkpoint, + ) + + def forward(self, points): + coord, feat, offset, condition, context = points + feat = self.proj_act(self.proj_norm(self.proj_linear(feat), condition, context)) + return self.blocks([coord, feat, offset, condition, context]) + + +@MODELS.register_module("PT-v2m3") +class PointTransformerV2(nn.Module): + def __init__( + self, + in_channels, + num_classes, + patch_embed_depth=1, + patch_embed_channels=48, + patch_embed_groups=6, + patch_embed_neighbours=8, + enc_depths=(2, 2, 6, 2), + enc_channels=(96, 192, 384, 512), + enc_groups=(12, 24, 48, 64), + enc_neighbours=(16, 16, 16, 16), + dec_depths=(1, 1, 1, 1), + dec_channels=(48, 96, 192, 384), + dec_groups=(6, 12, 24, 48), + dec_neighbours=(16, 16, 16, 16), + grid_sizes=(0.06, 0.12, 0.24, 0.48), + attn_qkv_bias=True, + pe_multiplier=False, + pe_bias=True, + attn_drop_rate=0.0, + drop_path_rate=0, + enable_checkpoint=False, + unpool_backend="map", + context_channels=256, + conditions=("ScanNet", "S3DIS", "Structured3D"), + norm_decouple=True, + norm_adaptive=True, + norm_affine=False, + ): + super(PointTransformerV2, self).__init__() + self.in_channels = in_channels + self.num_classes = num_classes + self.num_stages = len(enc_depths) + assert self.num_stages == len(dec_depths) + assert self.num_stages == len(enc_channels) + assert self.num_stages == len(dec_channels) + assert self.num_stages == len(enc_groups) + assert self.num_stages == len(dec_groups) + assert self.num_stages == len(enc_neighbours) + assert self.num_stages == len(dec_neighbours) + assert self.num_stages == len(grid_sizes) + + norm_fn = partial( + PDBatchNorm, + eps=1e-3, + momentum=0.01, + conditions=conditions, + context_channels=context_channels, + decouple=norm_decouple, + adaptive=norm_adaptive, + affine=norm_affine, + ) + + self.patch_embed = GVAPatchEmbed( + in_channels=in_channels, + embed_channels=patch_embed_channels, + groups=patch_embed_groups, + depth=patch_embed_depth, + neighbours=patch_embed_neighbours, + norm_fn=norm_fn, + qkv_bias=attn_qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + enable_checkpoint=enable_checkpoint, + ) + + enc_dp_rates = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(enc_depths)) + ] + dec_dp_rates = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(dec_depths)) + ] + enc_channels = [patch_embed_channels] + list(enc_channels) + dec_channels = list(dec_channels) + [enc_channels[-1]] + self.enc_stages = nn.ModuleList() + self.dec_stages = nn.ModuleList() + for i in range(self.num_stages): + enc = Encoder( + depth=enc_depths[i], + in_channels=enc_channels[i], + embed_channels=enc_channels[i + 1], + groups=enc_groups[i], + grid_size=grid_sizes[i], + neighbours=enc_neighbours[i], + norm_fn=norm_fn, + qkv_bias=attn_qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + drop_path_rate=enc_dp_rates[ + sum(enc_depths[:i]) : sum(enc_depths[: i + 1]) + ], + enable_checkpoint=enable_checkpoint, + ) + dec = Decoder( + depth=dec_depths[i], + in_channels=dec_channels[i + 1], + skip_channels=enc_channels[i], + embed_channels=dec_channels[i], + groups=dec_groups[i], + neighbours=dec_neighbours[i], + norm_fn=norm_fn, + qkv_bias=attn_qkv_bias, + pe_multiplier=pe_multiplier, + pe_bias=pe_bias, + attn_drop_rate=attn_drop_rate, + drop_path_rate=dec_dp_rates[ + sum(dec_depths[:i]) : sum(dec_depths[: i + 1]) + ], + enable_checkpoint=enable_checkpoint, + unpool_backend=unpool_backend, + ) + self.enc_stages.append(enc) + self.dec_stages.append(dec) + self.seg_head = ( + nn.Sequential(nn.Linear(dec_channels[0], num_classes)) + if num_classes > 0 + else nn.Identity() + ) + + def forward(self, data_dict): + coord = data_dict["coord"] + feat = data_dict["feat"] + offset = data_dict["offset"].int() + condition = data_dict["condition"][0] + context = data_dict["context"] if "context" in data_dict.keys() else None + + # a batch of point cloud is a list of coord, feat and offset + points = [coord, feat, offset, condition, context] + points = self.patch_embed(points) + skips = [[points]] + for i in range(self.num_stages): + points, cluster = self.enc_stages[i](points) + skips[-1].append(cluster) # record grid cluster of pooling + skips.append([points]) # record points info of current stage + + points = skips.pop(-1)[0] # unpooling points info in the last enc stage + for i in reversed(range(self.num_stages)): + skip_points, cluster = skips.pop(-1) + points = self.dec_stages[i](points, skip_points, cluster) + coord, feat, offset, _, _ = points + seg_logits = self.seg_head(feat) + return seg_logits diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v3/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v3/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5fe25f32abaf4d60241dcf21507f85a47e46f070 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v3/__init__.py @@ -0,0 +1 @@ +from .point_transformer_v3m1_base import * diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v3/point_transformer_v3m1_base.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v3/point_transformer_v3m1_base.py new file mode 100644 index 0000000000000000000000000000000000000000..f9f567162dc424324ee5c30a0803fe3ea465f9b1 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/point_transformer_v3/point_transformer_v3m1_base.py @@ -0,0 +1,714 @@ +""" +Point Transformer - V3 Mode1 + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from functools import partial +from addict import Dict +import math +import torch +import torch.nn as nn +import spconv.pytorch as spconv +import torch_scatter +from timm.models.layers import DropPath + +try: + import flash_attn +except ImportError: + flash_attn = None + +from pointcept.models.point_prompt_training import PDNorm +from pointcept.models.builder import MODELS +from pointcept.models.utils.misc import offset2bincount +from pointcept.models.utils.structure import Point +from pointcept.models.modules import PointModule, PointSequential + + +class RPE(torch.nn.Module): + def __init__(self, patch_size, num_heads): + super().__init__() + self.patch_size = patch_size + self.num_heads = num_heads + self.pos_bnd = int((4 * patch_size) ** (1 / 3) * 2) + self.rpe_num = 2 * self.pos_bnd + 1 + self.rpe_table = torch.nn.Parameter(torch.zeros(3 * self.rpe_num, num_heads)) + torch.nn.init.trunc_normal_(self.rpe_table, std=0.02) + + def forward(self, coord): + idx = ( + coord.clamp(-self.pos_bnd, self.pos_bnd) # clamp into bnd + + self.pos_bnd # relative position to positive index + + torch.arange(3, device=coord.device) * self.rpe_num # x, y, z stride + ) + out = self.rpe_table.index_select(0, idx.reshape(-1)) + out = out.view(idx.shape + (-1,)).sum(3) + out = out.permute(0, 3, 1, 2) # (N, K, K, H) -> (N, H, K, K) + return out + + +class SerializedAttention(PointModule): + def __init__( + self, + channels, + num_heads, + patch_size, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + order_index=0, + enable_rpe=False, + enable_flash=True, + upcast_attention=True, + upcast_softmax=True, + ): + super().__init__() + assert channels % num_heads == 0 + self.channels = channels + self.num_heads = num_heads + self.scale = qk_scale or (channels // num_heads) ** -0.5 + self.order_index = order_index + self.upcast_attention = upcast_attention + self.upcast_softmax = upcast_softmax + self.enable_rpe = enable_rpe + self.enable_flash = enable_flash + if enable_flash: + assert ( + enable_rpe is False + ), "Set enable_rpe to False when enable Flash Attention" + assert ( + upcast_attention is False + ), "Set upcast_attention to False when enable Flash Attention" + assert ( + upcast_softmax is False + ), "Set upcast_softmax to False when enable Flash Attention" + assert flash_attn is not None, "Make sure flash_attn is installed." + self.patch_size = patch_size + self.attn_drop = attn_drop + else: + # when disable flash attention, we still don't want to use mask + # consequently, patch size will auto set to the + # min number of patch_size_max and number of points + self.patch_size_max = patch_size + self.patch_size = 0 + self.attn_drop = torch.nn.Dropout(attn_drop) + + self.qkv = torch.nn.Linear(channels, channels * 3, bias=qkv_bias) + self.proj = torch.nn.Linear(channels, channels) + self.proj_drop = torch.nn.Dropout(proj_drop) + self.softmax = torch.nn.Softmax(dim=-1) + self.rpe = RPE(patch_size, num_heads) if self.enable_rpe else None + + @torch.no_grad() + def get_rel_pos(self, point, order): + K = self.patch_size + rel_pos_key = f"rel_pos_{self.order_index}" + if rel_pos_key not in point.keys(): + grid_coord = point.grid_coord[order] + grid_coord = grid_coord.reshape(-1, K, 3) + point[rel_pos_key] = grid_coord.unsqueeze(2) - grid_coord.unsqueeze(1) + return point[rel_pos_key] + + @torch.no_grad() + def get_padding_and_inverse(self, point): + pad_key = "pad" + unpad_key = "unpad" + cu_seqlens_key = "cu_seqlens_key" + if ( + pad_key not in point.keys() + or unpad_key not in point.keys() + or cu_seqlens_key not in point.keys() + ): + offset = point.offset + bincount = offset2bincount(offset) + bincount_pad = ( + torch.div( + bincount + self.patch_size - 1, + self.patch_size, + rounding_mode="trunc", + ) + * self.patch_size + ) + # only pad point when num of points larger than patch_size + mask_pad = bincount > self.patch_size + bincount_pad = ~mask_pad * bincount + mask_pad * bincount_pad + _offset = nn.functional.pad(offset, (1, 0)) + _offset_pad = nn.functional.pad(torch.cumsum(bincount_pad, dim=0), (1, 0)) + pad = torch.arange(_offset_pad[-1], device=offset.device) + unpad = torch.arange(_offset[-1], device=offset.device) + cu_seqlens = [] + for i in range(len(offset)): + unpad[_offset[i] : _offset[i + 1]] += _offset_pad[i] - _offset[i] + if bincount[i] != bincount_pad[i]: + pad[ + _offset_pad[i + 1] + - self.patch_size + + (bincount[i] % self.patch_size) : _offset_pad[i + 1] + ] = pad[ + _offset_pad[i + 1] + - 2 * self.patch_size + + (bincount[i] % self.patch_size) : _offset_pad[i + 1] + - self.patch_size + ] + pad[_offset_pad[i] : _offset_pad[i + 1]] -= _offset_pad[i] - _offset[i] + cu_seqlens.append( + torch.arange( + _offset_pad[i], + _offset_pad[i + 1], + step=self.patch_size, + dtype=torch.int32, + device=offset.device, + ) + ) + point[pad_key] = pad + point[unpad_key] = unpad + point[cu_seqlens_key] = nn.functional.pad( + torch.concat(cu_seqlens), (0, 1), value=_offset_pad[-1] + ) + return point[pad_key], point[unpad_key], point[cu_seqlens_key] + + def forward(self, point): + if not self.enable_flash: + self.patch_size = min( + offset2bincount(point.offset).min().tolist(), self.patch_size_max + ) + + H = self.num_heads + K = self.patch_size + C = self.channels + + pad, unpad, cu_seqlens = self.get_padding_and_inverse(point) + + order = point.serialized_order[self.order_index][pad] + inverse = unpad[point.serialized_inverse[self.order_index]] + + # padding and reshape feat and batch for serialized point patch + qkv = self.qkv(point.feat)[order] + + if not self.enable_flash: + # encode and reshape qkv: (N', K, 3, H, C') => (3, N', H, K, C') + q, k, v = ( + qkv.reshape(-1, K, 3, H, C // H).permute(2, 0, 3, 1, 4).unbind(dim=0) + ) + # attn + if self.upcast_attention: + q = q.float() + k = k.float() + attn = (q * self.scale) @ k.transpose(-2, -1) # (N', H, K, K) + if self.enable_rpe: + attn = attn + self.rpe(self.get_rel_pos(point, order)) + if self.upcast_softmax: + attn = attn.float() + attn = self.softmax(attn) + attn = self.attn_drop(attn).to(qkv.dtype) + feat = (attn @ v).transpose(1, 2).reshape(-1, C) + else: + feat = flash_attn.flash_attn_varlen_qkvpacked_func( + qkv.half().reshape(-1, 3, H, C // H), + cu_seqlens, + max_seqlen=self.patch_size, + dropout_p=self.attn_drop if self.training else 0, + softmax_scale=self.scale, + ).reshape(-1, C) + feat = feat.to(qkv.dtype) + feat = feat[inverse] + + # ffn + feat = self.proj(feat) + feat = self.proj_drop(feat) + point.feat = feat + return point + + +class MLP(nn.Module): + def __init__( + self, + in_channels, + hidden_channels=None, + out_channels=None, + act_layer=nn.GELU, + drop=0.0, + ): + super().__init__() + out_channels = out_channels or in_channels + hidden_channels = hidden_channels or in_channels + self.fc1 = nn.Linear(in_channels, hidden_channels) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_channels, out_channels) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class Block(PointModule): + def __init__( + self, + channels, + num_heads, + patch_size=48, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.0, + norm_layer=nn.LayerNorm, + act_layer=nn.GELU, + pre_norm=True, + order_index=0, + cpe_indice_key=None, + enable_rpe=False, + enable_flash=True, + upcast_attention=True, + upcast_softmax=True, + ): + super().__init__() + self.channels = channels + self.pre_norm = pre_norm + + self.cpe = PointSequential( + spconv.SubMConv3d( + channels, + channels, + kernel_size=3, + bias=True, + indice_key=cpe_indice_key, + ), + nn.Linear(channels, channels), + norm_layer(channels), + ) + + self.norm1 = PointSequential(norm_layer(channels)) + self.attn = SerializedAttention( + channels=channels, + patch_size=patch_size, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=proj_drop, + order_index=order_index, + enable_rpe=enable_rpe, + enable_flash=enable_flash, + upcast_attention=upcast_attention, + upcast_softmax=upcast_softmax, + ) + self.norm2 = PointSequential(norm_layer(channels)) + self.mlp = PointSequential( + MLP( + in_channels=channels, + hidden_channels=int(channels * mlp_ratio), + out_channels=channels, + act_layer=act_layer, + drop=proj_drop, + ) + ) + self.drop_path = PointSequential( + DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + ) + + def forward(self, point: Point): + shortcut = point.feat + point = self.cpe(point) + point.feat = shortcut + point.feat + shortcut = point.feat + if self.pre_norm: + point = self.norm1(point) + point = self.drop_path(self.attn(point)) + point.feat = shortcut + point.feat + if not self.pre_norm: + point = self.norm1(point) + + shortcut = point.feat + if self.pre_norm: + point = self.norm2(point) + point = self.drop_path(self.mlp(point)) + point.feat = shortcut + point.feat + if not self.pre_norm: + point = self.norm2(point) + point.sparse_conv_feat = point.sparse_conv_feat.replace_feature(point.feat) + return point + + +class SerializedPooling(PointModule): + def __init__( + self, + in_channels, + out_channels, + stride=2, + norm_layer=None, + act_layer=None, + reduce="max", + shuffle_orders=True, + traceable=True, # record parent and cluster + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + + assert stride == 2 ** (math.ceil(stride) - 1).bit_length() # 2, 4, 8 + # TODO: add support to grid pool (any stride) + self.stride = stride + assert reduce in ["sum", "mean", "min", "max"] + self.reduce = reduce + self.shuffle_orders = shuffle_orders + self.traceable = traceable + + self.proj = nn.Linear(in_channels, out_channels) + if norm_layer is not None: + self.norm = PointSequential(norm_layer(out_channels)) + if act_layer is not None: + self.act = PointSequential(act_layer()) + + def forward(self, point: Point): + pooling_depth = (math.ceil(self.stride) - 1).bit_length() + if pooling_depth > point.serialized_depth: + pooling_depth = 0 + assert { + "serialized_code", + "serialized_order", + "serialized_inverse", + "serialized_depth", + }.issubset( + point.keys() + ), "Run point.serialization() point cloud before SerializedPooling" + + code = point.serialized_code >> pooling_depth * 3 + code_, cluster, counts = torch.unique( + code[0], + sorted=True, + return_inverse=True, + return_counts=True, + ) + # indices of point sorted by cluster, for torch_scatter.segment_csr + _, indices = torch.sort(cluster) + # index pointer for sorted point, for torch_scatter.segment_csr + idx_ptr = torch.cat([counts.new_zeros(1), torch.cumsum(counts, dim=0)]) + # head_indices of each cluster, for reduce attr e.g. code, batch + head_indices = indices[idx_ptr[:-1]] + # generate down code, order, inverse + code = code[:, head_indices] + order = torch.argsort(code) + inverse = torch.zeros_like(order).scatter_( + dim=1, + index=order, + src=torch.arange(0, code.shape[1], device=order.device).repeat( + code.shape[0], 1 + ), + ) + + if self.shuffle_orders: + perm = torch.randperm(code.shape[0]) + code = code[perm] + order = order[perm] + inverse = inverse[perm] + + # collect information + point_dict = Dict( + feat=torch_scatter.segment_csr( + self.proj(point.feat)[indices], idx_ptr, reduce=self.reduce + ), + coord=torch_scatter.segment_csr( + point.coord[indices], idx_ptr, reduce="mean" + ), + grid_coord=point.grid_coord[head_indices] >> pooling_depth, + serialized_code=code, + serialized_order=order, + serialized_inverse=inverse, + serialized_depth=point.serialized_depth - pooling_depth, + batch=point.batch[head_indices], + ) + + if "condition" in point.keys(): + point_dict["condition"] = point.condition + if "context" in point.keys(): + point_dict["context"] = point.context + + if self.traceable: + point_dict["pooling_inverse"] = cluster + point_dict["pooling_parent"] = point + point = Point(point_dict) + if self.norm is not None: + point = self.norm(point) + if self.act is not None: + point = self.act(point) + point.sparsify() + return point + + +class SerializedUnpooling(PointModule): + def __init__( + self, + in_channels, + skip_channels, + out_channels, + norm_layer=None, + act_layer=None, + traceable=False, # record parent and cluster + ): + super().__init__() + self.proj = PointSequential(nn.Linear(in_channels, out_channels)) + self.proj_skip = PointSequential(nn.Linear(skip_channels, out_channels)) + + if norm_layer is not None: + self.proj.add(norm_layer(out_channels)) + self.proj_skip.add(norm_layer(out_channels)) + + if act_layer is not None: + self.proj.add(act_layer()) + self.proj_skip.add(act_layer()) + + self.traceable = traceable + + def forward(self, point): + assert "pooling_parent" in point.keys() + assert "pooling_inverse" in point.keys() + parent = point.pop("pooling_parent") + inverse = point.pop("pooling_inverse") + point = self.proj(point) + parent = self.proj_skip(parent) + parent.feat = parent.feat + point.feat[inverse] + + if self.traceable: + parent["unpooling_parent"] = point + return parent + + +class Embedding(PointModule): + def __init__( + self, + in_channels, + embed_channels, + norm_layer=None, + act_layer=None, + ): + super().__init__() + self.in_channels = in_channels + self.embed_channels = embed_channels + + # TODO: check remove spconv + self.stem = PointSequential( + conv=spconv.SubMConv3d( + in_channels, + embed_channels, + kernel_size=5, + padding=1, + bias=False, + indice_key="stem", + ) + ) + if norm_layer is not None: + self.stem.add(norm_layer(embed_channels), name="norm") + if act_layer is not None: + self.stem.add(act_layer(), name="act") + + def forward(self, point: Point): + point = self.stem(point) + return point + + +@MODELS.register_module("PT-v3m1") +class PointTransformerV3(PointModule): + def __init__( + self, + in_channels=6, + order=("z", "z-trans"), + stride=(2, 2, 2, 2), + enc_depths=(2, 2, 2, 6, 2), + enc_channels=(32, 64, 128, 256, 512), + enc_num_head=(2, 4, 8, 16, 32), + enc_patch_size=(48, 48, 48, 48, 48), + dec_depths=(2, 2, 2, 2), + dec_channels=(64, 64, 128, 256), + dec_num_head=(4, 4, 8, 16), + dec_patch_size=(48, 48, 48, 48), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + pre_norm=True, + shuffle_orders=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=False, + pdnorm_ln=False, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "S3DIS", "Structured3D"), + ): + super().__init__() + self.num_stages = len(enc_depths) + self.order = [order] if isinstance(order, str) else order + self.cls_mode = cls_mode + self.shuffle_orders = shuffle_orders + + assert self.num_stages == len(stride) + 1 + assert self.num_stages == len(enc_depths) + assert self.num_stages == len(enc_channels) + assert self.num_stages == len(enc_num_head) + assert self.num_stages == len(enc_patch_size) + assert self.cls_mode or self.num_stages == len(dec_depths) + 1 + assert self.cls_mode or self.num_stages == len(dec_channels) + 1 + assert self.cls_mode or self.num_stages == len(dec_num_head) + 1 + assert self.cls_mode or self.num_stages == len(dec_patch_size) + 1 + + # norm layers + if pdnorm_bn: + bn_layer = partial( + PDNorm, + norm_layer=partial( + nn.BatchNorm1d, eps=1e-3, momentum=0.01, affine=pdnorm_affine + ), + conditions=pdnorm_conditions, + decouple=pdnorm_decouple, + adaptive=pdnorm_adaptive, + ) + else: + bn_layer = partial(nn.BatchNorm1d, eps=1e-3, momentum=0.01) + if pdnorm_ln: + ln_layer = partial( + PDNorm, + norm_layer=partial(nn.LayerNorm, elementwise_affine=pdnorm_affine), + conditions=pdnorm_conditions, + decouple=pdnorm_decouple, + adaptive=pdnorm_adaptive, + ) + else: + ln_layer = nn.LayerNorm + # activation layers + act_layer = nn.GELU + + self.embedding = Embedding( + in_channels=in_channels, + embed_channels=enc_channels[0], + norm_layer=bn_layer, + act_layer=act_layer, + ) + + # encoder + enc_drop_path = [ + x.item() for x in torch.linspace(0, drop_path, sum(enc_depths)) + ] + self.enc = PointSequential() + for s in range(self.num_stages): + enc_drop_path_ = enc_drop_path[ + sum(enc_depths[:s]) : sum(enc_depths[: s + 1]) + ] + enc = PointSequential() + if s > 0: + enc.add( + SerializedPooling( + in_channels=enc_channels[s - 1], + out_channels=enc_channels[s], + stride=stride[s - 1], + norm_layer=bn_layer, + act_layer=act_layer, + ), + name="down", + ) + for i in range(enc_depths[s]): + enc.add( + Block( + channels=enc_channels[s], + num_heads=enc_num_head[s], + patch_size=enc_patch_size[s], + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=proj_drop, + drop_path=enc_drop_path_[i], + norm_layer=ln_layer, + act_layer=act_layer, + pre_norm=pre_norm, + order_index=i % len(self.order), + cpe_indice_key=f"stage{s}", + enable_rpe=enable_rpe, + enable_flash=enable_flash, + upcast_attention=upcast_attention, + upcast_softmax=upcast_softmax, + ), + name=f"block{i}", + ) + if len(enc) != 0: + self.enc.add(module=enc, name=f"enc{s}") + + # decoder + if not self.cls_mode: + dec_drop_path = [ + x.item() for x in torch.linspace(0, drop_path, sum(dec_depths)) + ] + self.dec = PointSequential() + dec_channels = list(dec_channels) + [enc_channels[-1]] + for s in reversed(range(self.num_stages - 1)): + dec_drop_path_ = dec_drop_path[ + sum(dec_depths[:s]) : sum(dec_depths[: s + 1]) + ] + dec_drop_path_.reverse() + dec = PointSequential() + dec.add( + SerializedUnpooling( + in_channels=dec_channels[s + 1], + skip_channels=enc_channels[s], + out_channels=dec_channels[s], + norm_layer=bn_layer, + act_layer=act_layer, + ), + name="up", + ) + for i in range(dec_depths[s]): + dec.add( + Block( + channels=dec_channels[s], + num_heads=dec_num_head[s], + patch_size=dec_patch_size[s], + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=proj_drop, + drop_path=dec_drop_path_[i], + norm_layer=ln_layer, + act_layer=act_layer, + pre_norm=pre_norm, + order_index=i % len(self.order), + cpe_indice_key=f"stage{s}", + enable_rpe=enable_rpe, + enable_flash=enable_flash, + upcast_attention=upcast_attention, + upcast_softmax=upcast_softmax, + ), + name=f"block{i}", + ) + self.dec.add(module=dec, name=f"dec{s}") + + def forward(self, data_dict): + point = Point(data_dict) + point.serialization(order=self.order, shuffle_orders=self.shuffle_orders) + point.sparsify() + + point = self.embedding(point) + point = self.enc(point) + if not self.cls_mode: + point = self.dec(point) + # else: + # point.feat = torch_scatter.segment_csr( + # src=point.feat, + # indptr=nn.functional.pad(point.offset, (1, 0)), + # reduce="mean", + # ) + return point diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..aafc3859f43bd47da8cd966a57d4307cc771525f --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/__init__.py @@ -0,0 +1,4 @@ +from .mink_unet import * +from .spconv_unet_v1m1_base import * +from .spconv_unet_v1m2_bn_momentum import * +from .spconv_unet_v1m3_pdnorm import * diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/mink_unet.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/mink_unet.py new file mode 100644 index 0000000000000000000000000000000000000000..1ff8a01d0500abc207b82a8252a8131b5e671469 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/mink_unet.py @@ -0,0 +1,442 @@ +""" +SparseUNet Driven by MinkowskiEngine + +Modified from chrischoy/SpatioTemporalSegmentation + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import torch +import torch.nn as nn + +try: + import MinkowskiEngine as ME +except ImportError: + ME = None + +from pointcept.models.builder import MODELS + + +def offset2batch(offset): + return ( + torch.cat( + [ + ( + torch.tensor([i] * (o - offset[i - 1])) + if i > 0 + else torch.tensor([i] * o) + ) + for i, o in enumerate(offset) + ], + dim=0, + ) + .long() + .to(offset.device) + ) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__( + self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + bn_momentum=0.1, + dimension=-1, + ): + super(BasicBlock, self).__init__() + assert dimension > 0 + + self.conv1 = ME.MinkowskiConvolution( + inplanes, + planes, + kernel_size=3, + stride=stride, + dilation=dilation, + dimension=dimension, + ) + self.norm1 = ME.MinkowskiBatchNorm(planes, momentum=bn_momentum) + self.conv2 = ME.MinkowskiConvolution( + planes, + planes, + kernel_size=3, + stride=1, + dilation=dilation, + dimension=dimension, + ) + self.norm2 = ME.MinkowskiBatchNorm(planes, momentum=bn_momentum) + self.relu = ME.MinkowskiReLU(inplace=True) + self.downsample = downsample + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.norm2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__( + self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + bn_momentum=0.1, + dimension=-1, + ): + super(Bottleneck, self).__init__() + assert dimension > 0 + + self.conv1 = ME.MinkowskiConvolution( + inplanes, planes, kernel_size=1, dimension=dimension + ) + self.norm1 = ME.MinkowskiBatchNorm(planes, momentum=bn_momentum) + + self.conv2 = ME.MinkowskiConvolution( + planes, + planes, + kernel_size=3, + stride=stride, + dilation=dilation, + dimension=dimension, + ) + self.norm2 = ME.MinkowskiBatchNorm(planes, momentum=bn_momentum) + + self.conv3 = ME.MinkowskiConvolution( + planes, planes * self.expansion, kernel_size=1, dimension=dimension + ) + self.norm3 = ME.MinkowskiBatchNorm( + planes * self.expansion, momentum=bn_momentum + ) + + self.relu = ME.MinkowskiReLU(inplace=True) + self.downsample = downsample + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.norm2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.norm3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class MinkUNetBase(nn.Module): + BLOCK = None + PLANES = None + DILATIONS = (1, 1, 1, 1, 1, 1, 1, 1) + LAYERS = (2, 2, 2, 2, 2, 2, 2, 2) + PLANES = (32, 64, 128, 256, 256, 128, 96, 96) + INIT_DIM = 32 + OUT_TENSOR_STRIDE = 1 + + def __init__(self, in_channels, out_channels, dimension=3): + super().__init__() + assert ME is not None, "Please follow `README.md` to install MinkowskiEngine.`" + self.D = dimension + assert self.BLOCK is not None + # Output of the first conv concated to conv6 + self.inplanes = self.INIT_DIM + self.conv0p1s1 = ME.MinkowskiConvolution( + in_channels, self.inplanes, kernel_size=5, dimension=self.D + ) + + self.bn0 = ME.MinkowskiBatchNorm(self.inplanes) + + self.conv1p1s2 = ME.MinkowskiConvolution( + self.inplanes, self.inplanes, kernel_size=2, stride=2, dimension=self.D + ) + self.bn1 = ME.MinkowskiBatchNorm(self.inplanes) + + self.block1 = self._make_layer(self.BLOCK, self.PLANES[0], self.LAYERS[0]) + + self.conv2p2s2 = ME.MinkowskiConvolution( + self.inplanes, self.inplanes, kernel_size=2, stride=2, dimension=self.D + ) + self.bn2 = ME.MinkowskiBatchNorm(self.inplanes) + + self.block2 = self._make_layer(self.BLOCK, self.PLANES[1], self.LAYERS[1]) + + self.conv3p4s2 = ME.MinkowskiConvolution( + self.inplanes, self.inplanes, kernel_size=2, stride=2, dimension=self.D + ) + + self.bn3 = ME.MinkowskiBatchNorm(self.inplanes) + self.block3 = self._make_layer(self.BLOCK, self.PLANES[2], self.LAYERS[2]) + + self.conv4p8s2 = ME.MinkowskiConvolution( + self.inplanes, self.inplanes, kernel_size=2, stride=2, dimension=self.D + ) + self.bn4 = ME.MinkowskiBatchNorm(self.inplanes) + self.block4 = self._make_layer(self.BLOCK, self.PLANES[3], self.LAYERS[3]) + + self.convtr4p16s2 = ME.MinkowskiConvolutionTranspose( + self.inplanes, self.PLANES[4], kernel_size=2, stride=2, dimension=self.D + ) + self.bntr4 = ME.MinkowskiBatchNorm(self.PLANES[4]) + + self.inplanes = self.PLANES[4] + self.PLANES[2] * self.BLOCK.expansion + self.block5 = self._make_layer(self.BLOCK, self.PLANES[4], self.LAYERS[4]) + self.convtr5p8s2 = ME.MinkowskiConvolutionTranspose( + self.inplanes, self.PLANES[5], kernel_size=2, stride=2, dimension=self.D + ) + self.bntr5 = ME.MinkowskiBatchNorm(self.PLANES[5]) + + self.inplanes = self.PLANES[5] + self.PLANES[1] * self.BLOCK.expansion + self.block6 = self._make_layer(self.BLOCK, self.PLANES[5], self.LAYERS[5]) + self.convtr6p4s2 = ME.MinkowskiConvolutionTranspose( + self.inplanes, self.PLANES[6], kernel_size=2, stride=2, dimension=self.D + ) + self.bntr6 = ME.MinkowskiBatchNorm(self.PLANES[6]) + + self.inplanes = self.PLANES[6] + self.PLANES[0] * self.BLOCK.expansion + self.block7 = self._make_layer(self.BLOCK, self.PLANES[6], self.LAYERS[6]) + self.convtr7p2s2 = ME.MinkowskiConvolutionTranspose( + self.inplanes, self.PLANES[7], kernel_size=2, stride=2, dimension=self.D + ) + self.bntr7 = ME.MinkowskiBatchNorm(self.PLANES[7]) + + self.inplanes = self.PLANES[7] + self.INIT_DIM + self.block8 = self._make_layer(self.BLOCK, self.PLANES[7], self.LAYERS[7]) + + self.final = ME.MinkowskiConvolution( + self.PLANES[7] * self.BLOCK.expansion, + out_channels, + kernel_size=1, + bias=True, + dimension=self.D, + ) + self.relu = ME.MinkowskiReLU(inplace=True) + + self.weight_initialization() + + def weight_initialization(self): + for m in self.modules(): + if isinstance(m, ME.MinkowskiConvolution): + ME.utils.kaiming_normal_(m.kernel, mode="fan_out", nonlinearity="relu") + + if isinstance(m, ME.MinkowskiBatchNorm): + nn.init.constant_(m.bn.weight, 1) + nn.init.constant_(m.bn.bias, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilation=1, bn_momentum=0.1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + ME.MinkowskiConvolution( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + dimension=self.D, + ), + ME.MinkowskiBatchNorm(planes * block.expansion), + ) + layers = [] + layers.append( + block( + self.inplanes, + planes, + stride=stride, + dilation=dilation, + downsample=downsample, + dimension=self.D, + ) + ) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append( + block( + self.inplanes, planes, stride=1, dilation=dilation, dimension=self.D + ) + ) + + return nn.Sequential(*layers) + + def forward(self, data_dict): + grid_coord = data_dict["grid_coord"] + feat = data_dict["feat"] + offset = data_dict["offset"] + batch = offset2batch(offset) + in_field = ME.TensorField( + feat, + coordinates=torch.cat([batch.unsqueeze(-1).int(), grid_coord.int()], dim=1), + quantization_mode=ME.SparseTensorQuantizationMode.UNWEIGHTED_AVERAGE, + minkowski_algorithm=ME.MinkowskiAlgorithm.SPEED_OPTIMIZED, + device=feat.device, + ) + x = in_field.sparse() + + out = self.conv0p1s1(x) + out = self.bn0(out) + out_p1 = self.relu(out) + + out = self.conv1p1s2(out_p1) + out = self.bn1(out) + out = self.relu(out) + out_b1p2 = self.block1(out) + + out = self.conv2p2s2(out_b1p2) + out = self.bn2(out) + out = self.relu(out) + out_b2p4 = self.block2(out) + + out = self.conv3p4s2(out_b2p4) + out = self.bn3(out) + out = self.relu(out) + out_b3p8 = self.block3(out) + + # tensor_stride=16 + out = self.conv4p8s2(out_b3p8) + out = self.bn4(out) + out = self.relu(out) + out = self.block4(out) + + # tensor_stride=8 + out = self.convtr4p16s2(out) + out = self.bntr4(out) + out = self.relu(out) + + out = ME.cat(out, out_b3p8) + out = self.block5(out) + + # tensor_stride=4 + out = self.convtr5p8s2(out) + out = self.bntr5(out) + out = self.relu(out) + + out = ME.cat(out, out_b2p4) + out = self.block6(out) + + # tensor_stride=2 + out = self.convtr6p4s2(out) + out = self.bntr6(out) + out = self.relu(out) + + out = ME.cat(out, out_b1p2) + out = self.block7(out) + + # tensor_stride=1 + out = self.convtr7p2s2(out) + out = self.bntr7(out) + out = self.relu(out) + + out = ME.cat(out, out_p1) + out = self.block8(out) + + return self.final(out).slice(in_field).F + + +@MODELS.register_module() +class MinkUNet14(MinkUNetBase): + BLOCK = BasicBlock + LAYERS = (1, 1, 1, 1, 1, 1, 1, 1) + + +@MODELS.register_module() +class MinkUNet18(MinkUNetBase): + BLOCK = BasicBlock + LAYERS = (2, 2, 2, 2, 2, 2, 2, 2) + + +@MODELS.register_module() +class MinkUNet34(MinkUNetBase): + BLOCK = BasicBlock + LAYERS = (2, 3, 4, 6, 2, 2, 2, 2) + + +@MODELS.register_module() +class MinkUNet50(MinkUNetBase): + BLOCK = Bottleneck + LAYERS = (2, 3, 4, 6, 2, 2, 2, 2) + + +@MODELS.register_module() +class MinkUNet101(MinkUNetBase): + BLOCK = Bottleneck + LAYERS = (2, 3, 4, 23, 2, 2, 2, 2) + + +@MODELS.register_module() +class MinkUNet14A(MinkUNet14): + PLANES = (32, 64, 128, 256, 128, 128, 96, 96) + + +@MODELS.register_module() +class MinkUNet14B(MinkUNet14): + PLANES = (32, 64, 128, 256, 128, 128, 128, 128) + + +@MODELS.register_module() +class MinkUNet14C(MinkUNet14): + PLANES = (32, 64, 128, 256, 192, 192, 128, 128) + + +@MODELS.register_module() +class MinkUNet14D(MinkUNet14): + PLANES = (32, 64, 128, 256, 384, 384, 384, 384) + + +@MODELS.register_module() +class MinkUNet18A(MinkUNet18): + PLANES = (32, 64, 128, 256, 128, 128, 96, 96) + + +@MODELS.register_module() +class MinkUNet18B(MinkUNet18): + PLANES = (32, 64, 128, 256, 128, 128, 128, 128) + + +@MODELS.register_module() +class MinkUNet18D(MinkUNet18): + PLANES = (32, 64, 128, 256, 384, 384, 384, 384) + + +@MODELS.register_module() +class MinkUNet34A(MinkUNet34): + PLANES = (32, 64, 128, 256, 256, 128, 96, 96) + + +@MODELS.register_module() +class MinkUNet34B(MinkUNet34): + PLANES = (32, 64, 128, 256, 256, 128, 64, 32) + + +@MODELS.register_module() +class MinkUNet34C(MinkUNet34): + PLANES = (32, 64, 128, 256, 256, 128, 96, 96) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/spconv_unet_v1m1_base.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/spconv_unet_v1m1_base.py new file mode 100644 index 0000000000000000000000000000000000000000..dfcacb00b8dfb8a38aa9ab6968b0c9c63a63301c --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/spconv_unet_v1m1_base.py @@ -0,0 +1,463 @@ +""" +SparseUNet Driven by SpConv (recommend) + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from functools import partial +from collections import OrderedDict + +import torch +import torch.nn as nn + +import spconv.pytorch as spconv +from torch_geometric.utils import scatter + +from timm.models.layers import trunc_normal_ + +from pointcept.models.builder import MODELS +from pointcept.models.utils import offset2batch + + +class BasicBlock(spconv.SparseModule): + expansion = 1 + + def __init__( + self, + in_channels, + embed_channels, + stride=1, + norm_fn=None, + indice_key=None, + bias=False, + ): + super().__init__() + + assert norm_fn is not None + + if in_channels == embed_channels: + self.proj = spconv.SparseSequential(nn.Identity()) + else: + self.proj = spconv.SparseSequential( + spconv.SubMConv3d( + in_channels, embed_channels, kernel_size=1, bias=False + ), + norm_fn(embed_channels), + ) + + self.conv1 = spconv.SubMConv3d( + in_channels, + embed_channels, + kernel_size=3, + stride=stride, + padding=1, + bias=bias, + indice_key=indice_key, + ) + self.bn1 = norm_fn(embed_channels) + self.relu = nn.ReLU() + self.conv2 = spconv.SubMConv3d( + embed_channels, + embed_channels, + kernel_size=3, + stride=stride, + padding=1, + bias=bias, + indice_key=indice_key, + ) + self.bn2 = norm_fn(embed_channels) + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = out.replace_feature(self.bn1(out.features)) + out = out.replace_feature(self.relu(out.features)) + + out = self.conv2(out) + out = out.replace_feature(self.bn2(out.features)) + + out = out.replace_feature(out.features + self.proj(residual).features) + out = out.replace_feature(self.relu(out.features)) + + return out + + +@MODELS.register_module("SpUNet-v1m1") +class SpUNetBase(nn.Module): + def __init__( + self, + in_channels, + num_classes, + base_channels=32, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + ): + super().__init__() + assert len(layers) % 2 == 0 + assert len(layers) == len(channels) + self.in_channels = in_channels + self.num_classes = num_classes + self.base_channels = base_channels + self.channels = channels + self.layers = layers + self.num_stages = len(layers) // 2 + self.cls_mode = cls_mode + + norm_fn = partial(nn.BatchNorm1d, eps=1e-3, momentum=0.01) + block = BasicBlock + + self.conv_input = spconv.SparseSequential( + spconv.SubMConv3d( + in_channels, + base_channels, + kernel_size=5, + padding=1, + bias=False, + indice_key="stem", + ), + norm_fn(base_channels), + nn.ReLU(), + ) + + enc_channels = base_channels + dec_channels = channels[-1] + self.down = nn.ModuleList() + self.up = nn.ModuleList() + self.enc = nn.ModuleList() + self.dec = nn.ModuleList() if not self.cls_mode else None + + for s in range(self.num_stages): + # encode num_stages + self.down.append( + spconv.SparseSequential( + spconv.SparseConv3d( + enc_channels, + channels[s], + kernel_size=2, + stride=2, + bias=False, + indice_key=f"spconv{s + 1}", + ), + norm_fn(channels[s]), + nn.ReLU(), + ) + ) + self.enc.append( + spconv.SparseSequential( + OrderedDict( + [ + # (f"block{i}", block(enc_channels, channels[s], norm_fn=norm_fn, indice_key=f"subm{s + 1}")) + # if i == 0 else + ( + f"block{i}", + block( + channels[s], + channels[s], + norm_fn=norm_fn, + indice_key=f"subm{s + 1}", + ), + ) + for i in range(layers[s]) + ] + ) + ) + ) + if not self.cls_mode: + # decode num_stages + self.up.append( + spconv.SparseSequential( + spconv.SparseInverseConv3d( + channels[len(channels) - s - 2], + dec_channels, + kernel_size=2, + bias=False, + indice_key=f"spconv{s + 1}", + ), + norm_fn(dec_channels), + nn.ReLU(), + ) + ) + self.dec.append( + spconv.SparseSequential( + OrderedDict( + [ + ( + ( + f"block{i}", + block( + dec_channels + enc_channels, + dec_channels, + norm_fn=norm_fn, + indice_key=f"subm{s}", + ), + ) + if i == 0 + else ( + f"block{i}", + block( + dec_channels, + dec_channels, + norm_fn=norm_fn, + indice_key=f"subm{s}", + ), + ) + ) + for i in range(layers[len(channels) - s - 1]) + ] + ) + ) + ) + + enc_channels = channels[s] + dec_channels = channels[len(channels) - s - 2] + + final_in_channels = ( + channels[-1] if not self.cls_mode else channels[self.num_stages - 1] + ) + self.final = ( + spconv.SubMConv3d( + final_in_channels, num_classes, kernel_size=1, padding=1, bias=True + ) + if num_classes > 0 + else spconv.Identity() + ) + self.apply(self._init_weights) + + @staticmethod + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, spconv.SubMConv3d): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm1d): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def forward(self, input_dict): + grid_coord = input_dict["grid_coord"] + feat = input_dict["feat"] + offset = input_dict["offset"] + + batch = offset2batch(offset) + sparse_shape = torch.add(torch.max(grid_coord, dim=0).values, 96).tolist() + x = spconv.SparseConvTensor( + features=feat, + indices=torch.cat( + [batch.unsqueeze(-1).int(), grid_coord.int()], dim=1 + ).contiguous(), + spatial_shape=sparse_shape, + batch_size=batch[-1].tolist() + 1, + ) + x = self.conv_input(x) + skips = [x] + # enc forward + for s in range(self.num_stages): + x = self.down[s](x) + x = self.enc[s](x) + skips.append(x) + x = skips.pop(-1) + if not self.cls_mode: + # dec forward + for s in reversed(range(self.num_stages)): + x = self.up[s](x) + skip = skips.pop(-1) + x = x.replace_feature(torch.cat((x.features, skip.features), dim=1)) + x = self.dec[s](x) + + x = self.final(x) + if self.cls_mode: + x = x.replace_feature( + scatter(x.features, x.indices[:, 0].long(), reduce="mean", dim=0) + ) + return x.features + + +@MODELS.register_module() +class SpUNetNoSkipBase(nn.Module): + def __init__( + self, + in_channels, + out_channels, + base_channels=32, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + ): + super().__init__() + assert len(layers) % 2 == 0 + assert len(layers) == len(channels) + self.in_channels = in_channels + self.out_channels = out_channels + self.base_channels = base_channels + self.channels = channels + self.layers = layers + self.num_stages = len(layers) // 2 + + norm_fn = partial(nn.BatchNorm1d, eps=1e-3, momentum=0.01) + block = BasicBlock + + self.conv_input = spconv.SparseSequential( + spconv.SubMConv3d( + in_channels, + base_channels, + kernel_size=5, + padding=1, + bias=False, + indice_key="stem", + ), + norm_fn(base_channels), + nn.ReLU(), + ) + + enc_channels = base_channels + dec_channels = channels[-1] + self.down = nn.ModuleList() + self.up = nn.ModuleList() + self.enc = nn.ModuleList() + self.dec = nn.ModuleList() + + for s in range(self.num_stages): + # encode num_stages + self.down.append( + spconv.SparseSequential( + spconv.SparseConv3d( + enc_channels, + channels[s], + kernel_size=2, + stride=2, + bias=False, + indice_key=f"spconv{s + 1}", + ), + norm_fn(channels[s]), + nn.ReLU(), + ) + ) + self.enc.append( + spconv.SparseSequential( + OrderedDict( + [ + # (f"block{i}", block(enc_channels, channels[s], norm_fn=norm_fn, indice_key=f"subm{s + 1}")) + # if i == 0 else + ( + f"block{i}", + block( + channels[s], + channels[s], + norm_fn=norm_fn, + indice_key=f"subm{s + 1}", + ), + ) + for i in range(layers[s]) + ] + ) + ) + ) + + # decode num_stages + self.up.append( + spconv.SparseSequential( + spconv.SparseInverseConv3d( + channels[len(channels) - s - 2], + dec_channels, + kernel_size=2, + bias=False, + indice_key=f"spconv{s + 1}", + ), + norm_fn(dec_channels), + nn.ReLU(), + ) + ) + self.dec.append( + spconv.SparseSequential( + OrderedDict( + [ + ( + ( + f"block{i}", + block( + dec_channels, + dec_channels, + norm_fn=norm_fn, + indice_key=f"subm{s}", + ), + ) + if i == 0 + else ( + f"block{i}", + block( + dec_channels, + dec_channels, + norm_fn=norm_fn, + indice_key=f"subm{s}", + ), + ) + ) + for i in range(layers[len(channels) - s - 1]) + ] + ) + ) + ) + enc_channels = channels[s] + dec_channels = channels[len(channels) - s - 2] + + self.final = ( + spconv.SubMConv3d( + channels[-1], out_channels, kernel_size=1, padding=1, bias=True + ) + if out_channels > 0 + else spconv.Identity() + ) + self.apply(self._init_weights) + + @staticmethod + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, spconv.SubMConv3d): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm1d): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def forward(self, data_dict): + grid_coord = data_dict["grid_coord"] + feat = data_dict["feat"] + offset = data_dict["offset"] + batch = offset2batch(offset) + sparse_shape = torch.add(torch.max(grid_coord, dim=0).values, 1).tolist() + x = spconv.SparseConvTensor( + features=feat, + indices=torch.cat( + [batch.unsqueeze(-1).int(), grid_coord.int()], dim=1 + ).contiguous(), + spatial_shape=sparse_shape, + batch_size=batch[-1].tolist() + 1, + ) + x = self.conv_input(x) + skips = [x] + # enc forward + for s in range(self.num_stages): + x = self.down[s](x) + x = self.enc[s](x) + skips.append(x) + x = skips.pop(-1) + # dec forward + for s in reversed(range(self.num_stages)): + x = self.up[s](x) + # skip = skips.pop(-1) + # x = x.replace_feature(torch.cat((x.features, skip.features), dim=1)) + x = self.dec[s](x) + + x = self.final(x) + return x.features diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/spconv_unet_v1m2_bn_momentum.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/spconv_unet_v1m2_bn_momentum.py new file mode 100644 index 0000000000000000000000000000000000000000..979b1b8b5488d6c55bbc20ad0cede5002c8a5c67 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/spconv_unet_v1m2_bn_momentum.py @@ -0,0 +1,290 @@ +""" +SparseUNet Driven by SpConv (recommend) + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from functools import partial +from collections import OrderedDict + +import torch +import torch.nn as nn + +try: + import spconv.pytorch as spconv +except ImportError: + import warnings + + warnings.warn("Please follow `README.md` to install spconv2.`") + +from timm.models.layers import trunc_normal_ +from pointcept.models.builder import MODELS + + +def offset2batch(offset): + return ( + torch.cat( + [ + ( + torch.tensor([i] * (o - offset[i - 1])) + if i > 0 + else torch.tensor([i] * o) + ) + for i, o in enumerate(offset) + ], + dim=0, + ) + .long() + .to(offset.device) + ) + + +class BasicBlock(spconv.SparseModule): + expansion = 1 + + def __init__( + self, + in_channels, + embed_channels, + stride=1, + norm_fn=None, + indice_key=None, + bias=False, + ): + super().__init__() + + assert norm_fn is not None + + if in_channels == embed_channels: + self.proj = spconv.SparseSequential(nn.Identity()) + else: + self.proj = spconv.SparseSequential( + spconv.SubMConv3d( + in_channels, embed_channels, kernel_size=1, bias=False + ), + norm_fn(embed_channels, momentum=0.02), + ) + + self.conv1 = spconv.SubMConv3d( + in_channels, + embed_channels, + kernel_size=3, + stride=stride, + padding=1, + bias=bias, + indice_key=indice_key, + ) + self.bn1 = norm_fn(embed_channels) + self.relu = nn.ReLU() + self.conv2 = spconv.SubMConv3d( + embed_channels, + embed_channels, + kernel_size=3, + stride=stride, + padding=1, + bias=bias, + indice_key=indice_key, + ) + self.bn2 = norm_fn(embed_channels) + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = out.replace_feature(self.bn1(out.features)) + out = out.replace_feature(self.relu(out.features)) + + out = self.conv2(out) + out = out.replace_feature(self.bn2(out.features)) + + out = out.replace_feature(out.features + self.proj(residual).features) + out = out.replace_feature(self.relu(out.features)) + + return out + + +@MODELS.register_module("SpUNet-v1m2") +class SpUNetBase(nn.Module): + def __init__( + self, + in_channels, + num_classes, + base_channels=32, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + bn_momentum=0.1, + ): + super().__init__() + assert len(layers) % 2 == 0 + assert len(layers) == len(channels) + self.in_channels = in_channels + self.num_classes = num_classes + self.base_channels = base_channels + self.channels = channels + self.layers = layers + self.num_stages = len(layers) // 2 + + norm_fn = partial(nn.BatchNorm1d, eps=1e-5, momentum=bn_momentum) + block = BasicBlock + + self.conv_input = spconv.SparseSequential( + spconv.SubMConv3d( + in_channels, + base_channels, + kernel_size=5, + padding=1, + bias=False, + indice_key="stem", + ), + norm_fn(base_channels, momentum=0.02), + nn.ReLU(), + ) + + enc_channels = base_channels + dec_channels = channels[-1] + self.down = nn.ModuleList() + self.up = nn.ModuleList() + self.enc = nn.ModuleList() + self.dec = nn.ModuleList() + + for s in range(self.num_stages): + # encode num_stages + self.down.append( + spconv.SparseSequential( + spconv.SparseConv3d( + enc_channels, + channels[s], + kernel_size=2, + stride=2, + bias=False, + indice_key=f"spconv{s + 1}", + ), + norm_fn(channels[s], momentum=0.02), + nn.ReLU(), + ) + ) + self.enc.append( + spconv.SparseSequential( + OrderedDict( + [ + # (f"block{i}", block(enc_channels, channels[s], norm_fn=norm_fn, indice_key=f"subm{s + 1}")) + # if i == 0 else + ( + f"block{i}", + block( + channels[s], + channels[s], + norm_fn=norm_fn, + indice_key=f"subm{s + 1}", + ), + ) + for i in range(layers[s]) + ] + ) + ) + ) + + # decode num_stages + self.up.append( + spconv.SparseSequential( + spconv.SparseInverseConv3d( + channels[len(channels) - s - 2], + dec_channels, + kernel_size=2, + bias=False, + indice_key=f"spconv{s + 1}", + ), + norm_fn(dec_channels, momentum=0.02), + nn.ReLU(), + ) + ) + self.dec.append( + spconv.SparseSequential( + OrderedDict( + [ + ( + ( + f"block{i}", + block( + dec_channels + enc_channels, + dec_channels, + norm_fn=norm_fn, + indice_key=f"subm{s}", + ), + ) + if i == 0 + else ( + f"block{i}", + block( + dec_channels, + dec_channels, + norm_fn=norm_fn, + indice_key=f"subm{s}", + ), + ) + ) + for i in range(layers[len(channels) - s - 1]) + ] + ) + ) + ) + enc_channels = channels[s] + dec_channels = channels[len(channels) - s - 2] + + self.final = ( + spconv.SubMConv3d( + channels[-1], num_classes, kernel_size=1, padding=1, bias=True + ) + if num_classes > 0 + else spconv.Identity() + ) + self.apply(self._init_weights) + + @staticmethod + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, spconv.SubMConv3d): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm1d): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def forward(self, data_dict): + grid_coord = data_dict["grid_coord"] + feat = data_dict["feat"] + offset = data_dict["offset"] + + batch = offset2batch(offset) + sparse_shape = torch.add(torch.max(grid_coord, dim=0).values, 1).tolist() + x = spconv.SparseConvTensor( + features=feat, + indices=torch.cat( + [batch.unsqueeze(-1).int(), grid_coord.int()], dim=1 + ).contiguous(), + spatial_shape=sparse_shape, + batch_size=batch[-1].tolist() + 1, + ) + x = self.conv_input(x) + skips = [x] + # enc forward + for s in range(self.num_stages): + x = self.down[s](x) + x = self.enc[s](x) + skips.append(x) + x = skips.pop(-1) + # dec forward + for s in reversed(range(self.num_stages)): + x = self.up[s](x) + skip = skips.pop(-1) + x = x.replace_feature(torch.cat((x.features, skip.features), dim=1)) + x = self.dec[s](x) + + x = self.final(x) + return x.features diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/spconv_unet_v1m3_pdnorm.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/spconv_unet_v1m3_pdnorm.py new file mode 100644 index 0000000000000000000000000000000000000000..968f8f2c5a19cf016f9427812a8071ca83d612d5 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/sparse_unet/spconv_unet_v1m3_pdnorm.py @@ -0,0 +1,429 @@ +""" +SparseUNet V1M3 + +Enable Prompt-Driven Normalization for Point Prompt Training + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from functools import partial +from collections import OrderedDict + +import torch +import torch.nn as nn + +import spconv.pytorch as spconv +from torch_geometric.utils import scatter + +from timm.models.layers import trunc_normal_ + +from pointcept.models.builder import MODELS +from pointcept.models.utils import offset2batch + + +class PDBatchNorm(torch.nn.Module): + def __init__( + self, + num_features, + context_channels=256, + eps=1e-3, + momentum=0.01, + conditions=("ScanNet", "S3DIS", "Structured3D"), + decouple=True, + adaptive=False, + affine=True, + ): + super().__init__() + self.conditions = conditions + self.decouple = decouple + self.adaptive = adaptive + self.affine = affine + if self.decouple: + self.bns = nn.ModuleList( + [ + nn.BatchNorm1d( + num_features=num_features, + eps=eps, + momentum=momentum, + affine=affine, + ) + for _ in conditions + ] + ) + else: + self.bn = nn.BatchNorm1d( + num_features=num_features, eps=eps, momentum=momentum, affine=affine + ) + if self.adaptive: + self.modulation = nn.Sequential( + nn.SiLU(), nn.Linear(context_channels, 2 * num_features, bias=True) + ) + + def forward(self, feat, condition=None, context=None): + if self.decouple: + assert condition in self.conditions + bn = self.bns[self.conditions.index(condition)] + else: + bn = self.bn + feat = bn(feat) + if self.adaptive: + assert context is not None + shift, scale = self.modulation(context).chunk(2, dim=1) + feat = feat * (1.0 + scale) + shift + return feat + + +class BasicBlock(spconv.SparseModule): + expansion = 1 + + def __init__( + self, + in_channels, + embed_channels, + stride=1, + norm_fn=None, + indice_key=None, + bias=False, + ): + super().__init__() + + assert norm_fn is not None + + self.in_channels = in_channels + self.embed_channels = embed_channels + if in_channels == embed_channels: + self.proj = spconv.SparseSequential(nn.Identity()) + else: + # TODO remove norm after project + self.proj_conv = spconv.SubMConv3d( + in_channels, embed_channels, kernel_size=1, bias=False + ) + self.proj_norm = norm_fn(embed_channels) + + self.conv1 = spconv.SubMConv3d( + in_channels, + embed_channels, + kernel_size=3, + stride=stride, + padding=1, + bias=bias, + indice_key=indice_key, + ) + self.bn1 = norm_fn(embed_channels) + self.relu = nn.ReLU() + self.conv2 = spconv.SubMConv3d( + embed_channels, + embed_channels, + kernel_size=3, + stride=stride, + padding=1, + bias=bias, + indice_key=indice_key, + ) + self.bn2 = norm_fn(embed_channels) + self.stride = stride + + def forward(self, x): + x, condition, context = x + residual = x + + out = self.conv1(x) + out = out.replace_feature(self.bn1(out.features, condition, context)) + out = out.replace_feature(self.relu(out.features)) + + out = self.conv2(out) + out = out.replace_feature(self.bn2(out.features, condition, context)) + + if self.in_channels == self.embed_channels: + residual = self.proj(residual) + else: + residual = residual.replace_feature( + self.proj_norm(self.proj_conv(residual).features, condition, context) + ) + out = out.replace_feature(out.features + residual.features) + out = out.replace_feature(self.relu(out.features)) + return out, condition, context + + +class SPConvDown(nn.Module): + def __init__( + self, + in_channels, + out_channels, + indice_key, + kernel_size=2, + bias=False, + norm_fn=None, + ): + super().__init__() + self.conv = spconv.SparseConv3d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=kernel_size, + bias=bias, + indice_key=indice_key, + ) + self.bn = norm_fn(out_channels) + self.relu = nn.ReLU() + + def forward(self, x): + x, condition, context = x + out = self.conv(x) + out = out.replace_feature(self.bn(out.features, condition, context)) + out = out.replace_feature(self.relu(out.features)) + return out + + +class SPConvUp(nn.Module): + def __init__( + self, + in_channels, + out_channels, + indice_key, + kernel_size=2, + bias=False, + norm_fn=None, + ): + super().__init__() + self.conv = spconv.SparseInverseConv3d( + in_channels, + out_channels, + kernel_size=kernel_size, + bias=bias, + indice_key=indice_key, + ) + self.bn = norm_fn(out_channels) + self.relu = nn.ReLU() + + def forward(self, x): + x, condition, context = x + out = self.conv(x) + out = out.replace_feature(self.bn(out.features, condition, context)) + out = out.replace_feature(self.relu(out.features)) + return out + + +class SPConvPatchEmbedding(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size=5, norm_fn=None): + super().__init__() + self.conv = spconv.SubMConv3d( + in_channels, + out_channels, + kernel_size=kernel_size, + padding=1, + bias=False, + indice_key="stem", + ) + self.bn = norm_fn(out_channels) + self.relu = nn.ReLU() + + def forward(self, x): + x, condition, context = x + out = self.conv(x) + out = out.replace_feature(self.bn(out.features, condition, context)) + out = out.replace_feature(self.relu(out.features)) + return out + + +@MODELS.register_module("SpUNet-v1m3") +class SpUNetBase(nn.Module): + def __init__( + self, + in_channels, + num_classes=0, + base_channels=32, + context_channels=256, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 3, 4, 6, 2, 2, 2, 2), + cls_mode=False, + conditions=("ScanNet", "S3DIS", "Structured3D"), + zero_init=True, + norm_decouple=True, + norm_adaptive=True, + norm_affine=False, + ): + super().__init__() + assert len(layers) % 2 == 0 + assert len(layers) == len(channels) + self.in_channels = in_channels + self.num_classes = num_classes + self.base_channels = base_channels + self.channels = channels + self.layers = layers + self.num_stages = len(layers) // 2 + self.cls_mode = cls_mode + self.conditions = conditions + self.zero_init = zero_init + + norm_fn = partial( + PDBatchNorm, + eps=1e-3, + momentum=0.01, + conditions=conditions, + context_channels=context_channels, + decouple=norm_decouple, + adaptive=norm_adaptive, + affine=norm_affine, + ) + block = BasicBlock + + self.conv_input = SPConvPatchEmbedding( + in_channels, base_channels, kernel_size=5, norm_fn=norm_fn + ) + + enc_channels = base_channels + dec_channels = channels[-1] + self.down = nn.ModuleList() + self.up = nn.ModuleList() + self.enc = nn.ModuleList() + self.dec = nn.ModuleList() if not self.cls_mode else None + + for s in range(self.num_stages): + # encode num_stages + self.down.append( + SPConvDown( + enc_channels, + channels[s], + kernel_size=2, + bias=False, + indice_key=f"spconv{s + 1}", + norm_fn=norm_fn, + ) + ) + self.enc.append( + spconv.SparseSequential( + OrderedDict( + [ + # (f"block{i}", block(enc_channels, channels[s], norm_fn=norm_fn, indice_key=f"subm{s + 1}")) + # if i == 0 else + ( + f"block{i}", + block( + channels[s], + channels[s], + norm_fn=norm_fn, + indice_key=f"subm{s + 1}", + ), + ) + for i in range(layers[s]) + ] + ) + ) + ) + if not self.cls_mode: + # decode num_stages + self.up.append( + SPConvUp( + channels[len(channels) - s - 2], + dec_channels, + kernel_size=2, + bias=False, + indice_key=f"spconv{s + 1}", + norm_fn=norm_fn, + ) + ) + self.dec.append( + spconv.SparseSequential( + OrderedDict( + [ + ( + ( + f"block{i}", + block( + dec_channels + enc_channels, + dec_channels, + norm_fn=norm_fn, + indice_key=f"subm{s}", + ), + ) + if i == 0 + else ( + f"block{i}", + block( + dec_channels, + dec_channels, + norm_fn=norm_fn, + indice_key=f"subm{s}", + ), + ) + ) + for i in range(layers[len(channels) - s - 1]) + ] + ) + ) + ) + + enc_channels = channels[s] + dec_channels = channels[len(channels) - s - 2] + + final_in_channels = ( + channels[-1] if not self.cls_mode else channels[self.num_stages - 1] + ) + self.final = ( + spconv.SubMConv3d( + final_in_channels, num_classes, kernel_size=1, padding=1, bias=True + ) + if num_classes > 0 + else spconv.Identity() + ) + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, spconv.SubMConv3d): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm1d): + if m.affine: + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, PDBatchNorm): + if self.zero_init: + nn.init.constant_(m.modulation[-1].weight, 0) + nn.init.constant_(m.modulation[-1].bias, 0) + + def forward(self, input_dict): + grid_coord = input_dict["grid_coord"] + feat = input_dict["feat"] + offset = input_dict["offset"] + condition = input_dict["condition"][0] + context = input_dict["context"] if "context" in input_dict.keys() else None + + batch = offset2batch(offset) + sparse_shape = torch.add(torch.max(grid_coord, dim=0).values, 96).tolist() + x = spconv.SparseConvTensor( + features=feat, + indices=torch.cat( + [batch.unsqueeze(-1).int(), grid_coord.int()], dim=1 + ).contiguous(), + spatial_shape=sparse_shape, + batch_size=batch[-1].tolist() + 1, + ) + x = self.conv_input([x, condition, context]) + skips = [x] + # enc forward + for s in range(self.num_stages): + x = self.down[s]([x, condition, context]) + x, _, _ = self.enc[s]([x, condition, context]) + skips.append(x) + x = skips.pop(-1) + if not self.cls_mode: + # dec forward + for s in reversed(range(self.num_stages)): + x = self.up[s]([x, condition, context]) + skip = skips.pop(-1) + x = x.replace_feature(torch.cat((x.features, skip.features), dim=1)) + x, _, _ = self.dec[s]([x, condition, context]) + + x = self.final(x) + if self.cls_mode: + x = x.replace_feature( + scatter(x.features, x.indices[:, 0].long(), reduce="mean", dim=0) + ) + return x.features diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/spvcnn/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/spvcnn/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ecdc75a6878026437124c187323ca9676bf35c76 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/spvcnn/__init__.py @@ -0,0 +1 @@ +from .ts_spvcnn import * diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/spvcnn/ts_spvcnn.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/spvcnn/ts_spvcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..c26f1ea8d41a8edfbd1542b0f9e607e660441b38 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/spvcnn/ts_spvcnn.py @@ -0,0 +1,438 @@ +""" +SPVCNN + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import torch +import torch.nn as nn + +try: + import torchsparse + import torchsparse.nn as spnn + import torchsparse.nn.functional as F + from torchsparse.nn.utils import get_kernel_offsets + from torchsparse import PointTensor, SparseTensor +except ImportError: + torchsparse = None + + +from pointcept.models.utils import offset2batch +from pointcept.models.builder import MODELS + + +def initial_voxelize(z): + pc_hash = F.sphash(torch.floor(z.C).int()) + sparse_hash = torch.unique(pc_hash) + idx_query = F.sphashquery(pc_hash, sparse_hash) + counts = F.spcount(idx_query.int(), len(sparse_hash)) + + inserted_coords = F.spvoxelize(torch.floor(z.C), idx_query, counts) + inserted_coords = torch.round(inserted_coords).int() + inserted_feat = F.spvoxelize(z.F, idx_query, counts) + + new_tensor = SparseTensor(inserted_feat, inserted_coords, 1) + new_tensor.cmaps.setdefault(new_tensor.stride, new_tensor.coords) + z.additional_features["idx_query"][1] = idx_query + z.additional_features["counts"][1] = counts + return new_tensor + + +# x: SparseTensor, z: PointTensor +# return: SparseTensor +def point_to_voxel(x, z): + if ( + z.additional_features is None + or z.additional_features.get("idx_query") is None + or z.additional_features["idx_query"].get(x.s) is None + ): + pc_hash = F.sphash( + torch.cat( + [ + torch.floor(z.C[:, :3] / x.s[0]).int() * x.s[0], + z.C[:, -1].int().view(-1, 1), + ], + 1, + ) + ) + sparse_hash = F.sphash(x.C) + idx_query = F.sphashquery(pc_hash, sparse_hash) + counts = F.spcount(idx_query.int(), x.C.shape[0]) + z.additional_features["idx_query"][x.s] = idx_query + z.additional_features["counts"][x.s] = counts + else: + idx_query = z.additional_features["idx_query"][x.s] + counts = z.additional_features["counts"][x.s] + + inserted_feat = F.spvoxelize(z.F, idx_query, counts) + new_tensor = SparseTensor(inserted_feat, x.C, x.s) + new_tensor.cmaps = x.cmaps + new_tensor.kmaps = x.kmaps + + return new_tensor + + +# x: SparseTensor, z: PointTensor +# return: PointTensor +def voxel_to_point(x, z, nearest=False): + if ( + z.idx_query is None + or z.weights is None + or z.idx_query.get(x.s) is None + or z.weights.get(x.s) is None + ): + off = spnn.utils.get_kernel_offsets(2, x.s, 1, device=z.F.device) + old_hash = F.sphash( + torch.cat( + [ + torch.floor(z.C[:, :3] / x.s[0]).int() * x.s[0], + z.C[:, -1].int().view(-1, 1), + ], + 1, + ), + off, + ) + pc_hash = F.sphash(x.C.to(z.F.device)) + idx_query = F.sphashquery(old_hash, pc_hash) + weights = ( + F.calc_ti_weights(z.C, idx_query, scale=x.s[0]).transpose(0, 1).contiguous() + ) + idx_query = idx_query.transpose(0, 1).contiguous() + if nearest: + weights[:, 1:] = 0.0 + idx_query[:, 1:] = -1 + new_feat = F.spdevoxelize(x.F, idx_query, weights) + new_tensor = PointTensor( + new_feat, z.C, idx_query=z.idx_query, weights=z.weights + ) + new_tensor.additional_features = z.additional_features + new_tensor.idx_query[x.s] = idx_query + new_tensor.weights[x.s] = weights + z.idx_query[x.s] = idx_query + z.weights[x.s] = weights + + else: + new_feat = F.spdevoxelize(x.F, z.idx_query.get(x.s), z.weights.get(x.s)) + new_tensor = PointTensor( + new_feat, z.C, idx_query=z.idx_query, weights=z.weights + ) + new_tensor.additional_features = z.additional_features + + return new_tensor + + +class BasicConvolutionBlock(nn.Module): + def __init__(self, inc, outc, ks=3, stride=1, dilation=1): + super().__init__() + self.net = nn.Sequential( + spnn.Conv3d(inc, outc, kernel_size=ks, dilation=dilation, stride=stride), + spnn.BatchNorm(outc), + spnn.ReLU(True), + ) + + def forward(self, x): + out = self.net(x) + return out + + +class BasicDeconvolutionBlock(nn.Module): + def __init__(self, inc, outc, ks=3, stride=1): + super().__init__() + self.net = nn.Sequential( + spnn.Conv3d(inc, outc, kernel_size=ks, stride=stride, transposed=True), + spnn.BatchNorm(outc), + spnn.ReLU(True), + ) + + def forward(self, x): + return self.net(x) + + +class ResidualBlock(nn.Module): + def __init__(self, inc, outc, ks=3, stride=1, dilation=1): + super().__init__() + self.net = nn.Sequential( + spnn.Conv3d(inc, outc, kernel_size=ks, dilation=dilation, stride=stride), + spnn.BatchNorm(outc), + spnn.ReLU(True), + spnn.Conv3d(outc, outc, kernel_size=ks, dilation=dilation, stride=1), + spnn.BatchNorm(outc), + ) + + if inc == outc and stride == 1: + self.downsample = nn.Identity() + else: + self.downsample = nn.Sequential( + spnn.Conv3d(inc, outc, kernel_size=1, dilation=1, stride=stride), + spnn.BatchNorm(outc), + ) + + self.relu = spnn.ReLU(True) + + def forward(self, x): + out = self.relu(self.net(x) + self.downsample(x)) + return out + + +@MODELS.register_module() +class SPVCNN(nn.Module): + def __init__( + self, + in_channels, + out_channels, + base_channels=32, + channels=(32, 64, 128, 256, 256, 128, 96, 96), + layers=(2, 2, 2, 2, 2, 2, 2, 2), + ): # not implement + super().__init__() + + assert ( + torchsparse is not None + ), "Please follow `README.md` to install torchsparse.`" + assert len(layers) % 2 == 0 + assert len(layers) == len(channels) + self.in_channels = in_channels + self.out_channels = out_channels + self.base_channels = base_channels + self.channels = channels + self.layers = layers + self.num_stages = len(layers) // 2 + + self.stem = nn.Sequential( + spnn.Conv3d(in_channels, base_channels, kernel_size=3, stride=1), + spnn.BatchNorm(base_channels), + spnn.ReLU(True), + spnn.Conv3d(base_channels, base_channels, kernel_size=3, stride=1), + spnn.BatchNorm(base_channels), + spnn.ReLU(True), + ) + + self.stage1 = nn.Sequential( + *[ + BasicConvolutionBlock( + base_channels, base_channels, ks=2, stride=2, dilation=1 + ), + ResidualBlock(base_channels, channels[0], ks=3, stride=1, dilation=1), + ] + + [ + ResidualBlock(channels[0], channels[0], ks=3, stride=1, dilation=1) + for _ in range(layers[0] - 1) + ] + ) + + self.stage2 = nn.Sequential( + *[ + BasicConvolutionBlock( + channels[0], channels[0], ks=2, stride=2, dilation=1 + ), + ResidualBlock(channels[0], channels[1], ks=3, stride=1, dilation=1), + ] + + [ + ResidualBlock(channels[1], channels[1], ks=3, stride=1, dilation=1) + for _ in range(layers[1] - 1) + ] + ) + + self.stage3 = nn.Sequential( + *[ + BasicConvolutionBlock( + channels[1], channels[1], ks=2, stride=2, dilation=1 + ), + ResidualBlock(channels[1], channels[2], ks=3, stride=1, dilation=1), + ] + + [ + ResidualBlock(channels[2], channels[2], ks=3, stride=1, dilation=1) + for _ in range(layers[2] - 1) + ] + ) + + self.stage4 = nn.Sequential( + *[ + BasicConvolutionBlock( + channels[2], channels[2], ks=2, stride=2, dilation=1 + ), + ResidualBlock(channels[2], channels[3], ks=3, stride=1, dilation=1), + ] + + [ + ResidualBlock(channels[3], channels[3], ks=3, stride=1, dilation=1) + for _ in range(layers[3] - 1) + ] + ) + + self.up1 = nn.ModuleList( + [ + BasicDeconvolutionBlock(channels[3], channels[4], ks=2, stride=2), + nn.Sequential( + *[ + ResidualBlock( + channels[4] + channels[2], + channels[4], + ks=3, + stride=1, + dilation=1, + ) + ] + + [ + ResidualBlock( + channels[4], channels[4], ks=3, stride=1, dilation=1 + ) + for _ in range(layers[4] - 1) + ] + ), + ] + ) + + self.up2 = nn.ModuleList( + [ + BasicDeconvolutionBlock(channels[4], channels[5], ks=2, stride=2), + nn.Sequential( + *[ + ResidualBlock( + channels[5] + channels[1], + channels[5], + ks=3, + stride=1, + dilation=1, + ) + ] + + [ + ResidualBlock( + channels[5], channels[5], ks=3, stride=1, dilation=1 + ) + for _ in range(layers[5] - 1) + ] + ), + ] + ) + + self.up3 = nn.ModuleList( + [ + BasicDeconvolutionBlock(channels[5], channels[6], ks=2, stride=2), + nn.Sequential( + *[ + ResidualBlock( + channels[6] + channels[0], + channels[6], + ks=3, + stride=1, + dilation=1, + ) + ] + + [ + ResidualBlock( + channels[6], channels[6], ks=3, stride=1, dilation=1 + ) + for _ in range(layers[6] - 1) + ] + ), + ] + ) + + self.up4 = nn.ModuleList( + [ + BasicDeconvolutionBlock(channels[6], channels[7], ks=2, stride=2), + nn.Sequential( + *[ + ResidualBlock( + channels[7] + base_channels, + channels[7], + ks=3, + stride=1, + dilation=1, + ) + ] + + [ + ResidualBlock( + channels[7], channels[7], ks=3, stride=1, dilation=1 + ) + for _ in range(layers[7] - 1) + ] + ), + ] + ) + + self.classifier = nn.Sequential(nn.Linear(channels[7], out_channels)) + + self.point_transforms = nn.ModuleList( + [ + nn.Sequential( + nn.Linear(base_channels, channels[3]), + nn.BatchNorm1d(channels[3]), + nn.ReLU(True), + ), + nn.Sequential( + nn.Linear(channels[3], channels[5]), + nn.BatchNorm1d(channels[5]), + nn.ReLU(True), + ), + nn.Sequential( + nn.Linear(channels[5], channels[7]), + nn.BatchNorm1d(channels[7]), + nn.ReLU(True), + ), + ] + ) + + self.weight_initialization() + self.dropout = nn.Dropout(0.3, True) + + def weight_initialization(self): + for m in self.modules(): + if isinstance(m, nn.BatchNorm1d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def forward(self, data_dict): + grid_coord = data_dict["grid_coord"] + feat = data_dict["feat"] + offset = data_dict["offset"] + batch = offset2batch(offset) + + # x: SparseTensor z: PointTensor + z = PointTensor( + feat, + torch.cat( + [grid_coord.float(), batch.unsqueeze(-1).float()], dim=1 + ).contiguous(), + ) + x0 = initial_voxelize(z) + + x0 = self.stem(x0) + z0 = voxel_to_point(x0, z, nearest=False) + z0.F = z0.F + + x1 = point_to_voxel(x0, z0) + x1 = self.stage1(x1) + x2 = self.stage2(x1) + x3 = self.stage3(x2) + x4 = self.stage4(x3) + z1 = voxel_to_point(x4, z0) + z1.F = z1.F + self.point_transforms[0](z0.F) + + y1 = point_to_voxel(x4, z1) + y1.F = self.dropout(y1.F) + y1 = self.up1[0](y1) + y1 = torchsparse.cat([y1, x3]) + y1 = self.up1[1](y1) + + y2 = self.up2[0](y1) + y2 = torchsparse.cat([y2, x2]) + y2 = self.up2[1](y2) + z2 = voxel_to_point(y2, z1) + z2.F = z2.F + self.point_transforms[1](z1.F) + + y3 = point_to_voxel(y2, z2) + y3.F = self.dropout(y3.F) + y3 = self.up3[0](y3) + y3 = torchsparse.cat([y3, x1]) + y3 = self.up3[1](y3) + + y4 = self.up4[0](y3) + y4 = torchsparse.cat([y4, x0]) + y4 = self.up4[1](y4) + z3 = voxel_to_point(y4, z2) + z3.F = z3.F + self.point_transforms[2](z2.F) + + out = self.classifier(z3.F) + return out diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/stratified_transformer/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/stratified_transformer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..24712d2bc0fe76a52a273172fe9c4f37c807a6c3 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/stratified_transformer/__init__.py @@ -0,0 +1,2 @@ +from .stratified_transformer_v1m1_origin import StratifiedTransformer +from .stratified_transformer_v1m2_refine import StratifiedTransformer diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/stratified_transformer/stratified_transformer_v1m1_origin.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/stratified_transformer/stratified_transformer_v1m1_origin.py new file mode 100644 index 0000000000000000000000000000000000000000..5bf18f71298efa1e96f0697c96e26d2127140c36 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/stratified_transformer/stratified_transformer_v1m1_origin.py @@ -0,0 +1,830 @@ +import torch +import torch.nn as nn + +try: + import torch_points_kernels as tp +except ImportError: + tp = None + +try: + from torch_points3d.modules.KPConv.kernels import KPConvLayer + from torch_points3d.core.common_modules import FastBatchNorm1d +except ImportError: + KPConvLayer = None + FastBatchNorm1d = None + +from torch_scatter import scatter_softmax +from timm.models.layers import DropPath, trunc_normal_ +from torch_geometric.nn.pool import voxel_grid + +try: + import pointops2.pointops as pointops +except ImportError: + pointops = None + +from pointcept.models.builder import MODELS + + +def offset2batch(offset): + return ( + torch.cat( + [ + ( + torch.tensor([i] * (o - offset[i - 1])) + if i > 0 + else torch.tensor([i] * o) + ) + for i, o in enumerate(offset) + ], + dim=0, + ) + .long() + .to(offset.device) + ) + + +def get_indice_pairs( + p2v_map, counts, new_p2v_map, new_counts, downsample_idx, batch, xyz, window_size, i +): + # p2v_map: [n, k] + # counts: [n, ] + + n, k = p2v_map.shape + mask = torch.arange(k).unsqueeze(0).cuda() < counts.unsqueeze(-1) # [n, k] + mask_mat = mask.unsqueeze(-1) & mask.unsqueeze(-2) # [n, k, k] + index_0 = p2v_map.unsqueeze(-1).expand(-1, -1, k)[mask_mat] # [M, ] + index_1 = p2v_map.unsqueeze(1).expand(-1, k, -1)[mask_mat] # [M, ] + + downsample_mask = torch.zeros_like(batch).bool() # [N, ] + downsample_mask[downsample_idx.long()] = True + + downsample_mask = downsample_mask[new_p2v_map] # [n, k] + n, k = new_p2v_map.shape + mask = torch.arange(k).unsqueeze(0).cuda() < new_counts.unsqueeze(-1) # [n, k] + downsample_mask = downsample_mask & mask + mask_mat = mask.unsqueeze(-1) & downsample_mask.unsqueeze(-2) # [n, k, k] + xyz_min = xyz.min(0)[0] + if i % 2 == 0: + window_coord = (xyz[new_p2v_map] - xyz_min) // window_size # [n, k, 3] + else: + window_coord = ( + xyz[new_p2v_map] + 1 / 2 * window_size - xyz_min + ) // window_size # [n, k, 3] + + mask_mat_prev = (window_coord.unsqueeze(2) != window_coord.unsqueeze(1)).any( + -1 + ) # [n, k, k] + mask_mat = mask_mat & mask_mat_prev # [n, k, k] + + new_index_0 = new_p2v_map.unsqueeze(-1).expand(-1, -1, k)[mask_mat] # [M, ] + new_index_1 = new_p2v_map.unsqueeze(1).expand(-1, k, -1)[mask_mat] # [M, ] + + index_0 = torch.cat([index_0, new_index_0], 0) + index_1 = torch.cat([index_1, new_index_1], 0) + return index_0, index_1 + + +def grid_sample(pos, batch, size, start, return_p2v=True): + # pos: float [N, 3] + # batch: long [N] + # size: float [3, ] + # start: float [3, ] / None + + cluster = voxel_grid(pos, batch, size, start=start) # [N, ] + + if return_p2v == False: + unique, cluster = torch.unique(cluster, sorted=True, return_inverse=True) + return cluster + + unique, cluster, counts = torch.unique( + cluster, sorted=True, return_inverse=True, return_counts=True + ) + + # obtain p2v_map + n = unique.shape[0] + k = counts.max().item() + p2v_map = cluster.new_zeros(n, k) # [n, k] + mask = torch.arange(k).cuda().unsqueeze(0) < counts.unsqueeze(-1) # [n, k] + p2v_map[mask] = torch.argsort(cluster) + + return cluster, p2v_map, counts + + +class Mlp(nn.Module): + """Multilayer perceptron.""" + + def __init__( + self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.0, + ): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop, inplace=True) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class TransitionDown(nn.Module): + def __init__(self, in_channels, out_channels, ratio, k, norm_layer=nn.LayerNorm): + super().__init__() + self.ratio = ratio + self.k = k + self.norm = norm_layer(in_channels) if norm_layer else None + self.linear = nn.Linear(in_channels, out_channels, bias=False) + self.pool = nn.MaxPool1d(k) + + def forward(self, feats, xyz, offset): + n_offset, count = [int(offset[0].item() * self.ratio) + 1], int( + offset[0].item() * self.ratio + ) + 1 + for i in range(1, offset.shape[0]): + count += ((offset[i].item() - offset[i - 1].item()) * self.ratio) + 1 + n_offset.append(count) + n_offset = torch.cuda.IntTensor(n_offset) + idx = pointops.furthestsampling(xyz, offset, n_offset) # (m) + n_xyz = xyz[idx.long(), :] # (m, 3) + + feats = pointops.queryandgroup( + self.k, xyz, n_xyz, feats, None, offset, n_offset, use_xyz=False + ) # (m, nsample, 3+c) + m, k, c = feats.shape + feats = ( + self.linear(self.norm(feats.view(m * k, c)).view(m, k, c)) + .transpose(1, 2) + .contiguous() + ) + feats = self.pool(feats).squeeze(-1) # (m, c) + + return feats, n_xyz, n_offset + + +class WindowAttention(nn.Module): + """Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__( + self, + dim, + window_size, + num_heads, + quant_size, + rel_query=True, + rel_key=False, + rel_value=False, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + ): + super().__init__() + self.dim = dim + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + self.window_size = window_size + + self.quant_size = quant_size + self.rel_query = rel_query + self.rel_key = rel_key + self.rel_value = rel_value + + quant_grid_length = int((2 * window_size + 1e-4) // quant_size) + + if rel_query: + self.relative_pos_query_table = nn.Parameter( + torch.zeros(2 * quant_grid_length, num_heads, head_dim, 3) + ) + trunc_normal_(self.relative_pos_query_table, std=0.02) + if rel_key: + self.relative_pos_key_table = nn.Parameter( + torch.zeros(2 * quant_grid_length, num_heads, head_dim, 3) + ) + trunc_normal_(self.relative_pos_key_table, std=0.02) + if rel_value: + self.relative_pos_value_table = nn.Parameter( + torch.zeros(2 * quant_grid_length, num_heads, head_dim, 3) + ) + trunc_normal_(self.relative_pos_value_table, std=0.02) + + self.quant_grid_length = quant_grid_length + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop, inplace=True) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop, inplace=True) + + self.softmax = nn.Softmax(dim=-1) + + # def forward(self, feats, xyz, index_0, index_1): + def forward(self, feats, xyz, index_0, index_1, index_0_offsets, n_max): + """Forward function. + + Args: + feats: N, C + xyz: N, 3 + index_0: M, + index_1: M, + """ + + N, C = feats.shape + M = index_0.shape[0] + + assert index_0.shape[0] == index_1.shape[0] + + # Query, Key, Value + qkv = ( + self.qkv(feats) + .reshape(N, 3, self.num_heads, C // self.num_heads) + .permute(1, 0, 2, 3) + .contiguous() + ) + query, key, value = qkv[0], qkv[1], qkv[2] # [N, num_heads, C//num_heads] + query = query * self.scale + attn_flat = pointops.attention_step1_v2( + query.float(), key.float(), index_1.int(), index_0_offsets.int(), n_max + ) + + # # Position embedding + relative_position = xyz[index_0] - xyz[index_1] + relative_position = torch.round(relative_position * 100000) / 100000 + relative_position_index = ( + relative_position + 2 * self.window_size - 0.0001 + ) // self.quant_size + assert (relative_position_index >= 0).all() + assert (relative_position_index <= 2 * self.quant_grid_length - 1).all() + + assert self.rel_query and self.rel_key + if self.rel_query and self.rel_key: + relative_position_bias = pointops.dot_prod_with_idx_v3( + query.float(), + index_0_offsets.int(), + n_max, + key.float(), + index_1.int(), + self.relative_pos_query_table.float(), + self.relative_pos_key_table.float(), + relative_position_index.int(), + ) + elif self.rel_query: + relative_position_bias = pointops.dot_prod_with_idx( + query.float(), + index_0.int(), + self.relative_pos_query_table.float(), + relative_position_index.int(), + ) # [M, num_heads] + elif self.rel_key: + relative_position_bias = pointops.dot_prod_with_idx( + key.float(), + index_1.int(), + self.relative_pos_key_table.float(), + relative_position_index.int(), + ) # [M, num_heads] + else: + relative_position_bias = 0.0 + + attn_flat = attn_flat + relative_position_bias # [M, num_heads] + + softmax_attn_flat = scatter_softmax( + src=attn_flat, index=index_0, dim=0 + ) # [M, num_heads] + + if self.rel_value: + x = pointops.attention_step2_with_rel_pos_value_v2( + softmax_attn_flat.float(), + value.float(), + index_0_offsets.int(), + n_max, + index_1.int(), + self.relative_pos_value_table.float(), + relative_position_index.int(), + ) + else: + x = pointops.attention_step2( + softmax_attn_flat.float(), value.float(), index_0.int(), index_1.int() + ) + + x = x.view(N, C) + + x = self.proj(x) + x = self.proj_drop(x) # [N, C] + + return x + + +class SwinTransformerBlock(nn.Module): + def __init__( + self, + dim, + num_heads, + window_size, + quant_size, + rel_query=True, + rel_key=False, + rel_value=False, + drop_path=0.0, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + mode=4, + ): # mode=4:mean + super().__init__() + self.mode = mode + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, + window_size, + num_heads=num_heads, + quant_size=quant_size, + rel_query=rel_query, + rel_key=rel_key, + rel_value=rel_value, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + ) + + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer + ) + + def forward(self, feats, xyz, index_0, index_1, index_0_offsets, n_max): + # feats: [N, c] + # pos: [N, 3] + + short_cut = feats + + feats = self.norm1(feats) + + feats = self.attn( + feats, xyz, index_0, index_1, index_0_offsets, n_max + ) # index_0 MUST be in ascending order + + feats = short_cut + self.drop_path(feats) + feats = feats + self.drop_path(self.mlp(self.norm2(feats))) + + return feats + + +class BasicLayer(nn.Module): + def __init__( + self, + downsample_scale, + depth, + channel, + num_heads, + window_size, + grid_size, + quant_size, + rel_query=True, + rel_key=False, + rel_value=False, + drop_path=0.0, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + norm_layer=nn.LayerNorm, + downsample=None, + ratio=0.25, + k=16, + out_channels=None, + ): + super().__init__() + self.depth = depth + self.grid_size = grid_size + self.max_window_counts = 64 + self.window_size = window_size + self.downsample_scale = downsample_scale + + self.blocks = nn.ModuleList( + [ + SwinTransformerBlock( + channel, + num_heads, + window_size, + quant_size, + rel_query=rel_query, + rel_key=rel_key, + rel_value=rel_value, + drop_path=( + drop_path[i] if isinstance(drop_path, list) else drop_path + ), + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + norm_layer=norm_layer, + ) + for i in range(depth) + ] + ) + + self.downsample = ( + downsample(channel, out_channels, ratio, k) if downsample else None + ) + + def forward(self, feats, xyz, offset): + # feats: N, C + # xyz: N, 3 + + window_size = torch.tensor([self.window_size] * 3).type_as(xyz).to(xyz.device) + + offset_ = offset.clone() + offset_[1:] = offset_[1:] - offset_[:-1] + batch = ( + torch.cat([torch.tensor([ii] * o) for ii, o in enumerate(offset_)], 0) + .long() + .cuda() + ) + + v2p_map, p2v_map, counts = grid_sample(xyz, batch, window_size, start=None) + + shift_size = 1 / 2 * window_size + shift_v2p_map, shift_p2v_map, shift_counts = grid_sample( + xyz + shift_size, batch, window_size, start=xyz.min(0)[0] + ) + + downsample_scale = self.downsample_scale + new_offset, count = [offset[0].item() // downsample_scale + 1], offset[ + 0 + ].item() // downsample_scale + 1 + for i in range(1, offset.shape[0]): + count += (offset[i].item() - offset[i - 1].item()) // downsample_scale + 1 + new_offset.append(count) + + new_offset = torch.cuda.IntTensor(new_offset) + downsample_idx = pointops.furthestsampling( + xyz, offset.int(), new_offset.int() + ) # [N/16,] + + new_window_size = 2 * torch.tensor([self.window_size] * 3).type_as(xyz).to( + xyz.device + ) + + # offset_ = new_offset.clone() + # offset_[1:] = offset_[1:] - offset_[:-1] + # new_batch = torch.cat([torch.tensor([ii]*o) for ii,o in enumerate(offset_)], 0).long().cuda() + + new_v2p_map, new_p2v_map, new_counts = grid_sample( + xyz, batch, new_window_size, start=None + ) + + shift_size = 1 / 2 * new_window_size + shift_new_v2p_map, shift_new_p2v_map, shift_new_counts = grid_sample( + xyz + shift_size, batch, new_window_size, start=xyz.min(0)[0] + ) + + for i, blk in enumerate(self.blocks): + p2v_map_blk = p2v_map if i % 2 == 0 else shift_p2v_map + counts_blk = counts if i % 2 == 0 else shift_counts + + new_p2v_map_blk = new_p2v_map if i % 2 == 0 else shift_new_p2v_map + new_counts_blk = new_counts if i % 2 == 0 else shift_new_counts + + index_0, index_1 = get_indice_pairs( + p2v_map_blk, + counts_blk, + new_p2v_map_blk, + new_counts_blk, + downsample_idx, + batch, + xyz, + window_size, + i, + ) + + # rearrange index for acceleration + index_0, indices = torch.sort(index_0) # [M,] + index_1 = index_1[indices] # [M,] + index_0_counts = index_0.bincount() + n_max = index_0_counts.max() + index_0_offsets = index_0_counts.cumsum(dim=-1) # [N] + index_0_offsets = torch.cat( + [torch.zeros(1, dtype=torch.long).cuda(), index_0_offsets], 0 + ) # [N+1] + + feats = blk(feats, xyz, index_0, index_1, index_0_offsets, n_max) + + if self.downsample: + feats_down, xyz_down, offset_down = self.downsample(feats, xyz, offset) + else: + feats_down, xyz_down, offset_down = None, None, None + + return feats, xyz, offset, feats_down, xyz_down, offset_down + + +class Upsample(nn.Module): + def __init__(self, k, in_channels, out_channels, bn_momentum=0.02): + super().__init__() + self.k = k + self.in_channels = in_channels + self.out_channels = out_channels + + self.linear1 = nn.Sequential( + nn.LayerNorm(out_channels), nn.Linear(out_channels, out_channels) + ) + self.linear2 = nn.Sequential( + nn.LayerNorm(in_channels), nn.Linear(in_channels, out_channels) + ) + + def forward( + self, feats, xyz, support_xyz, offset, support_offset, support_feats=None + ): + feats = self.linear1(support_feats) + pointops.interpolation( + xyz, support_xyz, self.linear2(feats), offset, support_offset + ) + return feats, support_xyz, support_offset + + +class KPConvSimpleBlock(nn.Module): + def __init__( + self, + in_channels, + out_channels, + prev_grid_size, + sigma=1.0, + negative_slope=0.2, + bn_momentum=0.02, + ): + super().__init__() + self.kpconv = KPConvLayer( + in_channels, + out_channels, + point_influence=prev_grid_size * sigma, + add_one=False, + ) + self.bn = FastBatchNorm1d(out_channels, momentum=bn_momentum) + self.activation = nn.LeakyReLU(negative_slope=negative_slope) + + def forward(self, feats, xyz, batch, neighbor_idx): + # feats: [N, C] + # xyz: [N, 3] + # batch: [N,] + # neighbor_idx: [N, M] + + feats = self.kpconv(xyz, xyz, neighbor_idx, feats) + feats = self.activation(self.bn(feats)) + return feats + + +class KPConvResBlock(nn.Module): + def __init__( + self, + in_channels, + out_channels, + prev_grid_size, + sigma=1.0, + negative_slope=0.2, + bn_momentum=0.02, + ): + super().__init__() + d_2 = out_channels // 4 + activation = nn.LeakyReLU(negative_slope=negative_slope) + self.unary_1 = torch.nn.Sequential( + nn.Linear(in_channels, d_2, bias=False), + FastBatchNorm1d(d_2, momentum=bn_momentum), + activation, + ) + self.unary_2 = torch.nn.Sequential( + nn.Linear(d_2, out_channels, bias=False), + FastBatchNorm1d(out_channels, momentum=bn_momentum), + activation, + ) + self.kpconv = KPConvLayer( + d_2, d_2, point_influence=prev_grid_size * sigma, add_one=False + ) + self.bn = FastBatchNorm1d(out_channels, momentum=bn_momentum) + self.activation = activation + + if in_channels != out_channels: + self.shortcut_op = torch.nn.Sequential( + nn.Linear(in_channels, out_channels, bias=False), + FastBatchNorm1d(out_channels, momentum=bn_momentum), + ) + else: + self.shortcut_op = nn.Identity() + + def forward(self, feats, xyz, batch, neighbor_idx): + # feats: [N, C] + # xyz: [N, 3] + # batch: [N,] + # neighbor_idx: [N, M] + + shortcut = feats + feats = self.unary_1(feats) + feats = self.kpconv(xyz, xyz, neighbor_idx, feats) + feats = self.unary_2(feats) + shortcut = self.shortcut_op(shortcut) + feats += shortcut + return feats + + +@MODELS.register_module("ST-v1m1") +class StratifiedTransformer(nn.Module): + def __init__( + self, + downsample_scale, + depths, + channels, + num_heads, + window_size, + up_k, + grid_sizes, + quant_sizes, + rel_query=True, + rel_key=False, + rel_value=False, + drop_path_rate=0.2, + num_layers=4, + concat_xyz=False, + num_classes=13, + ratio=0.25, + k=16, + prev_grid_size=0.04, + sigma=1.0, + stem_transformer=False, + kp_ball_radius=0.02 * 2.5, + kp_max_neighbor=34, + ): + super().__init__() + assert ( + KPConvLayer is not None and FastBatchNorm1d is not None + ), "Please make sure torch_points3d is installed" + assert tp is not None, "Please make sure torch_points_kernels is installed" + assert pointops is not None, "Please make sure pointops2 is installed" + + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] # stochastic depth decay rule + + self.kp_ball_radius = kp_ball_radius + self.kp_max_neighbor = kp_max_neighbor + if stem_transformer: + self.stem_layer = nn.ModuleList( + [ + KPConvSimpleBlock( + 3 if not concat_xyz else 6, + channels[0], + prev_grid_size, + sigma=sigma, + ) + ] + ) + self.layer_start = 0 + else: + self.stem_layer = nn.ModuleList( + [ + KPConvSimpleBlock( + 3 if not concat_xyz else 6, + channels[0], + prev_grid_size, + sigma=sigma, + ), + KPConvResBlock( + channels[0], channels[0], prev_grid_size, sigma=sigma + ), + ] + ) + self.downsample = TransitionDown(channels[0], channels[1], ratio, k) + self.layer_start = 1 + + self.layers = nn.ModuleList( + [ + BasicLayer( + downsample_scale, + depths[i], + channels[i], + num_heads[i], + window_size[i], + grid_sizes[i], + quant_sizes[i], + rel_query=rel_query, + rel_key=rel_key, + rel_value=rel_value, + drop_path=dpr[sum(depths[:i]) : sum(depths[: i + 1])], + downsample=TransitionDown if i < num_layers - 1 else None, + ratio=ratio, + k=k, + out_channels=channels[i + 1] if i < num_layers - 1 else None, + ) + for i in range(self.layer_start, num_layers) + ] + ) + + self.upsamples = nn.ModuleList( + [ + Upsample(up_k, channels[i], channels[i - 1]) + for i in range(num_layers - 1, 0, -1) + ] + ) + + self.classifier = nn.Sequential( + nn.Linear(channels[0], channels[0]), + nn.BatchNorm1d(channels[0]), + nn.ReLU(inplace=True), + nn.Linear(channels[0], num_classes), + ) + + self.init_weights() + + def forward(self, data_dict): + feats = data_dict["feat"] + xyz = data_dict["coord"] + offset = data_dict["offset"].int() + batch = offset2batch(offset) + neighbor_idx = tp.ball_query( + self.kp_ball_radius, + self.kp_max_neighbor, + xyz, + xyz, + mode="partial_dense", + batch_x=batch, + batch_y=batch, + )[0] + + feats_stack = [] + xyz_stack = [] + offset_stack = [] + + for i, layer in enumerate(self.stem_layer): + feats = layer(feats, xyz, batch, neighbor_idx) + + feats = feats.contiguous() + + if self.layer_start == 1: + feats_stack.append(feats) + xyz_stack.append(xyz) + offset_stack.append(offset) + feats, xyz, offset = self.downsample(feats, xyz, offset) + + for i, layer in enumerate(self.layers): + feats, xyz, offset, feats_down, xyz_down, offset_down = layer( + feats, xyz, offset + ) + + feats_stack.append(feats) + xyz_stack.append(xyz) + offset_stack.append(offset) + + feats = feats_down + xyz = xyz_down + offset = offset_down + + feats = feats_stack.pop() + xyz = xyz_stack.pop() + offset = offset_stack.pop() + + for i, upsample in enumerate(self.upsamples): + feats, xyz, offset = upsample( + feats, + xyz, + xyz_stack.pop(), + offset, + offset_stack.pop(), + support_feats=feats_stack.pop(), + ) + + out = self.classifier(feats) + + return out + + def init_weights(self): + """Initialize the weights in backbone.""" + + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm) or isinstance(m, nn.BatchNorm1d): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + self.apply(_init_weights) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/stratified_transformer/stratified_transformer_v1m2_refine.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/stratified_transformer/stratified_transformer_v1m2_refine.py new file mode 100644 index 0000000000000000000000000000000000000000..234afc12a7be6ea1feb87259c8c77e1bf0a8b3d3 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/stratified_transformer/stratified_transformer_v1m2_refine.py @@ -0,0 +1,763 @@ +""" +Stratified Transformer + +Modified from https://github.com/dvlab-research/Stratified-Transformer + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from copy import deepcopy +import torch +import torch.nn as nn + +try: + import torch_points_kernels as tp +except ImportError: + tp = None + +try: + from torch_points3d.modules.KPConv.kernels import KPConvLayer + from torch_points3d.core.common_modules import FastBatchNorm1d +except ImportError: + KPConvLayer = None + FastBatchNorm1d = None + +from torch_scatter import scatter_softmax +from timm.models.layers import DropPath, trunc_normal_ +from torch_geometric.nn.pool import voxel_grid + +try: + import pointops2.pointops as pointops +except ImportError: + pointops = None + +from pointcept.models.builder import MODELS + + +def offset2batch(offset): + return ( + torch.cat( + [ + ( + torch.tensor([i] * (o - offset[i - 1])) + if i > 0 + else torch.tensor([i] * o) + ) + for i, o in enumerate(offset) + ], + dim=0, + ) + .long() + .to(offset.device) + ) + + +def grid_sample(coords, batch, size, start, return_p2v=True): + cluster = voxel_grid(coords, batch, size, start=start) + + if not return_p2v: + unique, cluster = torch.unique(cluster, sorted=True, return_inverse=True) + return cluster + else: + unique, cluster, counts = torch.unique( + cluster, sorted=True, return_inverse=True, return_counts=True + ) + + # obtain p2v_map + n = unique.shape[0] + k = counts.max().item() + p2v_map = cluster.new_zeros(n, k) + mask = torch.arange(k).cuda().unsqueeze(0) < counts.unsqueeze(-1) + p2v_map[mask] = torch.argsort(cluster) + return cluster, p2v_map, counts + + +class WindowAttention(nn.Module): + """Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + """ + + def __init__( + self, + embed_channels, + num_heads, + window_size, + quant_size, + attn_drop=0.0, + proj_drop=0.0, + scale=None, + rel_query=True, + rel_key=True, + rel_value=True, + qkv_bias=True, + ): + super().__init__() + self.embed_channels = embed_channels + self.head_channels = embed_channels // num_heads + self.num_heads = num_heads + self.scale = scale or self.head_channels**-0.5 + + self.window_size = window_size + self.quant_size = quant_size + + self.rel_query = rel_query + self.rel_key = rel_key + self.rel_value = rel_value + + self.quant_grid_length = int((2 * window_size + 1e-4) // quant_size) + + assert self.rel_query and self.rel_key + if rel_query: + self.relative_pos_query_table = nn.Parameter( + torch.zeros( + 2 * self.quant_grid_length, self.num_heads, self.head_channels, 3 + ) + ) + trunc_normal_(self.relative_pos_query_table, std=0.02) + + if rel_key: + self.relative_pos_key_table = nn.Parameter( + torch.zeros( + 2 * self.quant_grid_length, self.num_heads, self.head_channels, 3 + ) + ) + trunc_normal_(self.relative_pos_query_table, std=0.02) + + if rel_value: + self.relative_pos_value_table = nn.Parameter( + torch.zeros( + 2 * self.quant_grid_length, self.num_heads, self.head_channels, 3 + ) + ) + trunc_normal_(self.relative_pos_query_table, std=0.02) + + self.qkv = nn.Linear(embed_channels, embed_channels * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop, inplace=True) + self.proj = nn.Linear(embed_channels, embed_channels) + self.proj_drop = nn.Dropout(proj_drop, inplace=True) + + self.softmax = nn.Softmax(dim=-1) + + def forward(self, feats, coords, index_0, index_1, index_0_offsets, n_max): + n, c = feats.shape + m = index_0.shape[0] + + assert index_0.shape[0] == index_1.shape[0] + + qkv = ( + self.qkv(feats) + .reshape(n, 3, self.num_heads, c // self.num_heads) + .permute(1, 0, 2, 3) + .contiguous() + ) + query, key, value = qkv[0], qkv[1], qkv[2] + query = query * self.scale + attn_flat = pointops.attention_step1_v2( + query.float(), key.float(), index_1.int(), index_0_offsets.int(), n_max + ) + + # Position embedding + relative_position = coords[index_0] - coords[index_1] + relative_position = torch.round(relative_position * 100000) / 100000 + relative_position_index = torch.div( + relative_position + 2 * self.window_size - 1e-4, + self.quant_size, + rounding_mode="trunc", + ) + # relative_position_index = (relative_position + 2 * self.window_size - 1e-4) // self.quant_size + assert (relative_position_index >= 0).all() + assert (relative_position_index <= 2 * self.quant_grid_length - 1).all() + + if self.rel_query and self.rel_key: + relative_position_bias = pointops.dot_prod_with_idx_v3( + query.float(), + index_0_offsets.int(), + n_max, + key.float(), + index_1.int(), + self.relative_pos_query_table.float(), + self.relative_pos_key_table.float(), + relative_position_index.int(), + ) + elif self.rel_query: + relative_position_bias = pointops.dot_prod_with_idx( + query.float(), + index_0.int(), + self.relative_pos_query_table.float(), + relative_position_index.int(), + ) # [M, num_heads] + elif self.rel_key: + relative_position_bias = pointops.dot_prod_with_idx( + key.float(), + index_1.int(), + self.relative_pos_key_table.float(), + relative_position_index.int(), + ) # [M, num_heads] + else: + relative_position_bias = 0.0 + + attn_flat += relative_position_bias + softmax_attn_flat = scatter_softmax(src=attn_flat, index=index_0, dim=0) + + if self.rel_value: + x = pointops.attention_step2_with_rel_pos_value_v2( + softmax_attn_flat.float(), + value.float(), + index_0_offsets.int(), + n_max, + index_1.int(), + self.relative_pos_value_table.float(), + relative_position_index.int(), + ) + else: + x = pointops.attention_step2( + softmax_attn_flat.float(), value.float(), index_0.int(), index_1.int() + ) + + x = x.view(n, c) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class MLP(nn.Module): + def __init__(self, in_channels, hidden_channels=None, out_channels=None, drop=0.0): + super().__init__() + out_channels = out_channels or in_channels + hidden_channels = hidden_channels or in_channels + self.fc1 = nn.Linear(in_channels, hidden_channels) + self.act = nn.GELU() + self.fc2 = nn.Linear(hidden_channels, out_channels) + self.drop = nn.Dropout(drop, inplace=True) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class Block(nn.Module): + def __init__( + self, + embed_channels, + num_heads, + window_size, + quant_size, + mlp_expend_ratio=4.0, + drop_path=0.0, + qk_scale=None, + rel_query=True, + rel_key=True, + rel_value=True, + qkv_bias=True, + ): + super().__init__() + self.norm1 = nn.LayerNorm(embed_channels) + self.attn = WindowAttention( + embed_channels, + num_heads, + window_size, + quant_size, + scale=qk_scale, + rel_query=rel_query, + rel_key=rel_key, + rel_value=rel_value, + qkv_bias=qkv_bias, + ) + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = nn.LayerNorm(embed_channels) + self.mlp = MLP( + in_channels=embed_channels, + hidden_channels=int(embed_channels * mlp_expend_ratio), + ) + + def forward(self, feats, coords, index_0, index_1, index_0_offsets, n_max): + short_cut = feats + feats = self.norm1(feats) + feats = self.attn(feats, coords, index_0, index_1, index_0_offsets, n_max) + + feats = short_cut + self.drop_path(feats) + feats += self.drop_path(self.mlp(self.norm2(feats))) + return feats + + +class BasicLayer(nn.Module): + def __init__( + self, + embed_channels, + out_channels, + depth, + num_heads, + window_size, + quant_size, + mlp_expend_ratio=4.0, + down_ratio=0.25, + down_num_sample=16, + drop_path=None, + qk_scale=None, + down=True, + rel_query=True, + rel_key=True, + rel_value=True, + qkv_bias=True, + ): + super().__init__() + self.depth = depth + self.window_size = window_size + self.quant_size = quant_size + self.down_ratio = down_ratio + + if isinstance(drop_path, list): + drop_path = drop_path + assert len(drop_path) == depth + elif isinstance(drop_path, float): + drop_path = [deepcopy(drop_path) for _ in range(depth)] + else: + drop_path = [0.0 for _ in range(depth)] + + self.blocks = nn.ModuleList() + for i in range(depth): + block = Block( + embed_channels, + num_heads, + window_size, + quant_size, + mlp_expend_ratio=mlp_expend_ratio, + drop_path=drop_path[i], + qk_scale=qk_scale, + rel_query=rel_query, + rel_key=rel_key, + rel_value=rel_value, + qkv_bias=qkv_bias, + ) + self.blocks.append(block) + + self.down = ( + TransitionDown(embed_channels, out_channels, down_ratio, down_num_sample) + if down + else None + ) + + def forward(self, feats, coords, offset): + # window_size -> [window_size, window_size, window_size] + window_size = torch.tensor( + [self.window_size] * 3, dtype=coords.dtype, device=coords.device + ) + new_window_size = 2 * torch.tensor( + [self.window_size] * 3, dtype=coords.dtype, device=coords.device + ) + batch = offset2batch(offset) + + # compute new offset + new_offset = [int(offset[0].item() * self.down_ratio) + 1] + count = int(offset[0].item() * self.down_ratio) + 1 + for i in range(1, offset.shape[0]): + count += ( + int((offset[i].item() - offset[i - 1].item()) * self.down_ratio) + 1 + ) + new_offset.append(count) + new_offset = torch.cuda.IntTensor(new_offset) + down_idx = pointops.furthestsampling(coords, offset.int(), new_offset.int()) + + # compute window mapping + coords_min = coords.min(0).values + v2p_map, p2v_map, counts = grid_sample(coords, batch, window_size, start=None) + shift_size = window_size * 1 / 2 + shift_v2p_map, shift_p2v_map, shift_counts = grid_sample( + coords + shift_size, batch, window_size, start=coords_min + ) + + new_v2p_map, new_p2v_map, new_counts = grid_sample( + coords, batch, new_window_size, start=None + ) + shift_size = new_window_size * 1 / 2 + shift_new_v2p_map, shift_new_p2v_map, shift_new_counts = grid_sample( + coords + shift_size, batch, new_window_size, start=coords_min + ) + + # stratified attention + for i, blk in enumerate(self.blocks): + p2v_map_blk = p2v_map if i % 2 == 0 else shift_p2v_map + counts_blk = counts if i % 2 == 0 else shift_counts + + new_p2v_map_blk = new_p2v_map if i % 2 == 0 else shift_new_p2v_map + new_counts_blk = new_counts if i % 2 == 0 else shift_new_counts + + n, k = p2v_map_blk.shape + mask = torch.arange(k).unsqueeze(0).cuda() < counts_blk.unsqueeze(-1) + mask_mat = mask.unsqueeze(-1) & mask.unsqueeze(-2) + index_0 = p2v_map_blk.unsqueeze(-1).expand(-1, -1, k)[mask_mat] + index_1 = p2v_map_blk.unsqueeze(1).expand(-1, k, -1)[mask_mat] + + down_mask = torch.zeros_like(batch).bool() + down_mask[down_idx.long()] = True + down_mask = down_mask[new_p2v_map_blk] # [n, k], down sample mask + n, k = new_p2v_map_blk.shape + mask = torch.arange(k).unsqueeze(0).cuda() < new_counts_blk.unsqueeze( + -1 + ) # [n, k] + down_mask = down_mask & mask # down sample and window mask + # [n, k, k] query: dense point in large windows; key: sparse point in large windows + mask_mat = mask.unsqueeze(-1) & down_mask.unsqueeze(-2) + + if i % 2 == 0: + # [n, k, 3] + # window_coord = (coords[new_p2v_map_blk] - coords_min) // window_size + window_coord = torch.div( + coords[new_p2v_map_blk] - coords_min, + window_size, + rounding_mode="trunc", + ) + else: + # [n, k, 3] + # window_coord = (coords[new_p2v_map_blk] - coords_min + 1/2 * window_size) // window_size + window_coord = torch.div( + coords[new_p2v_map_blk] - coords_min + 1 / 2 * window_size, + window_size, + rounding_mode="trunc", + ) + # [n, k, k], whether pair points are in same small windows + mask_mat_prev = ( + window_coord.unsqueeze(2) != window_coord.unsqueeze(1) + ).any(-1) + mask_mat = mask_mat & mask_mat_prev + + new_index_0 = new_p2v_map_blk.unsqueeze(-1).expand(-1, -1, k)[mask_mat] + new_index_1 = new_p2v_map_blk.unsqueeze(1).expand(-1, k, -1)[mask_mat] + + index_0 = torch.cat([index_0, new_index_0], 0) + index_1 = torch.cat([index_1, new_index_1], 0) + + # rearrange index for acceleration + index_0, indices = torch.sort(index_0) + index_1 = index_1[indices] + index_0_counts = index_0.bincount() + n_max = index_0_counts.max() + index_0_offsets = index_0_counts.cumsum(dim=-1) + index_0_offsets = torch.cat( + [torch.zeros(1, dtype=torch.long).cuda(), index_0_offsets], 0 + ) + + feats = blk(feats, coords, index_0, index_1, index_0_offsets, n_max) + + if self.down: + feats_down, coords_down, offset_down = self.down(feats, coords, offset) + else: + feats_down, coords_down, offset_down = None, None, None + + return feats, coords, offset, feats_down, coords_down, offset_down + + +class TransitionDown(nn.Module): + def __init__(self, in_channels, out_channels, ratio, k, norm_layer=nn.LayerNorm): + super().__init__() + self.ratio = ratio + self.k = k + self.norm = norm_layer(in_channels) if norm_layer else None + self.linear = nn.Linear(in_channels, out_channels, bias=False) + self.pool = nn.MaxPool1d(k) + + def forward(self, feats, coords, offset): + new_offset, count = [int(offset[0].item() * self.ratio) + 1], int( + offset[0].item() * self.ratio + ) + 1 + for i in range(1, offset.shape[0]): + count += ((offset[i].item() - offset[i - 1].item()) * self.ratio) + 1 + new_offset.append(count) + new_offset = torch.cuda.IntTensor(new_offset) + idx = pointops.furthestsampling(coords, offset, new_offset) # (m) + new_coords = coords[idx.long(), :] # (m, 3) + + feats = pointops.queryandgroup( + self.k, coords, new_coords, feats, None, offset, new_offset, use_xyz=False + ) # (m, nsample, 3+c) + m, k, c = feats.shape + feats = ( + self.linear(self.norm(feats.view(m * k, c)).view(m, k, c)) + .transpose(1, 2) + .contiguous() + ) + feats = self.pool(feats).squeeze(-1) # (m, c) + return feats, new_coords, new_offset + + +class TransitionUp(nn.Module): + def __init__(self, in_channels, out_channels): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + + self.linear1 = nn.Sequential( + nn.LayerNorm(out_channels), nn.Linear(out_channels, out_channels) + ) + + self.linear2 = nn.Sequential( + nn.LayerNorm(in_channels), nn.Linear(in_channels, out_channels) + ) + + def forward(self, feats, coords, offset, skip_feats, skip_coords, skip_offset): + feats = self.linear1(skip_feats) + pointops.interpolation( + coords, skip_coords, self.linear2(feats), offset, skip_offset + ) + return feats, skip_coords, skip_offset + + +class KPConvSimpleBlock(nn.Module): + def __init__( + self, + in_channels, + out_channels, + prev_grid_size, + sigma=1.0, + negative_slope=0.2, + bn_momentum=0.02, + ): + super().__init__() + self.kpconv = KPConvLayer( + in_channels, + out_channels, + point_influence=prev_grid_size * sigma, + add_one=False, + ) + self.bn = FastBatchNorm1d(out_channels, momentum=bn_momentum) + self.activation = nn.LeakyReLU(negative_slope=negative_slope) + + def forward(self, feats, xyz, batch, neighbor_idx): + # feats: [N, C] + # coords: [N, 3] + # batch: [N,] + # neighbor_idx: [N, M] + + feats = self.kpconv(xyz, xyz, neighbor_idx, feats) + feats = self.activation(self.bn(feats)) + return feats + + +class KPConvResBlock(nn.Module): + def __init__( + self, + in_channels, + out_channels, + prev_grid_size, + sigma=1.0, + negative_slope=0.2, + bn_momentum=0.02, + ): + super().__init__() + d_2 = out_channels // 4 + activation = nn.LeakyReLU(negative_slope=negative_slope) + self.unary_1 = torch.nn.Sequential( + nn.Linear(in_channels, d_2, bias=False), + FastBatchNorm1d(d_2, momentum=bn_momentum), + activation, + ) + self.unary_2 = torch.nn.Sequential( + nn.Linear(d_2, out_channels, bias=False), + FastBatchNorm1d(out_channels, momentum=bn_momentum), + activation, + ) + self.kpconv = KPConvLayer( + d_2, d_2, point_influence=prev_grid_size * sigma, add_one=False + ) + self.bn = FastBatchNorm1d(out_channels, momentum=bn_momentum) + self.activation = activation + + if in_channels != out_channels: + self.shortcut_op = torch.nn.Sequential( + nn.Linear(in_channels, out_channels, bias=False), + FastBatchNorm1d(out_channels, momentum=bn_momentum), + ) + else: + self.shortcut_op = nn.Identity() + + def forward(self, feats, xyz, batch, neighbor_idx): + # feats: [N, C] + # coords: [N, 3] + # batch: [N,] + # neighbor_idx: [N, M] + + shortcut = feats + feats = self.unary_1(feats) + feats = self.kpconv(xyz, xyz, neighbor_idx, feats) + feats = self.unary_2(feats) + shortcut = self.shortcut_op(shortcut) + feats += shortcut + return feats + + +@MODELS.register_module("ST-v1m2") +class StratifiedTransformer(nn.Module): + def __init__( + self, + in_channels, + num_classes, + channels=(48, 96, 192, 384, 384), + num_heads=(6, 12, 24, 24), + depths=(3, 9, 3, 3), + window_size=(0.2, 0.4, 0.8, 1.6), + quant_size=(0.01, 0.02, 0.04, 0.08), + mlp_expend_ratio=4.0, + down_ratio=0.25, + down_num_sample=16, + kp_ball_radius=2.5 * 0.02, + kp_max_neighbor=34, + kp_grid_size=0.02, + kp_sigma=1.0, + drop_path_rate=0.2, + rel_query=True, + rel_key=True, + rel_value=True, + qkv_bias=True, + stem=True, + ): + super().__init__() + assert ( + KPConvLayer is not None and FastBatchNorm1d is not None + ), "Please make sure torch_points3d is installed" + assert tp is not None, "Please make sure torch_points_kernels is installed" + assert pointops is not None, "Please make sure pointops2 is installed" + # stochastic depth decay rule + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] + self.kp_ball_radius = kp_ball_radius + self.kp_max_neighbor = kp_max_neighbor + self.stem = stem + if stem: + self.point_embed = nn.ModuleList( + [ + KPConvSimpleBlock( + in_channels, channels[0], kp_grid_size, sigma=kp_sigma + ), + KPConvResBlock( + channels[0], channels[0], kp_grid_size, sigma=kp_sigma + ), + ] + ) + self.down = TransitionDown( + channels[0], channels[1], down_ratio, down_num_sample + ) + else: + assert channels[0] == channels[1] + self.point_embed = nn.ModuleList( + [ + KPConvSimpleBlock( + in_channels, channels[1], kp_grid_size, sigma=kp_sigma + ), + ] + ) + + num_layers = len(depths) + self.layers = nn.ModuleList() + for i in range(num_layers): + layer = BasicLayer( + embed_channels=channels[i + 1], + out_channels=channels[i + 2] if i < num_layers - 1 else channels[i + 1], + depth=depths[i], + num_heads=num_heads[i], + window_size=window_size[i], + quant_size=quant_size[i], + mlp_expend_ratio=mlp_expend_ratio, + down_ratio=down_ratio, + down_num_sample=down_num_sample, + drop_path=dpr[sum(depths[:i]) : sum(depths[: i + 1])], + rel_query=rel_query, + rel_key=rel_key, + rel_value=rel_value, + qkv_bias=qkv_bias, + down=True if i < num_layers - 1 else False, + ) + self.layers.append(layer) + + self.up = nn.ModuleList( + [ + TransitionUp(channels[i + 1], channels[i]) + for i in reversed(range(1, num_layers)) + ] + ) + if self.stem: + self.up.append(TransitionUp(channels[1], channels[0])) + + self.classifier = nn.Sequential( + nn.Linear(channels[0], channels[0]), + nn.BatchNorm1d(channels[0]), + nn.ReLU(inplace=True), + nn.Linear(channels[0], num_classes), + ) + + self.init_weights() + + def forward(self, data_dict): + feats = data_dict["feat"] + coords = data_dict["coord"] + offset = data_dict["offset"].int() + batch = offset2batch(offset) + neighbor_idx = tp.ball_query( + self.kp_ball_radius, + self.kp_max_neighbor, + coords, + coords, + mode="partial_dense", + batch_x=batch, + batch_y=batch, + )[0] + + feats_stack = [] + coords_stack = [] + offset_stack = [] + + for i, layer in enumerate(self.point_embed): + feats = layer(feats, coords, batch, neighbor_idx) + + feats = feats.contiguous() + if self.stem: + feats_stack.append(feats) + coords_stack.append(coords) + offset_stack.append(offset) + feats, coords, offset = self.down(feats, coords, offset) + + for i, layer in enumerate(self.layers): + feats, coords, offset, feats_down, coords_down, offset_down = layer( + feats, coords, offset + ) + + feats_stack.append(feats) + coords_stack.append(coords) + offset_stack.append(offset) + + feats = feats_down + coords = coords_down + offset = offset_down + + feats = feats_stack.pop() + coords = coords_stack.pop() + offset = offset_stack.pop() + + for i, up in enumerate(self.up): + feats, coords, offset = up( + feats, + coords, + offset, + feats_stack.pop(), + coords_stack.pop(), + offset_stack.pop(), + ) + + out = self.classifier(feats) + return out + + def init_weights(self): + """Initialize the weights in backbone.""" + + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm) or isinstance(m, nn.BatchNorm1d): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + self.apply(_init_weights) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/swin3d/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/swin3d/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..36050969d9abb027778008e4d6d8f77710f52392 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/swin3d/__init__.py @@ -0,0 +1 @@ +from .swin3d_v1m1_base import Swin3DUNet diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/swin3d/mink_layers.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/swin3d/mink_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..ee3e8cfc002e8311ac196335592c337644659612 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/swin3d/mink_layers.py @@ -0,0 +1,249 @@ +""" +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +""" + +import torch +import torch.nn as nn +import torch.nn.functional as F +import MinkowskiEngine as ME +import numpy as np + + +def assign_feats(sp, x): + return ME.SparseTensor( + features=x.float(), + coordinate_map_key=sp.coordinate_map_key, + coordinate_manager=sp.coordinate_manager, + ) + + +class MinkConvBN(nn.Module): + def __init__( + self, + in_channels, + out_channels, + kernel_size=3, + stride=1, + dilation=1, + bias=False, + dimension=3, + ): + super().__init__() + self.conv_layers = nn.Sequential( + ME.MinkowskiConvolution( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + dilation=dilation, + bias=bias, + dimension=dimension, + ), + ME.MinkowskiBatchNorm(out_channels), + ) + + def forward(self, x): + x = self.conv_layers(x) + return x + + +class MinkConvBNRelu(nn.Module): + def __init__( + self, + in_channels, + out_channels, + kernel_size=3, + stride=1, + dilation=1, + bias=False, + dimension=3, + ): + super().__init__() + self.conv_layers = nn.Sequential( + ME.MinkowskiConvolution( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + dilation=dilation, + bias=bias, + dimension=dimension, + ), + ME.MinkowskiBatchNorm(out_channels), + ME.MinkowskiReLU(inplace=True), + ) + + def forward(self, x): + x = self.conv_layers(x) + if x.F.dtype == torch.float16: + x = assign_feats(x, x.F.float()) + return x + + +class MinkDeConvBNRelu(nn.Module): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride, + dilation=1, + bias=False, + dimension=3, + ): + super().__init__() + self.conv_layers = nn.Sequential( + ME.MinkowskiConvolutionTranspose( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + dilation=dilation, + bias=bias, + dimension=dimension, + ), + ME.MinkowskiBatchNorm(out_channels), + ME.MinkowskiReLU(), + ) + + def forward(self, x): + x = self.conv_layers(x) + return x + + +class MinkResBlock(nn.Module): + def __init__(self, in_channels, out_channels, stride=1, dilation=1): + super(MinkResBlock, self).__init__() + + self.conv1 = ME.MinkowskiConvolution( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=3, + stride=stride, + dilation=dilation, + bias=False, + dimension=3, + ) + self.norm1 = ME.MinkowskiBatchNorm(out_channels) + self.conv2 = ME.MinkowskiConvolution( + in_channels=out_channels, + out_channels=out_channels, + kernel_size=3, + stride=1, + dilation=dilation, + bias=False, + dimension=3, + ) + + self.norm2 = ME.MinkowskiBatchNorm(out_channels) + self.relu = ME.MinkowskiReLU(inplace=True) + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.norm2(out) + + out += residual + out = self.relu(out) + + return out + + +class SparseTensorLinear(nn.Module): + def __init__(self, in_channels, out_channels, bias=False): + super().__init__() + self.linear = nn.Linear(in_channels, out_channels, bias=bias) + + def forward(self, sp): + x = self.linear(sp.F) + return assign_feats(sp, x.float()) + + +class SparseTensorLayerNorm(nn.Module): + def __init__(self, dim): + super().__init__() + self.norm = nn.LayerNorm(dim) + + def forward(self, sp): + x = self.norm(sp.F) + return assign_feats(sp, x.float()) + + +class MinkResBlock_v2(nn.Module): + def __init__(self, in_channels, out_channels): + super().__init__() + d_2 = out_channels // 4 + self.conv1 = torch.nn.Sequential( + SparseTensorLinear(in_channels, d_2, bias=False), + ME.MinkowskiBatchNorm(d_2), + ME.MinkowskiReLU(), + ) + self.unary_2 = torch.nn.Sequential( + SparseTensorLinear(d_2, out_channels, bias=False), + ME.MinkowskiBatchNorm(out_channels), + ME.MinkowskiReLU(), + ) + self.spconv = ME.MinkowskiConvolution( + in_channels=d_2, + out_channels=d_2, + kernel_size=5, + stride=1, + dilation=1, + bias=False, + dimension=3, + ) + if in_channels != out_channels: + self.shortcut_op = torch.nn.Sequential( + SparseTensorLinear(in_channels, out_channels, bias=False), + ME.MinkowskiBatchNorm(out_channels), + ) + else: + self.shortcut_op = nn.Identity() + + def forward(self, x): + # feats: [N, C] + # xyz: [N, 3] + # batch: [N,] + # neighbor_idx: [N, M] + shortcut = x + x = self.unary_1(x) + x = self.spconv(x) + x = self.unary_2(x) + shortcut = self.shortcut_op(shortcut) + x += shortcut + return x + + +class MinkResBlock_BottleNeck(nn.Module): + def __init__(self, in_channels, out_channels): + super(MinkResBlock_BottleNeck, self).__init__() + bottle_neck = out_channels // 4 + self.conv1x1a = MinkConvBNRelu( + in_channels, bottle_neck, kernel_size=1, stride=1 + ) + self.conv3x3 = MinkConvBNRelu(bottle_neck, bottle_neck, kernel_size=3, stride=1) + self.conv1x1b = MinkConvBN(bottle_neck, out_channels, kernel_size=1, stride=1) + if in_channels != out_channels: + self.conv1x1c = MinkConvBN( + in_channels, out_channels, kernel_size=1, stride=1 + ) + else: + self.conv1x1c = None + self.relu = ME.MinkowskiReLU(inplace=True) + + def forward(self, x): + residual = x + out = self.conv1x1a(x) + out = self.conv3x3(out) + out = self.conv1x1b(out) + if self.conv1x1c is not None: + residual = self.conv1x1c(residual) + out = self.relu(out + residual) + + return out diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/swin3d/swin3d_layers.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/swin3d/swin3d_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..e737e9677ae93f8f5f9188ba774fcd1d0fa42443 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/swin3d/swin3d_layers.py @@ -0,0 +1,876 @@ +""" +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +""" + +import numpy as np +import torch +import torch.nn as nn +from timm.models.layers import DropPath, trunc_normal_ +import MinkowskiEngine as ME +from MinkowskiEngine import SparseTensor +from Swin3D.sparse_dl.attn.attn_coff import ( + SelfAttnAIOFunction, + PosEmb, + TableDims, + IndexMode, + PrecisionMode, +) +import Swin3D.sparse_dl.knn +from Swin3D.sparse_dl.knn import KNN + +from .mink_layers import ( + assign_feats, + SparseTensorLayerNorm, + SparseTensorLinear, +) + + +def query_knn_feature( + K, src_xyz, query_xyz, src_feat, src_offset, query_offset, return_idx=False +): + """ + gather feature in the KNN neighborhood + """ + assert ( + src_xyz.is_contiguous() + and query_xyz.is_contiguous() + and src_feat.is_contiguous() + ) + if query_xyz is None: + query_xyz = src_xyz + query_offset = src_offset + + idx, _ = KNN.apply(K, src_xyz, query_xyz, src_offset, query_offset) + + n, m, c = src_xyz.shape[0], query_xyz.shape[0], src_feat.shape[1] + grouped_feat = src_feat[idx.view(-1).long(), :].view(m, K, c) + + if return_idx: + return grouped_feat, idx + else: + return grouped_feat + + +def knn_linear_interpolation( + src_xyz, query_xyz, src_feat, src_offset, query_offset, K=3 +): + """ + interpolation feature using distance in KNN neighborhood + """ + N, C = query_xyz.shape[0], src_feat.shape[1] + assert ( + src_xyz.is_contiguous() + and query_xyz.is_contiguous() + and src_feat.is_contiguous() + ) + # (N, K) + idx, dist = KNN.apply(K, src_xyz, query_xyz, src_offset, query_offset) + weight = 1.0 / (dist + 1e-8) + norm = torch.sum(weight, dim=1, keepdim=True) + weight = weight / norm + query_feat = torch.zeros((N, C), dtype=src_feat.dtype, device=src_feat.device) + for i in range(K): + query_feat += src_feat[idx[:, i].long(), :] * weight[:, i].unsqueeze(-1) + return query_feat + + +def sparse_self_attention( + w_w_id: torch.Tensor, w_sizes: torch.Tensor, protocol: str = "v1" +): + """ + Args: + indices [torch.Tensor]: sparse window index with shape [N, 2], N is the total + number of non-empty voxels with indices (window_id, within_window_id). window_id + is ordered and starts from 0; within_window_id is a sparse index to indicate the + offset of kernel_size ** 3. + feats [torch.Tensor]: sprase features of each non-empty voxel with shape [N, C] + Outputs: + [M, 3]: sparse indices of cofficient matrix (window_id, att_a_id, att_b_id). att_a_id + and att_b_id are the within_window_id + [M, 1]: the sparse coffient matrix + + Spaces: + W: total number of windows + N: total number of input voxels + M: total number of output cofficients + """ + w_sizes_2 = w_sizes**2 + + # w2n_indices - [W], mapping window index to window global offset in input + # space + w_cumsum = torch.cumsum(w_sizes, dim=-1) + w2n_indices = torch.cat( + [torch.zeros(1, dtype=w_cumsum.dtype, device=w_cumsum.device), w_cumsum[:-1]] + ) + + # w2m indices - [W], mapping window index to window global offset in output + # space + w2_cumsum = torch.cumsum(w_sizes_2, dim=-1) + w2m_indices = torch.cat( + [torch.zeros(1, dtype=w2_cumsum.dtype, device=w2_cumsum.device), w2_cumsum[:-1]] + ) + + # m2w indices - [M], mapping element global offset to the window index + m2w_indices = torch.zeros( + [w2_cumsum[-1]], dtype=w_sizes.dtype, device=w_sizes.device + ) + m2w_offset = torch.zeros( + [w2_cumsum[-1]], dtype=w_sizes.dtype, device=w_sizes.device + ) + m2w_indices[w2m_indices[1:]] = 1 + m2w_offset[w2m_indices[1:]] = w_sizes_2[:-1] + m2w_indices = torch.cumsum(m2w_indices, dim=-1) + m2w_offset = torch.cumsum(m2w_offset, dim=-1) + + # m_indices = [M], element global offset in output space + m_indices = torch.arange( + 0, w2_cumsum[-1], dtype=w_sizes.dtype, device=w_sizes.device + ) + + # m2n_indices - [M], mapping element global offset to the window global offset + # in input space + m2n_indices = w2n_indices[m2w_indices] + + m_offset = m_indices - m2w_offset + m2w_sizes = w_sizes[m2w_indices] + + # print_log_main("m_offset:", m_offset, m_offset.shape) + # print_log_main("m2n_indices:", m2n_indices, m2n_indices.shape) + + y_offset = m2n_indices + m_offset % m2w_sizes + x_offset = m2n_indices + torch.div(m_offset, m2w_sizes, rounding_mode="floor") + + # print_log_main("=================================") + # print_log_main(w_sizes[:5]) + # print_log_main(x_offset[:50]) + # print_log_main(y_offset[:50]) + # coord = torch.stack([m2w_indices, w_w_id[x_offset], w_w_id[y_offset]], axis=-1) + if protocol == "v1": + return x_offset, y_offset + elif protocol == "v2": + return x_offset, y_offset, m2w_indices, w_sizes, w2n_indices, w2m_indices + + +class Mlp(nn.Module): + def __init__( + self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.0, + ): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class GridCoordsDown(nn.Module): + """ + downsample the grid coordinates + keep the nearest point to the average point of the downsampled grid + """ + + def __init__(self, stride): + super().__init__() + self.stride = stride + self.avg_pool = ME.MinkowskiAvgPooling( + kernel_size=self.stride, stride=self.stride, dimension=3 + ) + self.unpool = ME.MinkowskiPoolingTranspose( + kernel_size=stride, stride=stride, dimension=3 + ) + self.max_pool = ME.MinkowskiMaxPooling( + kernel_size=self.stride, stride=self.stride, dimension=3 + ) + + def forward(self, coords_sp, sp, return_map=False): + device = sp.C.device + # is_pool = True means pooling map + # is_pool = False means conv map (query as center) + + N = sp.shape[0] + avg_coords_sp = self.avg_pool(coords_sp) + dist_sp = self.unpool(avg_coords_sp) - coords_sp + dist = dist_sp.F + dist = -torch.sqrt((dist**2).sum(dim=1)).unsqueeze(1) + dist_sp = assign_feats(dist_sp, dist) + min_dist_sp = self.max_pool(dist_sp) + map_pair = sp.coordinate_manager.kernel_map( + dist_sp.coordinate_map_key, + min_dist_sp.coordinate_map_key, + stride=self.stride, + kernel_size=self.stride, + is_pool=True, + )[0] + in_map, out_map = map_pair + broad_min_dist_sp = self.unpool(min_dist_sp) + mask = (broad_min_dist_sp.F == dist_sp.F).squeeze(1) + in_map = in_map[mask].long() + out_map = out_map[mask].long() + downsample_map = torch.zeros(N, dtype=torch.long, device=device) - 1 + downsample_map[out_map] = in_map + assert (downsample_map >= 0).all() + assert (dist_sp.F[downsample_map] == min_dist_sp.F).all() + new_coords = coords_sp.F[downsample_map] + new_coords_sp = assign_feats(sp, new_coords) + if return_map: + return new_coords_sp, downsample_map + else: + return new_coords_sp + + +def get_offset(batch): + offset = [] + bs = batch.max() + 1 + for i in range(bs): + offset.append(torch.sum(batch == i)) + offset = torch.cuda.IntTensor(offset) + offset = offset.cumsum(dim=0).int() + return offset + + +class GridDownsample(nn.Module): + """ + use stride to downsample voxel + use grid maxpooling with kernel_size + """ + + def __init__(self, in_channels, out_channels, kernel_size=2, stride=2): + super().__init__() + self.kernel_size = kernel_size + self.stride = stride + self.in_channels = in_channels + self.out_channels = out_channels + self.sp_pool = ME.MinkowskiMaxPooling( + kernel_size=kernel_size, stride=stride, dimension=3 + ) + self.coords_pool = GridCoordsDown(stride=stride) + self.norm = SparseTensorLayerNorm(in_channels) + self.linear = SparseTensorLinear(in_channels, out_channels) + + def forward(self, sp, coords_sp): + sp_down = self.sp_pool(self.linear(self.norm(sp))) + coords_sp_down = self.coords_pool(coords_sp, sp_down) + return sp_down, coords_sp_down + + def extra_repr(self) -> str: + return f"kernel_size={self.kernel_size}, stride={self.stride}, in_channels={self.in_channels}, out_channels={self.out_channels}" + + +class GridKNNDownsample(nn.Module): + """ + use stride to downsample voxel + use KNN to do maxpooling + """ + + def __init__(self, in_channels, out_channels, kernel_size=2, stride=2): + super().__init__() + self.stride = stride + self.in_channels = in_channels + self.out_channels = out_channels + self.k = 16 + self.sp_pool = ME.MinkowskiMaxPooling( + kernel_size=stride, stride=stride, dimension=3 + ) + self.coords_pool = GridCoordsDown(stride=stride) + self.norm = nn.LayerNorm(in_channels) + self.linear = nn.Linear(in_channels, out_channels, bias=False) + self.pool = nn.MaxPool1d(self.k) + + def forward(self, sp, coords_sp): + # calculate the voxel + sp_down = self.sp_pool(sp) + # for downsampled cRSE + coords_sp_down = self.coords_pool(coords_sp, sp_down) + offset = get_offset(sp.C[:, 0]) + n_offset = get_offset(sp_down.C[:, 0]) + + xyz = coords_sp.F[:, 1:4].detach().contiguous() + n_xyz = coords_sp_down.F[:, 1:4].detach().contiguous() + feats = query_knn_feature(self.k, xyz, n_xyz, sp.F, offset, n_offset) + m, k, c = feats.shape + feats = ( + self.linear(self.norm(feats.view(m * k, c)).view(m, k, c)) + .transpose(1, 2) + .contiguous() + ) + feats = self.pool(feats).squeeze(-1) + sp = assign_feats(sp_down, feats.float()) + coords_sp = coords_sp_down + return sp, coords_sp + + def extra_repr(self) -> str: + return f"kernel_size={self.k}, stride={self.stride}, in_channels={self.in_channels}, out_channels={self.out_channels}" + + +class Upsample(nn.Module): + """ + upsample using trilinear interpolation + follower by attn block according to self.attn + """ + + def __init__( + self, + in_channels, + out_channels, + num_heads, + window_size, + quant_size, + attn=True, + up_k=3, + cRSE="XYZ_RGB", + fp16_mode=0, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + + self.linear1 = nn.Sequential( + nn.LayerNorm(out_channels), nn.Linear(out_channels, out_channels) + ) + self.linear2 = nn.Sequential( + nn.LayerNorm(in_channels), nn.Linear(in_channels, out_channels) + ) + self.up_k = up_k + self.attn = attn and window_size > 0 + if self.attn: + self.block = BasicLayer( + dim=out_channels, + depth=1, + num_heads=num_heads, + window_size=window_size, + quant_size=quant_size, + drop_path=0.1, + downsample=None, + out_channels=None, + cRSE=cRSE, + fp16_mode=fp16_mode, + ) + + def forward(self, sp, coords_sp, sp_up, coords_sp_up): + feats = sp.F + support_feats = sp_up.F + xyz = coords_sp.F[:, 1:4].detach().contiguous() + support_xyz = coords_sp_up.F[:, 1:4].detach().contiguous() + offset = get_offset(sp.C[:, 0]) + support_offset = get_offset(sp_up.C[:, 0]) + + feats = self.linear1(support_feats) + knn_linear_interpolation( + xyz, support_xyz, self.linear2(feats), offset, support_offset, K=self.up_k + ) + sp_up = assign_feats(sp_up, feats) + if self.attn: + sp_up, _, _ = self.block(sp_up, coords_sp_up) + return sp_up + + def extra_repr(self) -> str: + return f"up_k={self.up_k}, in_channels={self.in_channels}, out_channels={self.out_channels}, attn={self.attn}" + + +class WindowAttention(nn.Module): + """ + Window based multi-head self attention (W-MSA) module with cRSE. + Designed for sparse structure + It supports both of shifted and non-shifted window. + + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + quant_size (int): quant_size for for finer cRSE table + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + cRSE (str | 'XYZ', 'XYZ_RGB', 'XYZ_RGB_NORM'): cRSE mode. Default: 'XYZ_RGB' + fp16_mode (int | 0, 1, 2): fp16 mode for attention module, Default: 0 + 0: fp32 forward and fp32 backward + 1: fp16 forward and fp32 backward + 2: fp16 forward and fp16 backward + """ + + def __init__( + self, + dim, + window_size, + quant_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + cRSE="XYZ_RGB", + fp16_mode=0, + ): + super().__init__() + self.dim = dim + self.window_size = window_size + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + # color in [-1, 1], color_windowsize = 2 + # normal in [-1, 1], normal_windowsize = 2 + self.color_windowsize = 2 + self.normal_windowsize = 2 + + self.fp16_mode = fp16_mode + + table_offsets = [] + self.cRSE = cRSE + if "XYZ" in cRSE: + self.xyz_quant_size = quant_size + quant_grid_length_xyz = window_size * self.xyz_quant_size + table_shape_xyz = (3, 2 * quant_grid_length_xyz, num_heads, head_dim) + self.query_xyz_table = nn.Parameter(torch.zeros(table_shape_xyz)) + trunc_normal_(self.query_xyz_table, std=0.02) + self.key_xyz_table = nn.Parameter(torch.zeros(table_shape_xyz)) + trunc_normal_(self.key_xyz_table, std=0.02) + self.value_xyz_table = nn.Parameter(torch.zeros(table_shape_xyz)) + trunc_normal_(self.value_xyz_table, std=0.02) + table_offsets += [np.prod(table_shape_xyz[1:])] * 3 + + if "RGB" in cRSE: + self.color_quant_size = quant_size * 2 + quant_grid_length_rgb = self.color_windowsize * self.color_quant_size + table_shape_rgb = (3, 2 * quant_grid_length_rgb, num_heads, head_dim) + self.query_rgb_table = nn.Parameter(torch.zeros(table_shape_rgb)) + trunc_normal_(self.query_rgb_table, std=0.02) + self.key_rgb_table = nn.Parameter(torch.zeros(table_shape_rgb)) + trunc_normal_(self.key_rgb_table, std=0.02) + self.value_rgb_table = nn.Parameter(torch.zeros(table_shape_rgb)) + trunc_normal_(self.value_rgb_table, std=0.02) + table_offsets += [np.prod(table_shape_rgb[1:])] * 3 + + if "NORM" in cRSE: + self.normal_quant_size = quant_size * 2 + quant_grid_length_norm = self.normal_windowsize * self.normal_quant_size + table_shape_norm = (3, 2 * quant_grid_length_norm, num_heads, head_dim) + self.query_norm_table = nn.Parameter(torch.zeros(table_shape_norm)) + trunc_normal_(self.query_norm_table, std=0.02) + self.key_norm_table = nn.Parameter(torch.zeros(table_shape_norm)) + trunc_normal_(self.key_norm_table, std=0.02) + self.value_norm_table = nn.Parameter(torch.zeros(table_shape_norm)) + trunc_normal_(self.value_norm_table, std=0.02) + table_offsets += [np.prod(table_shape_norm[1:])] * 3 + + self.table_offsets = table_offsets + + self.quant_size = quant_size + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop, inplace=True) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop, inplace=True) + + self.softmax = nn.Softmax(dim=-1) + + def forward(self, feats: torch.Tensor, attn_args): + """Forward function. + + Args: + feats: N, C + attn_args: arguments for computing attention + """ + num_v, _ = feats.shape + num_sc = self.dim // self.num_heads + + ( + x_offset, + y_offset, + m2w_indices, + w_sizes, + w2n_indices, + n2n_indices, + w2m_indices, + n_coords, + ) = attn_args + + # Query, Key, Value + qkv = self.qkv(feats) + qkv = ( + qkv.reshape(num_v, 3, self.num_heads, num_sc) + .permute(1, 0, 2, 3) + .contiguous() + ) + query, key, value = qkv[0], qkv[1], qkv[2] # [N, num_heads, C//num_heads] + query = query * self.scale + + table_offsets = torch.IntTensor(self.table_offsets).cuda() + query_table, key_table, value_table = [], [], [] + n_cRSE = [] + if "XYZ" in self.cRSE: + n_xyz = n_coords[:, 0:3] + n_xyz = n_xyz * self.quant_size + n_cRSE.append(n_xyz) + query_table.append(self.query_xyz_table.view(-1)) + key_table.append(self.key_xyz_table.view(-1)) + value_table.append(self.value_xyz_table.view(-1)) + if "RGB" in self.cRSE: + n_rgb = n_coords[:, 3:6] + n_rgb = n_rgb * self.color_quant_size + n_cRSE.append(n_rgb) + query_table.append(self.query_rgb_table.view(-1)) + key_table.append(self.key_rgb_table.view(-1)) + value_table.append(self.value_rgb_table.view(-1)) + if "NORM" in self.cRSE: + n_norm = n_coords[:, 6:9] + n_norm = n_norm * self.normal_quant_size + n_cRSE.append(n_norm) + query_table.append(self.query_norm_table.view(-1)) + key_table.append(self.key_norm_table.view(-1)) + value_table.append(self.value_norm_table.view(-1)) + + n_cRSE = torch.cat(n_cRSE, dim=1) + + indices = [m2w_indices, w_sizes, w2m_indices, w2n_indices, n2n_indices, n_cRSE] + query_table = torch.cat(query_table) + key_table = torch.cat(key_table) + value_table = torch.cat(value_table) + + if self.fp16_mode == 0: + # do not use fp16 + # cast q,k,v to fp32 in forward and backward + fp16_mode = PrecisionMode.HALF_NONE + elif self.fp16_mode == 1: + # use fp16 only in forward + fp16_mode = PrecisionMode.HALF_FORWARD + elif self.fp16_mode == 2: + # use fp16 both in forward and backward + fp16_mode = PrecisionMode.HALF_ALL + + updated_values = SelfAttnAIOFunction.apply( + query, + key, + value, + query_table, + key_table, + value_table, + table_offsets, + indices, + PosEmb.SEPARATE, + TableDims.D0, + IndexMode.INDIRECT, + fp16_mode, + ) + + updated_values = updated_values.flatten(1) + updated_feats = updated_values.view(num_v, self.dim) + + updated_feats = self.proj(updated_feats) + updated_feats = self.proj_drop(updated_feats) # [N, C] + + return updated_feats + + +class SwinTransformerBlock(nn.Module): + def __init__( + self, + dim, + num_heads, + window_size, + quant_size, + drop_path=0.0, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + cRSE="XYZ_RGB", + fp16_mode=0, + ): + super().__init__() + self.window_size = window_size + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, + window_size=self.window_size, + quant_size=quant_size, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + cRSE=cRSE, + fp16_mode=fp16_mode, + ) + + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer + ) + + def forward(self, feats, attn_args): + # feats: [N, c] + short_cut = feats + feats = self.norm1(feats) + feats = self.attn(feats, attn_args) # [N, c] + + feats = short_cut + self.drop_path(feats) + feats = feats + self.drop_path(self.mlp(self.norm2(feats))) + + return feats + + +class BasicLayer(nn.Module): + """A basic Swin3D layer for one stage. + + Args: + dim (int): Number of input channels. + depth (int): Number of blocks. + num_heads (int): Number of attention heads. + window_size (int): Local window size. + quant_size (int): quant_size for for finer cRSE table + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + cRSE (str | 'XYZ', 'XYZ_RGB', 'XYZ_RGB_NORM'): cRSE mode. Default: 'XYZ_RGB' + fp16_mode (int | 0, 1, 2): fp16 mode for attention module, Default: 0 + 0: fp32 forward and fp32 backward + 1: fp16 forward and fp32 backward + 2: fp16 forward and fp16 backward + """ + + def __init__( + self, + dim, + depth, + num_heads, + window_size, + quant_size, + out_channels=None, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop_path=0.0, + norm_layer=nn.LayerNorm, + downsample=None, + down_stride=2, + cRSE="XYZ_RGB", + fp16_mode=0, + ): + super().__init__() + self.window_size = window_size + self.depth = depth + self.dim = dim + self.num_heads = num_heads + self.quant_size = quant_size + self.cRSE = cRSE + self.fp16_mode = fp16_mode + + self.shift_size = window_size // 2 + # build blocks + self.blocks = nn.ModuleList( + [ + SwinTransformerBlock( + dim, + num_heads, + window_size, + quant_size, + drop_path=( + drop_path[i] if isinstance(drop_path, list) else drop_path + ), + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + norm_layer=norm_layer, + cRSE=cRSE, + fp16_mode=fp16_mode, + ) + for i in range(depth) + ] + ) + + self.pool = ME.MinkowskiMaxPooling( + kernel_size=self.window_size, stride=self.window_size, dimension=3 + ) + + if downsample is not None: + if out_channels is None: + out_channels = dim * 2 + self.downsample = downsample( + dim, out_channels, kernel_size=down_stride, stride=down_stride + ) + else: + self.downsample = None + + def get_map_pair(self, sp): + """ + use minkowski pool to calculate windows + get the mapping from voxel to window + """ + window_size = [self.window_size] * 3 + pool_sp = self.pool(sp) + windows = pool_sp.C + window_N = windows.shape[0] + + stride_in = sp.coordinate_map_key.get_tensor_stride() + x, y, z = [ + torch.arange(window_size[i], device=self.device) * stride_in[i] + for i in range(3) + ] + x, y, z = torch.meshgrid(x, y, z) + i = torch.zeros_like(x, device=self.device) + local_window = torch.stack([i, x, y, z], dim=-1).flatten(0, -2) + all_windows = windows.unsqueeze(1) + local_window.unsqueeze(0) + all_windows = all_windows.flatten(0, -2).int() + cm = sp.coordinate_manager + query_key, (map, inverse_map) = cm.insert_and_map( + all_windows, tensor_stride=stride_in + ) + map_pair = cm.kernel_map(query_key, sp.coordinate_map_key, kernel_size=1)[0] + return map_pair, window_N + + def get_window_mapping(self, sp): + """ + calculate the relationshape in the window: + w_w_id: non-empty idx inside the window(sorted by window) + w_w_xyz: xyz inside the window(sorted by window) + nempty_num: non-empty voxel number in each window + sort_idx: sort voxel according to window_id, to gather the point inside the same window + inv_sort_idx: inverse sort index + """ + map_pair, window_N = self.get_map_pair(sp) + window_size = self.window_size + nW = window_size**3 + in_map, out_map = map_pair + in_map, sort_idx = torch.sort(in_map) + # assert out_map == arange(out_map.shape[0]) + out_map = out_map[sort_idx] + sort_idx = out_map.long() + inv_sort_idx = torch.zeros_like(sort_idx) + inv_sort_idx[sort_idx] = torch.arange( + sort_idx.shape[0], dtype=sort_idx.dtype, device=self.device + ) + N = window_N * nW + v2w_mask = torch.zeros(N, dtype=torch.bool, device=self.device) + w_id = ( + torch.arange(window_N, dtype=torch.long, device=self.device) + .unsqueeze(1) + .repeat(1, nW) + .view(-1) + ) + w_w_id = ( + torch.arange(nW, dtype=torch.long, device=self.device) + .unsqueeze(0) + .repeat(window_N, 1) + .view(-1) + ) + v2w_mask[in_map.long()] = True + nempty_num = v2w_mask.view(-1, nW).sum(dim=-1) + w_id = w_id[in_map.long()] + w_w_id = w_w_id[in_map.long()] + w_w_xyz = torch.stack( + [ + w_w_id // window_size // window_size, + w_w_id // window_size % window_size, + w_w_id % window_size, + ], + dim=-1, + ) + return w_w_id, w_w_xyz, nempty_num, sort_idx, inv_sort_idx + + def get_index01(self, sp, local_xyz, colors): + """ + calculate the arguments for sparse attention + """ + ( + w_w_id, + w_w_xyz, + nempty_num, + n2n_indices, + inv_sort_idx, + ) = self.get_window_mapping(sp) + local_xyz = local_xyz[n2n_indices] + colors = colors[n2n_indices] + # recover the relative pos in the voxel + n_coords = w_w_xyz + local_xyz + n_coords = torch.cat([n_coords, colors], dim=1) + ( + x_offset, + y_offset, + m2w_indices, + w_sizes, + w2n_indices, + w2m_indices, + ) = sparse_self_attention(w_w_id, nempty_num, protocol="v2") + return ( + x_offset, + y_offset, + m2w_indices, + w_sizes, + w2n_indices, + n2n_indices, + w2m_indices, + n_coords, + ) + + def get_shifted_sp(self, sp): + """ + get the shifted sparse tensor for shift-window + """ + stride_in = sp.coordinate_map_key.get_tensor_stride() + shift_size = self.shift_size * stride_in[0] + shifted_C = sp.C.clone() + shifted_C[:, 1:] += shift_size + shifted_sp = SparseTensor( + features=sp.F, + coordinates=shifted_C, + device=self.device, + tensor_stride=stride_in, + ) + return shifted_sp + + def get_window_pos(self, sp): + stride_in = sp.coordinate_map_key.get_tensor_stride() + return (sp.C[:, 1:] / stride_in[0]) % self.window_size + + def forward(self, sp, coords_sp): + """ + xyz: position of point inside voxel + colors: other signal for cRSE, include colors and normals + local_xyz: relative position of point indide voxel(using for finer cRSE table) + """ + colors = coords_sp.F[:, 4:] + xyz = coords_sp.F[:, :4] + local_xyz = (xyz - coords_sp.C)[ + :, 1: + ] / coords_sp.coordinate_map_key.get_tensor_stride()[0] + self.device = sp.device + sp_shift = self.get_shifted_sp(sp) + + attn_args = self.get_index01(sp, local_xyz, colors) + attn_args_shift = self.get_index01(sp_shift, local_xyz, colors) + + feats = sp.F + for i, blk in enumerate(self.blocks): + attn_args_blk = attn_args if i % 2 == 0 else attn_args_shift + feats = blk(feats, attn_args_blk) # [N, C] + + sp = assign_feats(sp, feats) + if self.downsample is not None: + sp_down, coords_sp = self.downsample(sp, coords_sp) + return sp, sp_down, coords_sp + else: + return sp, sp, coords_sp + + def extra_repr(self) -> str: + return f"window_size={self.window_size}, depth={self.depth}, channel={self.dim}, num_heads={self.num_heads}, quant_size={self.quant_size}, cRSE={self.cRSE}, fp16_mode={self.fp16_mode}" diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/swin3d/swin3d_v1m1_base.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/swin3d/swin3d_v1m1_base.py new file mode 100644 index 0000000000000000000000000000000000000000..1295e5d791e8ac33d3d4c43be03d4f08ade1345f --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/swin3d/swin3d_v1m1_base.py @@ -0,0 +1,190 @@ +import torch +import torch.nn as nn +import MinkowskiEngine as ME +from MinkowskiEngine import SparseTensor +from timm.models.layers import trunc_normal_ + +from .mink_layers import MinkConvBNRelu, MinkResBlock +from .swin3d_layers import GridDownsample, GridKNNDownsample, BasicLayer, Upsample +from pointcept.models.builder import MODELS +from pointcept.models.utils import offset2batch, batch2offset + + +@MODELS.register_module("Swin3D-v1m1") +class Swin3DUNet(nn.Module): + def __init__( + self, + in_channels, + num_classes, + base_grid_size, + depths, + channels, + num_heads, + window_sizes, + quant_size, + drop_path_rate=0.2, + up_k=3, + num_layers=5, + stem_transformer=True, + down_stride=2, + upsample="linear", + knn_down=True, + cRSE="XYZ_RGB", + fp16_mode=0, + ): + super().__init__() + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] # stochastic depth decay rule + if knn_down: + downsample = GridKNNDownsample + else: + downsample = GridDownsample + + self.cRSE = cRSE + if stem_transformer: + self.stem_layer = MinkConvBNRelu( + in_channels=in_channels, + out_channels=channels[0], + kernel_size=3, + stride=1, + ) + self.layer_start = 0 + else: + self.stem_layer = nn.Sequential( + MinkConvBNRelu( + in_channels=in_channels, + out_channels=channels[0], + kernel_size=3, + stride=1, + ), + MinkResBlock(in_channels=channels[0], out_channels=channels[0]), + ) + self.downsample = downsample( + channels[0], channels[1], kernel_size=down_stride, stride=down_stride + ) + self.layer_start = 1 + self.layers = nn.ModuleList( + [ + BasicLayer( + dim=channels[i], + depth=depths[i], + num_heads=num_heads[i], + window_size=window_sizes[i], + quant_size=quant_size, + drop_path=dpr[sum(depths[:i]) : sum(depths[: i + 1])], + downsample=downsample if i < num_layers - 1 else None, + down_stride=down_stride if i == 0 else 2, + out_channels=channels[i + 1] if i < num_layers - 1 else None, + cRSE=cRSE, + fp16_mode=fp16_mode, + ) + for i in range(self.layer_start, num_layers) + ] + ) + + if "attn" in upsample: + up_attn = True + else: + up_attn = False + + self.upsamples = nn.ModuleList( + [ + Upsample( + channels[i], + channels[i - 1], + num_heads[i - 1], + window_sizes[i - 1], + quant_size, + attn=up_attn, + up_k=up_k, + cRSE=cRSE, + fp16_mode=fp16_mode, + ) + for i in range(num_layers - 1, 0, -1) + ] + ) + + self.classifier = nn.Sequential( + nn.Linear(channels[0], channels[0]), + nn.BatchNorm1d(channels[0]), + nn.ReLU(inplace=True), + nn.Linear(channels[0], num_classes), + ) + self.num_classes = num_classes + self.base_grid_size = base_grid_size + self.init_weights() + + def forward(self, data_dict): + grid_coord = data_dict["grid_coord"] + feat = data_dict["feat"] + coord_feat = data_dict["coord_feat"] + coord = data_dict["coord"] + offset = data_dict["offset"] + batch = offset2batch(offset) + in_field = ME.TensorField( + features=torch.cat( + [ + batch.unsqueeze(-1), + coord / self.base_grid_size, + coord_feat / 1.001, + feat, + ], + dim=1, + ), + coordinates=torch.cat([batch.unsqueeze(-1).int(), grid_coord.int()], dim=1), + quantization_mode=ME.SparseTensorQuantizationMode.UNWEIGHTED_AVERAGE, + minkowski_algorithm=ME.MinkowskiAlgorithm.SPEED_OPTIMIZED, + device=feat.device, + ) + + sp = in_field.sparse() + coords_sp = SparseTensor( + features=sp.F[:, : coord_feat.shape[-1] + 4], + coordinate_map_key=sp.coordinate_map_key, + coordinate_manager=sp.coordinate_manager, + ) + sp = SparseTensor( + features=sp.F[:, coord_feat.shape[-1] + 4 :], + coordinate_map_key=sp.coordinate_map_key, + coordinate_manager=sp.coordinate_manager, + ) + sp_stack = [] + coords_sp_stack = [] + sp = self.stem_layer(sp) + if self.layer_start > 0: + sp_stack.append(sp) + coords_sp_stack.append(coords_sp) + sp, coords_sp = self.downsample(sp, coords_sp) + + for i, layer in enumerate(self.layers): + coords_sp_stack.append(coords_sp) + sp, sp_down, coords_sp = layer(sp, coords_sp) + sp_stack.append(sp) + assert (coords_sp.C == sp_down.C).all() + sp = sp_down + + sp = sp_stack.pop() + coords_sp = coords_sp_stack.pop() + for i, upsample in enumerate(self.upsamples): + sp_i = sp_stack.pop() + coords_sp_i = coords_sp_stack.pop() + sp = upsample(sp, coords_sp, sp_i, coords_sp_i) + coords_sp = coords_sp_i + + output = self.classifier(sp.slice(in_field).F) + return output + + def init_weights(self): + """Initialize the weights in backbone.""" + + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm) or isinstance(m, nn.BatchNorm1d): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + self.apply(_init_weights) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..66e6bc0f62993abb3625a9598f54e7775aeb0008 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/__init__.py @@ -0,0 +1,4 @@ +from .misc import offset2batch, offset2bincount, batch2offset, off_diagonal +from .checkpoint import checkpoint +from .serialization import encode, decode +from .structure import Point diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/checkpoint.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..58820352bd5d1b37b3905b038816323253ffd3de --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/checkpoint.py @@ -0,0 +1,57 @@ +""" +Checkpoint Utils for Models + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import torch + + +class CheckpointFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, run_function, length, *args): + ctx.run_function = run_function + ctx.input_tensors = list(args[:length]) + ctx.input_params = list(args[length:]) + + with torch.no_grad(): + output_tensors = ctx.run_function(*ctx.input_tensors) + return output_tensors + + @staticmethod + def backward(ctx, *output_grads): + ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors] + with torch.enable_grad(): + # Fixes a bug where the first op in run_function modifies the + # Tensor storage in place, which is not allowed for detach()'d + # Tensors. + shallow_copies = [x.view_as(x) for x in ctx.input_tensors] + output_tensors = ctx.run_function(*shallow_copies) + input_grads = torch.autograd.grad( + output_tensors, + ctx.input_tensors + ctx.input_params, + output_grads, + allow_unused=True, + ) + del ctx.input_tensors + del ctx.input_params + del output_tensors + return (None, None) + input_grads + + +def checkpoint(func, inputs, params, flag): + """ + Evaluate a function without caching intermediate activations, allowing for + reduced memory at the expense of extra compute in the backward pass. + :param func: the function to evaluate. + :param inputs: the argument sequence to pass to `func`. + :param params: a sequence of parameters `func` depends on but does not + explicitly take as arguments. + :param flag: if False, disable gradient checkpointing. + """ + if flag: + args = tuple(inputs) + tuple(params) + return CheckpointFunction.apply(func, len(inputs), *args) + else: + return func(*inputs) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/misc.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..61dfdfb44a82fc0ef585ca5732518fe85e466889 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/misc.py @@ -0,0 +1,35 @@ +""" +General Utils for Models + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import torch + + +@torch.inference_mode() +def offset2bincount(offset): + return torch.diff( + offset, prepend=torch.tensor([0], device=offset.device, dtype=torch.long) + ) + + +@torch.inference_mode() +def offset2batch(offset): + bincount = offset2bincount(offset) + return torch.arange( + len(bincount), device=offset.device, dtype=torch.long + ).repeat_interleave(bincount) + + +@torch.inference_mode() +def batch2offset(batch): + return torch.cumsum(batch.bincount(), dim=0).long() + + +def off_diagonal(x): + # return a flattened view of the off-diagonal elements of a square matrix + n, m = x.shape + assert n == m + return x.flatten()[:-1].view(n - 1, n + 1)[:, 1:].flatten() diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/serialization/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/serialization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..058c5e1001c76d9c7014bf0bbb824eec4f54f476 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/serialization/__init__.py @@ -0,0 +1,8 @@ +from .default import ( + encode, + decode, + z_order_encode, + z_order_decode, + hilbert_encode, + hilbert_decode, +) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/serialization/default.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/serialization/default.py new file mode 100644 index 0000000000000000000000000000000000000000..15898b55625fc0e1125db9b713e900892f04176c --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/serialization/default.py @@ -0,0 +1,59 @@ +import torch +from .z_order import xyz2key as z_order_encode_ +from .z_order import key2xyz as z_order_decode_ +from .hilbert import encode as hilbert_encode_ +from .hilbert import decode as hilbert_decode_ + + +@torch.inference_mode() +def encode(grid_coord, batch=None, depth=16, order="z"): + assert order in {"z", "z-trans", "hilbert", "hilbert-trans"} + if order == "z": + code = z_order_encode(grid_coord, depth=depth) + elif order == "z-trans": + code = z_order_encode(grid_coord[:, [1, 0, 2]], depth=depth) + elif order == "hilbert": + code = hilbert_encode(grid_coord, depth=depth) + elif order == "hilbert-trans": + code = hilbert_encode(grid_coord[:, [1, 0, 2]], depth=depth) + else: + raise NotImplementedError + if batch is not None: + batch = batch.long() + code = batch << depth * 3 | code + return code + + +@torch.inference_mode() +def decode(code, depth=16, order="z"): + assert order in {"z", "hilbert"} + batch = code >> depth * 3 + code = code & ((1 << depth * 3) - 1) + if order == "z": + grid_coord = z_order_decode(code, depth=depth) + elif order == "hilbert": + grid_coord = hilbert_decode(code, depth=depth) + else: + raise NotImplementedError + return grid_coord, batch + + +def z_order_encode(grid_coord: torch.Tensor, depth: int = 16): + x, y, z = grid_coord[:, 0].long(), grid_coord[:, 1].long(), grid_coord[:, 2].long() + # we block the support to batch, maintain batched code in Point class + code = z_order_encode_(x, y, z, b=None, depth=depth) + return code + + +def z_order_decode(code: torch.Tensor, depth): + x, y, z = z_order_decode_(code, depth=depth) + grid_coord = torch.stack([x, y, z], dim=-1) # (N, 3) + return grid_coord + + +def hilbert_encode(grid_coord: torch.Tensor, depth: int = 16): + return hilbert_encode_(grid_coord, num_dims=3, num_bits=depth) + + +def hilbert_decode(code: torch.Tensor, depth: int = 16): + return hilbert_decode_(code, num_dims=3, num_bits=depth) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/serialization/hilbert.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/serialization/hilbert.py new file mode 100644 index 0000000000000000000000000000000000000000..c96a3a9e15be64059811eb86139f28c6016ad0fe --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/serialization/hilbert.py @@ -0,0 +1,303 @@ +""" +Hilbert Order +Modified from https://github.com/PrincetonLIPS/numpy-hilbert-curve + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com), Kaixin Xu +Please cite our work if the code is helpful to you. +""" + +import torch + + +def right_shift(binary, k=1, axis=-1): + """Right shift an array of binary values. + + Parameters: + ----------- + binary: An ndarray of binary values. + + k: The number of bits to shift. Default 1. + + axis: The axis along which to shift. Default -1. + + Returns: + -------- + Returns an ndarray with zero prepended and the ends truncated, along + whatever axis was specified.""" + + # If we're shifting the whole thing, just return zeros. + if binary.shape[axis] <= k: + return torch.zeros_like(binary) + + # Determine the padding pattern. + # padding = [(0,0)] * len(binary.shape) + # padding[axis] = (k,0) + + # Determine the slicing pattern to eliminate just the last one. + slicing = [slice(None)] * len(binary.shape) + slicing[axis] = slice(None, -k) + shifted = torch.nn.functional.pad( + binary[tuple(slicing)], (k, 0), mode="constant", value=0 + ) + + return shifted + + +def binary2gray(binary, axis=-1): + """Convert an array of binary values into Gray codes. + + This uses the classic X ^ (X >> 1) trick to compute the Gray code. + + Parameters: + ----------- + binary: An ndarray of binary values. + + axis: The axis along which to compute the gray code. Default=-1. + + Returns: + -------- + Returns an ndarray of Gray codes. + """ + shifted = right_shift(binary, axis=axis) + + # Do the X ^ (X >> 1) trick. + gray = torch.logical_xor(binary, shifted) + + return gray + + +def gray2binary(gray, axis=-1): + """Convert an array of Gray codes back into binary values. + + Parameters: + ----------- + gray: An ndarray of gray codes. + + axis: The axis along which to perform Gray decoding. Default=-1. + + Returns: + -------- + Returns an ndarray of binary values. + """ + + # Loop the log2(bits) number of times necessary, with shift and xor. + shift = 2 ** (torch.Tensor([gray.shape[axis]]).log2().ceil().int() - 1) + while shift > 0: + gray = torch.logical_xor(gray, right_shift(gray, shift)) + shift = torch.div(shift, 2, rounding_mode="floor") + return gray + + +def encode(locs, num_dims, num_bits): + """Decode an array of locations in a hypercube into a Hilbert integer. + + This is a vectorized-ish version of the Hilbert curve implementation by John + Skilling as described in: + + Skilling, J. (2004, April). Programming the Hilbert curve. In AIP Conference + Proceedings (Vol. 707, No. 1, pp. 381-387). American Institute of Physics. + + Params: + ------- + locs - An ndarray of locations in a hypercube of num_dims dimensions, in + which each dimension runs from 0 to 2**num_bits-1. The shape can + be arbitrary, as long as the last dimension of the same has size + num_dims. + + num_dims - The dimensionality of the hypercube. Integer. + + num_bits - The number of bits for each dimension. Integer. + + Returns: + -------- + The output is an ndarray of uint64 integers with the same shape as the + input, excluding the last dimension, which needs to be num_dims. + """ + + # Keep around the original shape for later. + orig_shape = locs.shape + bitpack_mask = 1 << torch.arange(0, 8).to(locs.device) + bitpack_mask_rev = bitpack_mask.flip(-1) + + if orig_shape[-1] != num_dims: + raise ValueError( + """ + The shape of locs was surprising in that the last dimension was of size + %d, but num_dims=%d. These need to be equal. + """ + % (orig_shape[-1], num_dims) + ) + + if num_dims * num_bits > 63: + raise ValueError( + """ + num_dims=%d and num_bits=%d for %d bits total, which can't be encoded + into a int64. Are you sure you need that many points on your Hilbert + curve? + """ + % (num_dims, num_bits, num_dims * num_bits) + ) + + # Treat the location integers as 64-bit unsigned and then split them up into + # a sequence of uint8s. Preserve the association by dimension. + locs_uint8 = locs.long().view(torch.uint8).reshape((-1, num_dims, 8)).flip(-1) + + # Now turn these into bits and truncate to num_bits. + gray = ( + locs_uint8.unsqueeze(-1) + .bitwise_and(bitpack_mask_rev) + .ne(0) + .byte() + .flatten(-2, -1)[..., -num_bits:] + ) + + # Run the decoding process the other way. + # Iterate forwards through the bits. + for bit in range(0, num_bits): + # Iterate forwards through the dimensions. + for dim in range(0, num_dims): + # Identify which ones have this bit active. + mask = gray[:, dim, bit] + + # Where this bit is on, invert the 0 dimension for lower bits. + gray[:, 0, bit + 1 :] = torch.logical_xor( + gray[:, 0, bit + 1 :], mask[:, None] + ) + + # Where the bit is off, exchange the lower bits with the 0 dimension. + to_flip = torch.logical_and( + torch.logical_not(mask[:, None]).repeat(1, gray.shape[2] - bit - 1), + torch.logical_xor(gray[:, 0, bit + 1 :], gray[:, dim, bit + 1 :]), + ) + gray[:, dim, bit + 1 :] = torch.logical_xor( + gray[:, dim, bit + 1 :], to_flip + ) + gray[:, 0, bit + 1 :] = torch.logical_xor(gray[:, 0, bit + 1 :], to_flip) + + # Now flatten out. + gray = gray.swapaxes(1, 2).reshape((-1, num_bits * num_dims)) + + # Convert Gray back to binary. + hh_bin = gray2binary(gray) + + # Pad back out to 64 bits. + extra_dims = 64 - num_bits * num_dims + padded = torch.nn.functional.pad(hh_bin, (extra_dims, 0), "constant", 0) + + # Convert binary values into uint8s. + hh_uint8 = ( + (padded.flip(-1).reshape((-1, 8, 8)) * bitpack_mask) + .sum(2) + .squeeze() + .type(torch.uint8) + ) + + # Convert uint8s into uint64s. + hh_uint64 = hh_uint8.view(torch.int64).squeeze() + + return hh_uint64 + + +def decode(hilberts, num_dims, num_bits): + """Decode an array of Hilbert integers into locations in a hypercube. + + This is a vectorized-ish version of the Hilbert curve implementation by John + Skilling as described in: + + Skilling, J. (2004, April). Programming the Hilbert curve. In AIP Conference + Proceedings (Vol. 707, No. 1, pp. 381-387). American Institute of Physics. + + Params: + ------- + hilberts - An ndarray of Hilbert integers. Must be an integer dtype and + cannot have fewer bits than num_dims * num_bits. + + num_dims - The dimensionality of the hypercube. Integer. + + num_bits - The number of bits for each dimension. Integer. + + Returns: + -------- + The output is an ndarray of unsigned integers with the same shape as hilberts + but with an additional dimension of size num_dims. + """ + + if num_dims * num_bits > 64: + raise ValueError( + """ + num_dims=%d and num_bits=%d for %d bits total, which can't be encoded + into a uint64. Are you sure you need that many points on your Hilbert + curve? + """ + % (num_dims, num_bits) + ) + + # Handle the case where we got handed a naked integer. + hilberts = torch.atleast_1d(hilberts) + + # Keep around the shape for later. + orig_shape = hilberts.shape + bitpack_mask = 2 ** torch.arange(0, 8).to(hilberts.device) + bitpack_mask_rev = bitpack_mask.flip(-1) + + # Treat each of the hilberts as a s equence of eight uint8. + # This treats all of the inputs as uint64 and makes things uniform. + hh_uint8 = ( + hilberts.ravel().type(torch.int64).view(torch.uint8).reshape((-1, 8)).flip(-1) + ) + + # Turn these lists of uints into lists of bits and then truncate to the size + # we actually need for using Skilling's procedure. + hh_bits = ( + hh_uint8.unsqueeze(-1) + .bitwise_and(bitpack_mask_rev) + .ne(0) + .byte() + .flatten(-2, -1)[:, -num_dims * num_bits :] + ) + + # Take the sequence of bits and Gray-code it. + gray = binary2gray(hh_bits) + + # There has got to be a better way to do this. + # I could index them differently, but the eventual packbits likes it this way. + gray = gray.reshape((-1, num_bits, num_dims)).swapaxes(1, 2) + + # Iterate backwards through the bits. + for bit in range(num_bits - 1, -1, -1): + # Iterate backwards through the dimensions. + for dim in range(num_dims - 1, -1, -1): + # Identify which ones have this bit active. + mask = gray[:, dim, bit] + + # Where this bit is on, invert the 0 dimension for lower bits. + gray[:, 0, bit + 1 :] = torch.logical_xor( + gray[:, 0, bit + 1 :], mask[:, None] + ) + + # Where the bit is off, exchange the lower bits with the 0 dimension. + to_flip = torch.logical_and( + torch.logical_not(mask[:, None]), + torch.logical_xor(gray[:, 0, bit + 1 :], gray[:, dim, bit + 1 :]), + ) + gray[:, dim, bit + 1 :] = torch.logical_xor( + gray[:, dim, bit + 1 :], to_flip + ) + gray[:, 0, bit + 1 :] = torch.logical_xor(gray[:, 0, bit + 1 :], to_flip) + + # Pad back out to 64 bits. + extra_dims = 64 - num_bits + padded = torch.nn.functional.pad(gray, (extra_dims, 0), "constant", 0) + + # Now chop these up into blocks of 8. + locs_chopped = padded.flip(-1).reshape((-1, num_dims, 8, 8)) + + # Take those blocks and turn them unto uint8s. + # from IPython import embed; embed() + locs_uint8 = (locs_chopped * bitpack_mask).sum(3).squeeze().type(torch.uint8) + + # Finally, treat these as uint64s. + flat_locs = locs_uint8.view(torch.int64) + + # Return them in the expected shape. + return flat_locs.reshape((*orig_shape, num_dims)) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/serialization/z_order.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/serialization/z_order.py new file mode 100644 index 0000000000000000000000000000000000000000..6fd01a5bcf4b6c76c5d75db4999326e174409ee3 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/serialization/z_order.py @@ -0,0 +1,126 @@ +# -------------------------------------------------------- +# Octree-based Sparse Convolutional Neural Networks +# Copyright (c) 2022 Peng-Shuai Wang +# Licensed under The MIT License [see LICENSE for details] +# Written by Peng-Shuai Wang +# -------------------------------------------------------- + +import torch +from typing import Optional, Union + + +class KeyLUT: + def __init__(self): + r256 = torch.arange(256, dtype=torch.int64) + r512 = torch.arange(512, dtype=torch.int64) + zero = torch.zeros(256, dtype=torch.int64) + device = torch.device("cpu") + + self._encode = { + device: ( + self.xyz2key(r256, zero, zero, 8), + self.xyz2key(zero, r256, zero, 8), + self.xyz2key(zero, zero, r256, 8), + ) + } + self._decode = {device: self.key2xyz(r512, 9)} + + def encode_lut(self, device=torch.device("cpu")): + if device not in self._encode: + cpu = torch.device("cpu") + self._encode[device] = tuple(e.to(device) for e in self._encode[cpu]) + return self._encode[device] + + def decode_lut(self, device=torch.device("cpu")): + if device not in self._decode: + cpu = torch.device("cpu") + self._decode[device] = tuple(e.to(device) for e in self._decode[cpu]) + return self._decode[device] + + def xyz2key(self, x, y, z, depth): + key = torch.zeros_like(x) + for i in range(depth): + mask = 1 << i + key = ( + key + | ((x & mask) << (2 * i + 2)) + | ((y & mask) << (2 * i + 1)) + | ((z & mask) << (2 * i + 0)) + ) + return key + + def key2xyz(self, key, depth): + x = torch.zeros_like(key) + y = torch.zeros_like(key) + z = torch.zeros_like(key) + for i in range(depth): + x = x | ((key & (1 << (3 * i + 2))) >> (2 * i + 2)) + y = y | ((key & (1 << (3 * i + 1))) >> (2 * i + 1)) + z = z | ((key & (1 << (3 * i + 0))) >> (2 * i + 0)) + return x, y, z + + +_key_lut = KeyLUT() + + +def xyz2key( + x: torch.Tensor, + y: torch.Tensor, + z: torch.Tensor, + b: Optional[Union[torch.Tensor, int]] = None, + depth: int = 16, +): + r"""Encodes :attr:`x`, :attr:`y`, :attr:`z` coordinates to the shuffled keys + based on pre-computed look up tables. The speed of this function is much + faster than the method based on for-loop. + + Args: + x (torch.Tensor): The x coordinate. + y (torch.Tensor): The y coordinate. + z (torch.Tensor): The z coordinate. + b (torch.Tensor or int): The batch index of the coordinates, and should be + smaller than 32768. If :attr:`b` is :obj:`torch.Tensor`, the size of + :attr:`b` must be the same as :attr:`x`, :attr:`y`, and :attr:`z`. + depth (int): The depth of the shuffled key, and must be smaller than 17 (< 17). + """ + + EX, EY, EZ = _key_lut.encode_lut(x.device) + x, y, z = x.long(), y.long(), z.long() + + mask = 255 if depth > 8 else (1 << depth) - 1 + key = EX[x & mask] | EY[y & mask] | EZ[z & mask] + if depth > 8: + mask = (1 << (depth - 8)) - 1 + key16 = EX[(x >> 8) & mask] | EY[(y >> 8) & mask] | EZ[(z >> 8) & mask] + key = key16 << 24 | key + + if b is not None: + b = b.long() + key = b << 48 | key + + return key + + +def key2xyz(key: torch.Tensor, depth: int = 16): + r"""Decodes the shuffled key to :attr:`x`, :attr:`y`, :attr:`z` coordinates + and the batch index based on pre-computed look up tables. + + Args: + key (torch.Tensor): The shuffled key. + depth (int): The depth of the shuffled key, and must be smaller than 17 (< 17). + """ + + DX, DY, DZ = _key_lut.decode_lut(key.device) + x, y, z = torch.zeros_like(key), torch.zeros_like(key), torch.zeros_like(key) + + b = key >> 48 + key = key & ((1 << 48) - 1) + + n = (depth + 2) // 3 + for i in range(n): + k = key >> (i * 9) & 511 + x = x | (DX[k] << (i * 3)) + y = y | (DY[k] << (i * 3)) + z = z | (DZ[k] << (i * 3)) + + return x, y, z, b diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/structure.py b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/structure.py new file mode 100644 index 0000000000000000000000000000000000000000..47fcd054067967f1ce5953d32df288ecc41c7aae --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/models/utils/structure.py @@ -0,0 +1,180 @@ +import torch +import spconv.pytorch as spconv + +try: + import ocnn +except ImportError: + ocnn = None +from addict import Dict + +from pointcept.models.utils.serialization import encode, decode +from pointcept.models.utils import offset2batch, batch2offset + + +class Point(Dict): + """ + Point Structure of Pointcept + + A Point (point cloud) in Pointcept is a dictionary that contains various properties of + a batched point cloud. The property with the following names have a specific definition + as follows: + + - "coord": original coordinate of point cloud; + - "grid_coord": grid coordinate for specific grid size (related to GridSampling); + Point also support the following optional attributes: + - "offset": if not exist, initialized as batch size is 1; + - "batch": if not exist, initialized as batch size is 1; + - "feat": feature of point cloud, default input of model; + - "grid_size": Grid size of point cloud (related to GridSampling); + (related to Serialization) + - "serialized_depth": depth of serialization, 2 ** depth * grid_size describe the maximum of point cloud range; + - "serialized_code": a list of serialization codes; + - "serialized_order": a list of serialization order determined by code; + - "serialized_inverse": a list of inverse mapping determined by code; + (related to Sparsify: SpConv) + - "sparse_shape": Sparse shape for Sparse Conv Tensor; + - "sparse_conv_feat": SparseConvTensor init with information provide by Point; + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # If one of "offset" or "batch" do not exist, generate by the existing one + if "batch" not in self.keys() and "offset" in self.keys(): + self["batch"] = offset2batch(self.offset) + elif "offset" not in self.keys() and "batch" in self.keys(): + self["offset"] = batch2offset(self.batch) + + def serialization(self, order="z", depth=None, shuffle_orders=False): + """ + Point Cloud Serialization + + relay on ["grid_coord" or "coord" + "grid_size", "batch", "feat"] + """ + assert "batch" in self.keys() + if "grid_coord" not in self.keys(): + # if you don't want to operate GridSampling in data augmentation, + # please add the following augmentation into your pipline: + # dict(type="Copy", keys_dict={"grid_size": 0.01}), + # (adjust `grid_size` to what your want) + assert {"grid_size", "coord"}.issubset(self.keys()) + self["grid_coord"] = torch.div( + self.coord - self.coord.min(0)[0], self.grid_size, rounding_mode="trunc" + ).int() + + if depth is None: + # Adaptive measure the depth of serialization cube (length = 2 ^ depth) + depth = int(self.grid_coord.max()).bit_length() + self["serialized_depth"] = depth + # Maximum bit length for serialization code is 63 (int64) + assert depth * 3 + len(self.offset).bit_length() <= 63 + # Here we follow OCNN and set the depth limitation to 16 (48bit) for the point position. + # Although depth is limited to less than 16, we can encode a 655.36^3 (2^16 * 0.01) meter^3 + # cube with a grid size of 0.01 meter. We consider it is enough for the current stage. + # We can unlock the limitation by optimizing the z-order encoding function if necessary. + assert depth <= 16 + + # The serialization codes are arranged as following structures: + # [Order1 ([n]), + # Order2 ([n]), + # ... + # OrderN ([n])] (k, n) + code = [ + encode(self.grid_coord, self.batch, depth, order=order_) for order_ in order + ] + code = torch.stack(code) + order = torch.argsort(code) + inverse = torch.zeros_like(order).scatter_( + dim=1, + index=order, + src=torch.arange(0, code.shape[1], device=order.device).repeat( + code.shape[0], 1 + ), + ) + + if shuffle_orders: + perm = torch.randperm(code.shape[0]) + code = code[perm] + order = order[perm] + inverse = inverse[perm] + + self["serialized_code"] = code + self["serialized_order"] = order + self["serialized_inverse"] = inverse + + def sparsify(self, pad=96): + """ + Point Cloud Serialization + + Point cloud is sparse, here we use "sparsify" to specifically refer to + preparing "spconv.SparseConvTensor" for SpConv. + + relay on ["grid_coord" or "coord" + "grid_size", "batch", "feat"] + + pad: padding sparse for sparse shape. + """ + assert {"feat", "batch"}.issubset(self.keys()) + if "grid_coord" not in self.keys(): + # if you don't want to operate GridSampling in data augmentation, + # please add the following augmentation into your pipline: + # dict(type="Copy", keys_dict={"grid_size": 0.01}), + # (adjust `grid_size` to what your want) + assert {"grid_size", "coord"}.issubset(self.keys()) + self["grid_coord"] = torch.div( + self.coord - self.coord.min(0)[0], self.grid_size, rounding_mode="trunc" + ).int() + if "sparse_shape" in self.keys(): + sparse_shape = self.sparse_shape + else: + sparse_shape = torch.add( + torch.max(self.grid_coord, dim=0).values, pad + ).tolist() + sparse_conv_feat = spconv.SparseConvTensor( + features=self.feat, + indices=torch.cat( + [self.batch.unsqueeze(-1).int(), self.grid_coord.int()], dim=1 + ).contiguous(), + spatial_shape=sparse_shape, + batch_size=self.batch[-1].tolist() + 1, + ) + self["sparse_shape"] = sparse_shape + self["sparse_conv_feat"] = sparse_conv_feat + + def octreetization(self, depth=None, full_depth=None): + """ + Point Cloud Octreelization + + Generate octree with OCNN + relay on ["grid_coord", "batch", "feat"] + """ + assert ( + ocnn is not None + ), "Please follow https://github.com/octree-nn/ocnn-pytorch install ocnn." + assert {"grid_coord", "feat", "batch"}.issubset(self.keys()) + # add 1 to make grid space support shift order + if depth is None: + if "depth" in self.keys(): + depth = self.depth + else: + depth = int(self.grid_coord.max() + 1).bit_length() + if full_depth is None: + full_depth = 2 + self["depth"] = depth + assert depth <= 16 # maximum in ocnn + + # [0, 2**depth] -> [0, 2] -> [-1, 1] + coord = self.grid_coord / 2 ** (self.depth - 1) - 1.0 + point = ocnn.octree.Points( + points=coord, + features=self.feat, + batch_id=self.batch.unsqueeze(-1), + batch_size=self.batch[-1] + 1, + ) + octree = ocnn.octree.Octree( + depth=depth, + full_depth=full_depth, + batch_size=self.batch[-1] + 1, + device=coord.device, + ) + octree.build_octree(point) + octree.construct_all_neigh() + self["octree"] = octree diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/utils/__init__.py b/submodules/PointTransformerV3/Pointcept/pointcept/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/utils/cache.py b/submodules/PointTransformerV3/Pointcept/pointcept/utils/cache.py new file mode 100644 index 0000000000000000000000000000000000000000..623897e42a7a4256a65a1a0e9a7b5c0c46ce5a3e --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/utils/cache.py @@ -0,0 +1,56 @@ +""" +Data Cache Utils + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import SharedArray + +try: + from multiprocessing.shared_memory import ShareableList +except ImportError: + import warnings + + warnings.warn("Please update python version >= 3.8 to enable shared_memory") +import numpy as np + + +def shared_array(name, var=None): + if var is not None: + # check exist + if os.path.exists(f"/dev/shm/{name}"): + return SharedArray.attach(f"shm://{name}") + # create shared_array + data = SharedArray.create(f"shm://{name}", var.shape, dtype=var.dtype) + data[...] = var[...] + data.flags.writeable = False + else: + data = SharedArray.attach(f"shm://{name}").copy() + return data + + +def shared_dict(name, var=None): + name = str(name) + assert "." not in name # '.' is used as sep flag + data = {} + if var is not None: + assert isinstance(var, dict) + keys = var.keys() + # current version only cache np.array + keys_valid = [] + for key in keys: + if isinstance(var[key], np.ndarray): + keys_valid.append(key) + keys = keys_valid + + ShareableList(sequence=keys, name=name + ".keys") + for key in keys: + if isinstance(var[key], np.ndarray): + data[key] = shared_array(name=f"{name}.{key}", var=var[key]) + else: + keys = list(ShareableList(name=name + ".keys")) + for key in keys: + data[key] = shared_array(name=f"{name}.{key}") + return data diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/utils/comm.py b/submodules/PointTransformerV3/Pointcept/pointcept/utils/comm.py new file mode 100644 index 0000000000000000000000000000000000000000..69e29e7c690fe0500d3d9a84b6a8749e2f4f4655 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/utils/comm.py @@ -0,0 +1,198 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +""" +This file contains primitives for multi-gpu communication. +This is useful when doing distributed training. +Modified from detectron2(https://github.com/facebookresearch/detectron2) + +Copyright (c) Xiaoyang Wu (xiaoyang.wu@connect.hku.hk). All Rights Reserved. +Please cite our work if you use any part of the code. +""" + +import functools +import numpy as np +import torch +import torch.distributed as dist + +_LOCAL_PROCESS_GROUP = None +""" +A torch process group which only includes processes that on the same machine as the current process. +This variable is set when processes are spawned by `launch()` in "engine/launch.py". +""" + + +def get_world_size() -> int: + if not dist.is_available(): + return 1 + if not dist.is_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank() -> int: + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + return dist.get_rank() + + +def get_local_rank() -> int: + """ + Returns: + The rank of the current process within the local (per-machine) process group. + """ + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + assert ( + _LOCAL_PROCESS_GROUP is not None + ), "Local process group is not created! Please use launch() to spawn processes!" + return dist.get_rank(group=_LOCAL_PROCESS_GROUP) + + +def get_local_size() -> int: + """ + Returns: + The size of the per-machine process group, + i.e. the number of processes per machine. + """ + if not dist.is_available(): + return 1 + if not dist.is_initialized(): + return 1 + return dist.get_world_size(group=_LOCAL_PROCESS_GROUP) + + +def is_main_process() -> bool: + return get_rank() == 0 + + +def synchronize(): + """ + Helper function to synchronize (barrier) among all processes when + using distributed training + """ + if not dist.is_available(): + return + if not dist.is_initialized(): + return + world_size = dist.get_world_size() + if world_size == 1: + return + if dist.get_backend() == dist.Backend.NCCL: + # This argument is needed to avoid warnings. + # It's valid only for NCCL backend. + dist.barrier(device_ids=[torch.cuda.current_device()]) + else: + dist.barrier() + + +@functools.lru_cache() +def _get_global_gloo_group(): + """ + Return a process group based on gloo backend, containing all the ranks + The result is cached. + """ + if dist.get_backend() == "nccl": + return dist.new_group(backend="gloo") + else: + return dist.group.WORLD + + +def all_gather(data, group=None): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors). + Args: + data: any picklable object + group: a torch process group. By default, will use a group which + contains all ranks on gloo backend. + Returns: + list[data]: list of data gathered from each rank + """ + if get_world_size() == 1: + return [data] + if group is None: + group = ( + _get_global_gloo_group() + ) # use CPU group by default, to reduce GPU RAM usage. + world_size = dist.get_world_size(group) + if world_size == 1: + return [data] + + output = [None for _ in range(world_size)] + dist.all_gather_object(output, data, group=group) + return output + + +def gather(data, dst=0, group=None): + """ + Run gather on arbitrary picklable data (not necessarily tensors). + Args: + data: any picklable object + dst (int): destination rank + group: a torch process group. By default, will use a group which + contains all ranks on gloo backend. + Returns: + list[data]: on dst, a list of data gathered from each rank. Otherwise, + an empty list. + """ + if get_world_size() == 1: + return [data] + if group is None: + group = _get_global_gloo_group() + world_size = dist.get_world_size(group=group) + if world_size == 1: + return [data] + rank = dist.get_rank(group=group) + + if rank == dst: + output = [None for _ in range(world_size)] + dist.gather_object(data, output, dst=dst, group=group) + return output + else: + dist.gather_object(data, None, dst=dst, group=group) + return [] + + +def shared_random_seed(): + """ + Returns: + int: a random number that is the same across all workers. + If workers need a shared RNG, they can use this shared seed to + create one. + All workers must call this function, otherwise it will deadlock. + """ + ints = np.random.randint(2**31) + all_ints = all_gather(ints) + return all_ints[0] + + +def reduce_dict(input_dict, average=True): + """ + Reduce the values in the dictionary from all processes so that process with rank + 0 has the reduced results. + Args: + input_dict (dict): inputs to be reduced. All the values must be scalar CUDA Tensor. + average (bool): whether to do average or sum + Returns: + a dict with the same keys as input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + # sort the keys so that they are consistent across processes + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.reduce(values, dst=0) + if dist.get_rank() == 0 and average: + # only main process gets accumulated, so only divide by + # world_size in this case + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/utils/config.py b/submodules/PointTransformerV3/Pointcept/pointcept/utils/config.py new file mode 100644 index 0000000000000000000000000000000000000000..316dd458b3760b38feeb33d941ad9ad060364a61 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/utils/config.py @@ -0,0 +1,694 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import ast +import copy +import os +import os.path as osp +import platform +import shutil +import sys +import tempfile +import uuid +import warnings +from argparse import Action, ArgumentParser +from collections import abc +from importlib import import_module + +from addict import Dict +from yapf.yapflib.yapf_api import FormatCode + +from .misc import import_modules_from_strings +from .path import check_file_exist + +if platform.system() == "Windows": + import regex as re +else: + import re + +BASE_KEY = "_base_" +DELETE_KEY = "_delete_" +DEPRECATION_KEY = "_deprecation_" +RESERVED_KEYS = ["filename", "text", "pretty_text"] + + +class ConfigDict(Dict): + def __missing__(self, name): + raise KeyError(name) + + def __getattr__(self, name): + try: + value = super(ConfigDict, self).__getattr__(name) + except KeyError: + ex = AttributeError( + f"'{self.__class__.__name__}' object has no " f"attribute '{name}'" + ) + except Exception as e: + ex = e + else: + return value + raise ex + + +def add_args(parser, cfg, prefix=""): + for k, v in cfg.items(): + if isinstance(v, str): + parser.add_argument("--" + prefix + k) + elif isinstance(v, int): + parser.add_argument("--" + prefix + k, type=int) + elif isinstance(v, float): + parser.add_argument("--" + prefix + k, type=float) + elif isinstance(v, bool): + parser.add_argument("--" + prefix + k, action="store_true") + elif isinstance(v, dict): + add_args(parser, v, prefix + k + ".") + elif isinstance(v, abc.Iterable): + parser.add_argument("--" + prefix + k, type=type(v[0]), nargs="+") + else: + print(f"cannot parse key {prefix + k} of type {type(v)}") + return parser + + +class Config: + """A facility for config and config files. + + It supports common file formats as configs: python/json/yaml. The interface + is the same as a dict object and also allows access config values as + attributes. + + Example: + >>> cfg = Config(dict(a=1, b=dict(b1=[0, 1]))) + >>> cfg.a + 1 + >>> cfg.b + {'b1': [0, 1]} + >>> cfg.b.b1 + [0, 1] + >>> cfg = Config.fromfile('tests/data/config/a.py') + >>> cfg.filename + "/home/kchen/projects/mmcv/tests/data/config/a.py" + >>> cfg.item4 + 'test' + >>> cfg + "Config [path: /home/kchen/projects/mmcv/tests/data/config/a.py]: " + "{'item1': [1, 2], 'item2': {'a': 0}, 'item3': True, 'item4': 'test'}" + """ + + @staticmethod + def _validate_py_syntax(filename): + with open(filename, "r", encoding="utf-8") as f: + # Setting encoding explicitly to resolve coding issue on windows + content = f.read() + try: + ast.parse(content) + except SyntaxError as e: + raise SyntaxError( + "There are syntax errors in config " f"file {filename}: {e}" + ) + + @staticmethod + def _substitute_predefined_vars(filename, temp_config_name): + file_dirname = osp.dirname(filename) + file_basename = osp.basename(filename) + file_basename_no_extension = osp.splitext(file_basename)[0] + file_extname = osp.splitext(filename)[1] + support_templates = dict( + fileDirname=file_dirname, + fileBasename=file_basename, + fileBasenameNoExtension=file_basename_no_extension, + fileExtname=file_extname, + ) + with open(filename, "r", encoding="utf-8") as f: + # Setting encoding explicitly to resolve coding issue on windows + config_file = f.read() + for key, value in support_templates.items(): + regexp = r"\{\{\s*" + str(key) + r"\s*\}\}" + value = value.replace("\\", "/") + config_file = re.sub(regexp, value, config_file) + with open(temp_config_name, "w", encoding="utf-8") as tmp_config_file: + tmp_config_file.write(config_file) + + @staticmethod + def _pre_substitute_base_vars(filename, temp_config_name): + """Substitute base variable placehoders to string, so that parsing + would work.""" + with open(filename, "r", encoding="utf-8") as f: + # Setting encoding explicitly to resolve coding issue on windows + config_file = f.read() + base_var_dict = {} + regexp = r"\{\{\s*" + BASE_KEY + r"\.([\w\.]+)\s*\}\}" + base_vars = set(re.findall(regexp, config_file)) + for base_var in base_vars: + randstr = f"_{base_var}_{uuid.uuid4().hex.lower()[:6]}" + base_var_dict[randstr] = base_var + regexp = r"\{\{\s*" + BASE_KEY + r"\." + base_var + r"\s*\}\}" + config_file = re.sub(regexp, f'"{randstr}"', config_file) + with open(temp_config_name, "w", encoding="utf-8") as tmp_config_file: + tmp_config_file.write(config_file) + return base_var_dict + + @staticmethod + def _substitute_base_vars(cfg, base_var_dict, base_cfg): + """Substitute variable strings to their actual values.""" + cfg = copy.deepcopy(cfg) + + if isinstance(cfg, dict): + for k, v in cfg.items(): + if isinstance(v, str) and v in base_var_dict: + new_v = base_cfg + for new_k in base_var_dict[v].split("."): + new_v = new_v[new_k] + cfg[k] = new_v + elif isinstance(v, (list, tuple, dict)): + cfg[k] = Config._substitute_base_vars(v, base_var_dict, base_cfg) + elif isinstance(cfg, tuple): + cfg = tuple( + Config._substitute_base_vars(c, base_var_dict, base_cfg) for c in cfg + ) + elif isinstance(cfg, list): + cfg = [ + Config._substitute_base_vars(c, base_var_dict, base_cfg) for c in cfg + ] + elif isinstance(cfg, str) and cfg in base_var_dict: + new_v = base_cfg + for new_k in base_var_dict[cfg].split("."): + new_v = new_v[new_k] + cfg = new_v + + return cfg + + @staticmethod + def _file2dict(filename, use_predefined_variables=True): + filename = osp.abspath(osp.expanduser(filename)) + check_file_exist(filename) + fileExtname = osp.splitext(filename)[1] + if fileExtname not in [".py", ".json", ".yaml", ".yml"]: + raise IOError("Only py/yml/yaml/json type are supported now!") + + with tempfile.TemporaryDirectory() as temp_config_dir: + temp_config_file = tempfile.NamedTemporaryFile( + dir=temp_config_dir, suffix=fileExtname + ) + if platform.system() == "Windows": + temp_config_file.close() + temp_config_name = osp.basename(temp_config_file.name) + # Substitute predefined variables + if use_predefined_variables: + Config._substitute_predefined_vars(filename, temp_config_file.name) + else: + shutil.copyfile(filename, temp_config_file.name) + # Substitute base variables from placeholders to strings + base_var_dict = Config._pre_substitute_base_vars( + temp_config_file.name, temp_config_file.name + ) + + if filename.endswith(".py"): + temp_module_name = osp.splitext(temp_config_name)[0] + sys.path.insert(0, temp_config_dir) + Config._validate_py_syntax(filename) + mod = import_module(temp_module_name) + sys.path.pop(0) + cfg_dict = { + name: value + for name, value in mod.__dict__.items() + if not name.startswith("__") + } + # delete imported module + del sys.modules[temp_module_name] + elif filename.endswith((".yml", ".yaml", ".json")): + raise NotImplementedError + # close temp file + temp_config_file.close() + + # check deprecation information + if DEPRECATION_KEY in cfg_dict: + deprecation_info = cfg_dict.pop(DEPRECATION_KEY) + warning_msg = ( + f"The config file {filename} will be deprecated " "in the future." + ) + if "expected" in deprecation_info: + warning_msg += f' Please use {deprecation_info["expected"]} ' "instead." + if "reference" in deprecation_info: + warning_msg += ( + " More information can be found at " + f'{deprecation_info["reference"]}' + ) + warnings.warn(warning_msg) + + cfg_text = filename + "\n" + with open(filename, "r", encoding="utf-8") as f: + # Setting encoding explicitly to resolve coding issue on windows + cfg_text += f.read() + + if BASE_KEY in cfg_dict: + cfg_dir = osp.dirname(filename) + base_filename = cfg_dict.pop(BASE_KEY) + base_filename = ( + base_filename if isinstance(base_filename, list) else [base_filename] + ) + + cfg_dict_list = list() + cfg_text_list = list() + for f in base_filename: + _cfg_dict, _cfg_text = Config._file2dict(osp.join(cfg_dir, f)) + cfg_dict_list.append(_cfg_dict) + cfg_text_list.append(_cfg_text) + + base_cfg_dict = dict() + for c in cfg_dict_list: + duplicate_keys = base_cfg_dict.keys() & c.keys() + if len(duplicate_keys) > 0: + raise KeyError( + "Duplicate key is not allowed among bases. " + f"Duplicate keys: {duplicate_keys}" + ) + base_cfg_dict.update(c) + + # Substitute base variables from strings to their actual values + cfg_dict = Config._substitute_base_vars( + cfg_dict, base_var_dict, base_cfg_dict + ) + + base_cfg_dict = Config._merge_a_into_b(cfg_dict, base_cfg_dict) + cfg_dict = base_cfg_dict + + # merge cfg_text + cfg_text_list.append(cfg_text) + cfg_text = "\n".join(cfg_text_list) + + return cfg_dict, cfg_text + + @staticmethod + def _merge_a_into_b(a, b, allow_list_keys=False): + """merge dict ``a`` into dict ``b`` (non-inplace). + + Values in ``a`` will overwrite ``b``. ``b`` is copied first to avoid + in-place modifications. + + Args: + a (dict): The source dict to be merged into ``b``. + b (dict): The origin dict to be fetch keys from ``a``. + allow_list_keys (bool): If True, int string keys (e.g. '0', '1') + are allowed in source ``a`` and will replace the element of the + corresponding index in b if b is a list. Default: False. + + Returns: + dict: The modified dict of ``b`` using ``a``. + + Examples: + # Normally merge a into b. + >>> Config._merge_a_into_b( + ... dict(obj=dict(a=2)), dict(obj=dict(a=1))) + {'obj': {'a': 2}} + + # Delete b first and merge a into b. + >>> Config._merge_a_into_b( + ... dict(obj=dict(_delete_=True, a=2)), dict(obj=dict(a=1))) + {'obj': {'a': 2}} + + # b is a list + >>> Config._merge_a_into_b( + ... {'0': dict(a=2)}, [dict(a=1), dict(b=2)], True) + [{'a': 2}, {'b': 2}] + """ + b = b.copy() + for k, v in a.items(): + if allow_list_keys and k.isdigit() and isinstance(b, list): + k = int(k) + if len(b) <= k: + raise KeyError(f"Index {k} exceeds the length of list {b}") + b[k] = Config._merge_a_into_b(v, b[k], allow_list_keys) + elif isinstance(v, dict) and k in b and not v.pop(DELETE_KEY, False): + allowed_types = (dict, list) if allow_list_keys else dict + if not isinstance(b[k], allowed_types): + raise TypeError( + f"{k}={v} in child config cannot inherit from base " + f"because {k} is a dict in the child config but is of " + f"type {type(b[k])} in base config. You may set " + f"`{DELETE_KEY}=True` to ignore the base config" + ) + b[k] = Config._merge_a_into_b(v, b[k], allow_list_keys) + else: + b[k] = v + return b + + @staticmethod + def fromfile(filename, use_predefined_variables=True, import_custom_modules=True): + cfg_dict, cfg_text = Config._file2dict(filename, use_predefined_variables) + if import_custom_modules and cfg_dict.get("custom_imports", None): + import_modules_from_strings(**cfg_dict["custom_imports"]) + return Config(cfg_dict, cfg_text=cfg_text, filename=filename) + + @staticmethod + def fromstring(cfg_str, file_format): + """Generate config from config str. + + Args: + cfg_str (str): Config str. + file_format (str): Config file format corresponding to the + config str. Only py/yml/yaml/json type are supported now! + + Returns: + obj:`Config`: Config obj. + """ + if file_format not in [".py", ".json", ".yaml", ".yml"]: + raise IOError("Only py/yml/yaml/json type are supported now!") + if file_format != ".py" and "dict(" in cfg_str: + # check if users specify a wrong suffix for python + warnings.warn('Please check "file_format", the file format may be .py') + with tempfile.NamedTemporaryFile( + "w", encoding="utf-8", suffix=file_format, delete=False + ) as temp_file: + temp_file.write(cfg_str) + # on windows, previous implementation cause error + # see PR 1077 for details + cfg = Config.fromfile(temp_file.name) + os.remove(temp_file.name) + return cfg + + @staticmethod + def auto_argparser(description=None): + """Generate argparser from config file automatically (experimental)""" + partial_parser = ArgumentParser(description=description) + partial_parser.add_argument("config", help="config file path") + cfg_file = partial_parser.parse_known_args()[0].config + cfg = Config.fromfile(cfg_file) + parser = ArgumentParser(description=description) + parser.add_argument("config", help="config file path") + add_args(parser, cfg) + return parser, cfg + + def __init__(self, cfg_dict=None, cfg_text=None, filename=None): + if cfg_dict is None: + cfg_dict = dict() + elif not isinstance(cfg_dict, dict): + raise TypeError("cfg_dict must be a dict, but " f"got {type(cfg_dict)}") + for key in cfg_dict: + if key in RESERVED_KEYS: + raise KeyError(f"{key} is reserved for config file") + + super(Config, self).__setattr__("_cfg_dict", ConfigDict(cfg_dict)) + super(Config, self).__setattr__("_filename", filename) + if cfg_text: + text = cfg_text + elif filename: + with open(filename, "r") as f: + text = f.read() + else: + text = "" + super(Config, self).__setattr__("_text", text) + + @property + def filename(self): + return self._filename + + @property + def text(self): + return self._text + + @property + def pretty_text(self): + indent = 4 + + def _indent(s_, num_spaces): + s = s_.split("\n") + if len(s) == 1: + return s_ + first = s.pop(0) + s = [(num_spaces * " ") + line for line in s] + s = "\n".join(s) + s = first + "\n" + s + return s + + def _format_basic_types(k, v, use_mapping=False): + if isinstance(v, str): + v_str = f"'{v}'" + else: + v_str = str(v) + + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f"{k_str}: {v_str}" + else: + attr_str = f"{str(k)}={v_str}" + attr_str = _indent(attr_str, indent) + + return attr_str + + def _format_list(k, v, use_mapping=False): + # check if all items in the list are dict + if all(isinstance(_, dict) for _ in v): + v_str = "[\n" + v_str += "\n".join( + f"dict({_indent(_format_dict(v_), indent)})," for v_ in v + ).rstrip(",") + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f"{k_str}: {v_str}" + else: + attr_str = f"{str(k)}={v_str}" + attr_str = _indent(attr_str, indent) + "]" + else: + attr_str = _format_basic_types(k, v, use_mapping) + return attr_str + + def _contain_invalid_identifier(dict_str): + contain_invalid_identifier = False + for key_name in dict_str: + contain_invalid_identifier |= not str(key_name).isidentifier() + return contain_invalid_identifier + + def _format_dict(input_dict, outest_level=False): + r = "" + s = [] + + use_mapping = _contain_invalid_identifier(input_dict) + if use_mapping: + r += "{" + for idx, (k, v) in enumerate(input_dict.items()): + is_last = idx >= len(input_dict) - 1 + end = "" if outest_level or is_last else "," + if isinstance(v, dict): + v_str = "\n" + _format_dict(v) + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f"{k_str}: dict({v_str}" + else: + attr_str = f"{str(k)}=dict({v_str}" + attr_str = _indent(attr_str, indent) + ")" + end + elif isinstance(v, list): + attr_str = _format_list(k, v, use_mapping) + end + else: + attr_str = _format_basic_types(k, v, use_mapping) + end + + s.append(attr_str) + r += "\n".join(s) + if use_mapping: + r += "}" + return r + + cfg_dict = self._cfg_dict.to_dict() + text = _format_dict(cfg_dict, outest_level=True) + # copied from setup.cfg + yapf_style = dict( + based_on_style="pep8", + blank_line_before_nested_class_or_def=True, + split_before_expression_after_opening_paren=True, + ) + text, _ = FormatCode(text, style_config=yapf_style, verify=True) + + return text + + def __repr__(self): + return f"Config (path: {self.filename}): {self._cfg_dict.__repr__()}" + + def __len__(self): + return len(self._cfg_dict) + + def __getattr__(self, name): + return getattr(self._cfg_dict, name) + + def __getitem__(self, name): + return self._cfg_dict.__getitem__(name) + + def __setattr__(self, name, value): + if isinstance(value, dict): + value = ConfigDict(value) + self._cfg_dict.__setattr__(name, value) + + def __setitem__(self, name, value): + if isinstance(value, dict): + value = ConfigDict(value) + self._cfg_dict.__setitem__(name, value) + + def __iter__(self): + return iter(self._cfg_dict) + + def __getstate__(self): + return (self._cfg_dict, self._filename, self._text) + + def __setstate__(self, state): + _cfg_dict, _filename, _text = state + super(Config, self).__setattr__("_cfg_dict", _cfg_dict) + super(Config, self).__setattr__("_filename", _filename) + super(Config, self).__setattr__("_text", _text) + + def dump(self, file=None): + cfg_dict = super(Config, self).__getattribute__("_cfg_dict").to_dict() + if self.filename.endswith(".py"): + if file is None: + return self.pretty_text + else: + with open(file, "w", encoding="utf-8") as f: + f.write(self.pretty_text) + else: + import mmcv + + if file is None: + file_format = self.filename.split(".")[-1] + return mmcv.dump(cfg_dict, file_format=file_format) + else: + mmcv.dump(cfg_dict, file) + + def merge_from_dict(self, options, allow_list_keys=True): + """Merge list into cfg_dict. + + Merge the dict parsed by MultipleKVAction into this cfg. + + Examples: + >>> options = {'models.backbone.depth': 50, + ... 'models.backbone.with_cp':True} + >>> cfg = Config(dict(models=dict(backbone=dict(type='ResNet')))) + >>> cfg.merge_from_dict(options) + >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict') + >>> assert cfg_dict == dict( + ... models=dict(backbone=dict(depth=50, with_cp=True))) + + # Merge list element + >>> cfg = Config(dict(pipeline=[ + ... dict(type='LoadImage'), dict(type='LoadAnnotations')])) + >>> options = dict(pipeline={'0': dict(type='SelfLoadImage')}) + >>> cfg.merge_from_dict(options, allow_list_keys=True) + >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict') + >>> assert cfg_dict == dict(pipeline=[ + ... dict(type='SelfLoadImage'), dict(type='LoadAnnotations')]) + + Args: + options (dict): dict of configs to merge from. + allow_list_keys (bool): If True, int string keys (e.g. '0', '1') + are allowed in ``options`` and will replace the element of the + corresponding index in the config if the config is a list. + Default: True. + """ + option_cfg_dict = {} + for full_key, v in options.items(): + d = option_cfg_dict + key_list = full_key.split(".") + for subkey in key_list[:-1]: + d.setdefault(subkey, ConfigDict()) + d = d[subkey] + subkey = key_list[-1] + d[subkey] = v + + cfg_dict = super(Config, self).__getattribute__("_cfg_dict") + super(Config, self).__setattr__( + "_cfg_dict", + Config._merge_a_into_b( + option_cfg_dict, cfg_dict, allow_list_keys=allow_list_keys + ), + ) + + +class DictAction(Action): + """ + argparse action to split an argument into KEY=VALUE form + on the first = and append to a dictionary. List options can + be passed as comma separated values, i.e 'KEY=V1,V2,V3', or with explicit + brackets, i.e. 'KEY=[V1,V2,V3]'. It also support nested brackets to build + list/tuple values. e.g. 'KEY=[(V1,V2),(V3,V4)]' + """ + + @staticmethod + def _parse_int_float_bool(val): + try: + return int(val) + except ValueError: + pass + try: + return float(val) + except ValueError: + pass + if val.lower() in ["true", "false"]: + return True if val.lower() == "true" else False + return val + + @staticmethod + def _parse_iterable(val): + """Parse iterable values in the string. + + All elements inside '()' or '[]' are treated as iterable values. + + Args: + val (str): Value string. + + Returns: + list | tuple: The expanded list or tuple from the string. + + Examples: + >>> DictAction._parse_iterable('1,2,3') + [1, 2, 3] + >>> DictAction._parse_iterable('[a, b, c]') + ['a', 'b', 'c'] + >>> DictAction._parse_iterable('[(1, 2, 3), [a, b], c]') + [(1, 2, 3), ['a', 'b'], 'c'] + """ + + def find_next_comma(string): + """Find the position of next comma in the string. + + If no ',' is found in the string, return the string length. All + chars inside '()' and '[]' are treated as one element and thus ',' + inside these brackets are ignored. + """ + assert (string.count("(") == string.count(")")) and ( + string.count("[") == string.count("]") + ), f"Imbalanced brackets exist in {string}" + end = len(string) + for idx, char in enumerate(string): + pre = string[:idx] + # The string before this ',' is balanced + if ( + (char == ",") + and (pre.count("(") == pre.count(")")) + and (pre.count("[") == pre.count("]")) + ): + end = idx + break + return end + + # Strip ' and " characters and replace whitespace. + val = val.strip("'\"").replace(" ", "") + is_tuple = False + if val.startswith("(") and val.endswith(")"): + is_tuple = True + val = val[1:-1] + elif val.startswith("[") and val.endswith("]"): + val = val[1:-1] + elif "," not in val: + # val is a single value + return DictAction._parse_int_float_bool(val) + + values = [] + while len(val) > 0: + comma_idx = find_next_comma(val) + element = DictAction._parse_iterable(val[:comma_idx]) + values.append(element) + val = val[comma_idx + 1 :] + if is_tuple: + values = tuple(values) + return values + + def __call__(self, parser, namespace, values, option_string=None): + options = {} + for kv in values: + key, val = kv.split("=", maxsplit=1) + options[key] = self._parse_iterable(val) + setattr(namespace, self.dest, options) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/utils/env.py b/submodules/PointTransformerV3/Pointcept/pointcept/utils/env.py new file mode 100644 index 0000000000000000000000000000000000000000..653f007dde5c4a7564e732da88dd47e7d37adf97 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/utils/env.py @@ -0,0 +1,36 @@ +""" +Environment Utils + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import random +import numpy as np +import torch +import torch.backends.cudnn as cudnn + +from datetime import datetime + + +def get_random_seed(): + seed = ( + os.getpid() + + int(datetime.now().strftime("%S%f")) + + int.from_bytes(os.urandom(2), "big") + ) + return seed + + +def set_seed(seed=None): + if seed is None: + seed = get_random_seed() + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + cudnn.benchmark = False + cudnn.deterministic = True + os.environ["PYTHONHASHSEED"] = str(seed) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/utils/events.py b/submodules/PointTransformerV3/Pointcept/pointcept/utils/events.py new file mode 100644 index 0000000000000000000000000000000000000000..718ee9191e5cce9343383baa1aae99f22c3d0734 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/utils/events.py @@ -0,0 +1,612 @@ +""" +Events Utils + +Modified from Detectron2 + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import datetime +import json +import logging +import os +import time +import torch +import numpy as np +import traceback +import sys + +from typing import List, Optional, Tuple +from collections import defaultdict +from contextlib import contextmanager + +__all__ = [ + "get_event_storage", + "JSONWriter", + "TensorboardXWriter", + "CommonMetricPrinter", + "EventStorage", + "ExceptionWriter", +] + +_CURRENT_STORAGE_STACK = [] + + +def get_event_storage(): + """ + Returns: + The :class:`EventStorage` object that's currently being used. + Throws an error if no :class:`EventStorage` is currently enabled. + """ + assert len( + _CURRENT_STORAGE_STACK + ), "get_event_storage() has to be called inside a 'with EventStorage(...)' context!" + return _CURRENT_STORAGE_STACK[-1] + + +class EventWriter: + """ + Base class for writers that obtain events from :class:`EventStorage` and process them. + """ + + def write(self): + raise NotImplementedError + + def close(self): + pass + + +class JSONWriter(EventWriter): + """ + Write scalars to a json file. + It saves scalars as one json per line (instead of a big json) for easy parsing. + Examples parsing such a json file: + :: + $ cat metrics.json | jq -s '.[0:2]' + [ + { + "data_time": 0.008433341979980469, + "iteration": 19, + "loss": 1.9228371381759644, + "loss_box_reg": 0.050025828182697296, + "loss_classifier": 0.5316952466964722, + "loss_mask": 0.7236229181289673, + "loss_rpn_box": 0.0856662318110466, + "loss_rpn_cls": 0.48198649287223816, + "lr": 0.007173333333333333, + "time": 0.25401854515075684 + }, + { + "data_time": 0.007216215133666992, + "iteration": 39, + "loss": 1.282649278640747, + "loss_box_reg": 0.06222952902317047, + "loss_classifier": 0.30682939291000366, + "loss_mask": 0.6970193982124329, + "loss_rpn_box": 0.038663312792778015, + "loss_rpn_cls": 0.1471673548221588, + "lr": 0.007706666666666667, + "time": 0.2490077018737793 + } + ] + $ cat metrics.json | jq '.loss_mask' + 0.7126231789588928 + 0.689423680305481 + 0.6776131987571716 + ... + """ + + def __init__(self, json_file, window_size=20): + """ + Args: + json_file (str): path to the json file. New data will be appended if the file exists. + window_size (int): the window size of median smoothing for the scalars whose + `smoothing_hint` are True. + """ + self._file_handle = open(json_file, "a") + self._window_size = window_size + self._last_write = -1 + + def write(self): + storage = get_event_storage() + to_save = defaultdict(dict) + + for k, (v, iter) in storage.latest_with_smoothing_hint( + self._window_size + ).items(): + # keep scalars that have not been written + if iter <= self._last_write: + continue + to_save[iter][k] = v + if len(to_save): + all_iters = sorted(to_save.keys()) + self._last_write = max(all_iters) + + for itr, scalars_per_iter in to_save.items(): + scalars_per_iter["iteration"] = itr + self._file_handle.write(json.dumps(scalars_per_iter, sort_keys=True) + "\n") + self._file_handle.flush() + try: + os.fsync(self._file_handle.fileno()) + except AttributeError: + pass + + def close(self): + self._file_handle.close() + + +class TensorboardXWriter(EventWriter): + """ + Write all scalars to a tensorboard file. + """ + + def __init__(self, log_dir: str, window_size: int = 20, **kwargs): + """ + Args: + log_dir (str): the directory to save the output events + window_size (int): the scalars will be median-smoothed by this window size + kwargs: other arguments passed to `torch.utils.tensorboard.SummaryWriter(...)` + """ + self._window_size = window_size + from torch.utils.tensorboard import SummaryWriter + + self._writer = SummaryWriter(log_dir, **kwargs) + self._last_write = -1 + + def write(self): + storage = get_event_storage() + new_last_write = self._last_write + for k, (v, iter) in storage.latest_with_smoothing_hint( + self._window_size + ).items(): + if iter > self._last_write: + self._writer.add_scalar(k, v, iter) + new_last_write = max(new_last_write, iter) + self._last_write = new_last_write + + # storage.put_{image,histogram} is only meant to be used by + # tensorboard writer. So we access its internal fields directly from here. + if len(storage._vis_data) >= 1: + for img_name, img, step_num in storage._vis_data: + self._writer.add_image(img_name, img, step_num) + # Storage stores all image data and rely on this writer to clear them. + # As a result it assumes only one writer will use its image data. + # An alternative design is to let storage store limited recent + # data (e.g. only the most recent image) that all writers can access. + # In that case a writer may not see all image data if its period is long. + storage.clear_images() + + if len(storage._histograms) >= 1: + for params in storage._histograms: + self._writer.add_histogram_raw(**params) + storage.clear_histograms() + + def close(self): + if hasattr(self, "_writer"): # doesn't exist when the code fails at import + self._writer.close() + + +class CommonMetricPrinter(EventWriter): + """ + Print **common** metrics to the terminal, including + iteration time, ETA, memory, all losses, and the learning rate. + It also applies smoothing using a window of 20 elements. + It's meant to print common metrics in common ways. + To print something in more customized ways, please implement a similar printer by yourself. + """ + + def __init__(self, max_iter: Optional[int] = None, window_size: int = 20): + """ + Args: + max_iter: the maximum number of iterations to train. + Used to compute ETA. If not given, ETA will not be printed. + window_size (int): the losses will be median-smoothed by this window size + """ + self.logger = logging.getLogger(__name__) + self._max_iter = max_iter + self._window_size = window_size + self._last_write = ( + None # (step, time) of last call to write(). Used to compute ETA + ) + + def _get_eta(self, storage) -> Optional[str]: + if self._max_iter is None: + return "" + iteration = storage.iter + try: + eta_seconds = storage.history("time").median(1000) * ( + self._max_iter - iteration - 1 + ) + storage.put_scalar("eta_seconds", eta_seconds, smoothing_hint=False) + return str(datetime.timedelta(seconds=int(eta_seconds))) + except KeyError: + # estimate eta on our own - more noisy + eta_string = None + if self._last_write is not None: + estimate_iter_time = (time.perf_counter() - self._last_write[1]) / ( + iteration - self._last_write[0] + ) + eta_seconds = estimate_iter_time * (self._max_iter - iteration - 1) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + self._last_write = (iteration, time.perf_counter()) + return eta_string + + def write(self): + storage = get_event_storage() + iteration = storage.iter + if iteration == self._max_iter: + # This hook only reports training progress (loss, ETA, etc) but not other data, + # therefore do not write anything after training succeeds, even if this method + # is called. + return + + try: + data_time = storage.history("data_time").avg(20) + except KeyError: + # they may not exist in the first few iterations (due to warmup) + # or when SimpleTrainer is not used + data_time = None + try: + iter_time = storage.history("time").global_avg() + except KeyError: + iter_time = None + try: + lr = "{:.5g}".format(storage.history("lr").latest()) + except KeyError: + lr = "N/A" + + eta_string = self._get_eta(storage) + + if torch.cuda.is_available(): + max_mem_mb = torch.cuda.max_memory_allocated() / 1024.0 / 1024.0 + else: + max_mem_mb = None + + # NOTE: max_mem is parsed by grep in "dev/parse_results.sh" + self.logger.info( + " {eta}iter: {iter} {losses} {time}{data_time}lr: {lr} {memory}".format( + eta=f"eta: {eta_string} " if eta_string else "", + iter=iteration, + losses=" ".join( + [ + "{}: {:.4g}".format(k, v.median(self._window_size)) + for k, v in storage.histories().items() + if "loss" in k + ] + ), + time=( + "time: {:.4f} ".format(iter_time) if iter_time is not None else "" + ), + data_time=( + "data_time: {:.4f} ".format(data_time) + if data_time is not None + else "" + ), + lr=lr, + memory=( + "max_mem: {:.0f}M".format(max_mem_mb) + if max_mem_mb is not None + else "" + ), + ) + ) + + +class EventStorage: + """ + The user-facing class that provides metric storage functionalities. + In the future we may add support for storing / logging other types of data if needed. + """ + + def __init__(self, start_iter=0): + """ + Args: + start_iter (int): the iteration number to start with + """ + self._history = defaultdict(AverageMeter) + self._smoothing_hints = {} + self._latest_scalars = {} + self._iter = start_iter + self._current_prefix = "" + self._vis_data = [] + self._histograms = [] + + # def put_image(self, img_name, img_tensor): + # """ + # Add an `img_tensor` associated with `img_name`, to be shown on + # tensorboard. + # Args: + # img_name (str): The name of the image to put into tensorboard. + # img_tensor (torch.Tensor or numpy.array): An `uint8` or `float` + # Tensor of shape `[channel, height, width]` where `channel` is + # 3. The image format should be RGB. The elements in img_tensor + # can either have values in [0, 1] (float32) or [0, 255] (uint8). + # The `img_tensor` will be visualized in tensorboard. + # """ + # self._vis_data.append((img_name, img_tensor, self._iter)) + + def put_scalar(self, name, value, n=1, smoothing_hint=False): + """ + Add a scalar `value` to the `HistoryBuffer` associated with `name`. + Args: + smoothing_hint (bool): a 'hint' on whether this scalar is noisy and should be + smoothed when logged. The hint will be accessible through + :meth:`EventStorage.smoothing_hints`. A writer may ignore the hint + and apply custom smoothing rule. + It defaults to True because most scalars we save need to be smoothed to + provide any useful signal. + """ + name = self._current_prefix + name + history = self._history[name] + history.update(value, n) + self._latest_scalars[name] = (value, self._iter) + + existing_hint = self._smoothing_hints.get(name) + if existing_hint is not None: + assert ( + existing_hint == smoothing_hint + ), "Scalar {} was put with a different smoothing_hint!".format(name) + else: + self._smoothing_hints[name] = smoothing_hint + + # def put_scalars(self, *, smoothing_hint=True, **kwargs): + # """ + # Put multiple scalars from keyword arguments. + # Examples: + # storage.put_scalars(loss=my_loss, accuracy=my_accuracy, smoothing_hint=True) + # """ + # for k, v in kwargs.items(): + # self.put_scalar(k, v, smoothing_hint=smoothing_hint) + # + # def put_histogram(self, hist_name, hist_tensor, bins=1000): + # """ + # Create a histogram from a tensor. + # Args: + # hist_name (str): The name of the histogram to put into tensorboard. + # hist_tensor (torch.Tensor): A Tensor of arbitrary shape to be converted + # into a histogram. + # bins (int): Number of histogram bins. + # """ + # ht_min, ht_max = hist_tensor.min().item(), hist_tensor.max().item() + # + # # Create a histogram with PyTorch + # hist_counts = torch.histc(hist_tensor, bins=bins) + # hist_edges = torch.linspace(start=ht_min, end=ht_max, steps=bins + 1, dtype=torch.float32) + # + # # Parameter for the add_histogram_raw function of SummaryWriter + # hist_params = dict( + # tag=hist_name, + # min=ht_min, + # max=ht_max, + # num=len(hist_tensor), + # sum=float(hist_tensor.sum()), + # sum_squares=float(torch.sum(hist_tensor**2)), + # bucket_limits=hist_edges[1:].tolist(), + # bucket_counts=hist_counts.tolist(), + # global_step=self._iter, + # ) + # self._histograms.append(hist_params) + + def history(self, name): + """ + Returns: + AverageMeter: the history for name + """ + ret = self._history.get(name, None) + if ret is None: + raise KeyError("No history metric available for {}!".format(name)) + return ret + + def histories(self): + """ + Returns: + dict[name -> HistoryBuffer]: the HistoryBuffer for all scalars + """ + return self._history + + def latest(self): + """ + Returns: + dict[str -> (float, int)]: mapping from the name of each scalar to the most + recent value and the iteration number its added. + """ + return self._latest_scalars + + def latest_with_smoothing_hint(self, window_size=20): + """ + Similar to :meth:`latest`, but the returned values + are either the un-smoothed original latest value, + or a median of the given window_size, + depend on whether the smoothing_hint is True. + This provides a default behavior that other writers can use. + """ + result = {} + for k, (v, itr) in self._latest_scalars.items(): + result[k] = ( + self._history[k].median(window_size) if self._smoothing_hints[k] else v, + itr, + ) + return result + + def smoothing_hints(self): + """ + Returns: + dict[name -> bool]: the user-provided hint on whether the scalar + is noisy and needs smoothing. + """ + return self._smoothing_hints + + def step(self): + """ + User should either: (1) Call this function to increment storage.iter when needed. Or + (2) Set `storage.iter` to the correct iteration number before each iteration. + The storage will then be able to associate the new data with an iteration number. + """ + self._iter += 1 + + @property + def iter(self): + """ + Returns: + int: The current iteration number. When used together with a trainer, + this is ensured to be the same as trainer.iter. + """ + return self._iter + + @iter.setter + def iter(self, val): + self._iter = int(val) + + @property + def iteration(self): + # for backward compatibility + return self._iter + + def __enter__(self): + _CURRENT_STORAGE_STACK.append(self) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + assert _CURRENT_STORAGE_STACK[-1] == self + _CURRENT_STORAGE_STACK.pop() + + @contextmanager + def name_scope(self, name): + """ + Yields: + A context within which all the events added to this storage + will be prefixed by the name scope. + """ + old_prefix = self._current_prefix + self._current_prefix = name.rstrip("/") + "/" + yield + self._current_prefix = old_prefix + + def clear_images(self): + """ + Delete all the stored images for visualization. This should be called + after images are written to tensorboard. + """ + self._vis_data = [] + + def clear_histograms(self): + """ + Delete all the stored histograms for visualization. + This should be called after histograms are written to tensorboard. + """ + self._histograms = [] + + def reset_history(self, name): + ret = self._history.get(name, None) + if ret is None: + raise KeyError("No history metric available for {}!".format(name)) + ret.reset() + + def reset_histories(self): + for name in self._history.keys(): + self._history[name].reset() + + +class AverageMeter: + """Computes and stores the average and current value""" + + def __init__(self): + self.val = 0 + self.avg = 0 + self.total = 0 + self.count = 0 + + def reset(self): + self.val = 0 + self.avg = 0 + self.total = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.total += val * n + self.count += n + self.avg = self.total / self.count + + +class HistoryBuffer: + """ + Track a series of scalar values and provide access to smoothed values over a + window or the global average of the series. + """ + + def __init__(self, max_length: int = 1000000) -> None: + """ + Args: + max_length: maximal number of values that can be stored in the + buffer. When the capacity of the buffer is exhausted, old + values will be removed. + """ + self._max_length: int = max_length + self._data: List[Tuple[float, float]] = [] # (value, iteration) pairs + self._count: int = 0 + self._global_avg: float = 0 + + def update(self, value: float, iteration: Optional[float] = None) -> None: + """ + Add a new scalar value produced at certain iteration. If the length + of the buffer exceeds self._max_length, the oldest element will be + removed from the buffer. + """ + if iteration is None: + iteration = self._count + if len(self._data) == self._max_length: + self._data.pop(0) + self._data.append((value, iteration)) + + self._count += 1 + self._global_avg += (value - self._global_avg) / self._count + + def latest(self) -> float: + """ + Return the latest scalar value added to the buffer. + """ + return self._data[-1][0] + + def median(self, window_size: int) -> float: + """ + Return the median of the latest `window_size` values in the buffer. + """ + return np.median([x[0] for x in self._data[-window_size:]]) + + def avg(self, window_size: int) -> float: + """ + Return the mean of the latest `window_size` values in the buffer. + """ + return np.mean([x[0] for x in self._data[-window_size:]]) + + def global_avg(self) -> float: + """ + Return the mean of all the elements in the buffer. Note that this + includes those getting removed due to limited buffer storage. + """ + return self._global_avg + + def values(self) -> List[Tuple[float, float]]: + """ + Returns: + list[(number, iteration)]: content of the current buffer. + """ + return self._data + + +class ExceptionWriter: + + def __init__(self): + self.logger = logging.getLogger(__name__) + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type: + tb = traceback.format_exception(exc_type, exc_val, exc_tb) + formatted_tb_str = "".join(tb) + self.logger.error(formatted_tb_str) + sys.exit(1) # This prevents double logging the error to the console diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/utils/logger.py b/submodules/PointTransformerV3/Pointcept/pointcept/utils/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..ddaf2c5a765c9f1325737c3cbc73e1169f13cdd4 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/utils/logger.py @@ -0,0 +1,172 @@ +""" +Logger Utils + +Modified from mmcv + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import logging +import torch +import torch.distributed as dist + +from termcolor import colored + +logger_initialized = {} +root_status = 0 + + +class _ColorfulFormatter(logging.Formatter): + def __init__(self, *args, **kwargs): + self._root_name = kwargs.pop("root_name") + "." + super(_ColorfulFormatter, self).__init__(*args, **kwargs) + + def formatMessage(self, record): + log = super(_ColorfulFormatter, self).formatMessage(record) + if record.levelno == logging.WARNING: + prefix = colored("WARNING", "red", attrs=["blink"]) + elif record.levelno == logging.ERROR or record.levelno == logging.CRITICAL: + prefix = colored("ERROR", "red", attrs=["blink", "underline"]) + else: + return log + return prefix + " " + log + + +def get_logger(name, log_file=None, log_level=logging.INFO, file_mode="a", color=False): + """Initialize and get a logger by name. + + If the logger has not been initialized, this method will initialize the + logger by adding one or two handlers, otherwise the initialized logger will + be directly returned. During initialization, a StreamHandler will always be + added. If `log_file` is specified and the process rank is 0, a FileHandler + will also be added. + + Args: + name (str): Logger name. + log_file (str | None): The log filename. If specified, a FileHandler + will be added to the logger. + log_level (int): The logger level. Note that only the process of + rank 0 is affected, and other processes will set the level to + "Error" thus be silent most of the time. + file_mode (str): The file mode used in opening log file. + Defaults to 'a'. + color (bool): Colorful log output. Defaults to True + + Returns: + logging.Logger: The expected logger. + """ + logger = logging.getLogger(name) + + if name in logger_initialized: + return logger + # handle hierarchical names + # e.g., logger "a" is initialized, then logger "a.b" will skip the + # initialization since it is a child of "a". + for logger_name in logger_initialized: + if name.startswith(logger_name): + return logger + + logger.propagate = False + + stream_handler = logging.StreamHandler() + handlers = [stream_handler] + + if dist.is_available() and dist.is_initialized(): + rank = dist.get_rank() + else: + rank = 0 + + # only rank 0 will add a FileHandler + if rank == 0 and log_file is not None: + # Here, the default behaviour of the official logger is 'a'. Thus, we + # provide an interface to change the file mode to the default + # behaviour. + file_handler = logging.FileHandler(log_file, file_mode) + handlers.append(file_handler) + + plain_formatter = logging.Formatter( + "[%(asctime)s %(levelname)s %(filename)s line %(lineno)d %(process)d] %(message)s" + ) + if color: + formatter = _ColorfulFormatter( + colored("[%(asctime)s %(name)s]: ", "green") + "%(message)s", + datefmt="%m/%d %H:%M:%S", + root_name=name, + ) + else: + formatter = plain_formatter + for handler in handlers: + handler.setFormatter(formatter) + handler.setLevel(log_level) + logger.addHandler(handler) + + if rank == 0: + logger.setLevel(log_level) + else: + logger.setLevel(logging.ERROR) + + logger_initialized[name] = True + + return logger + + +def print_log(msg, logger=None, level=logging.INFO): + """Print a log message. + + Args: + msg (str): The message to be logged. + logger (logging.Logger | str | None): The logger to be used. + Some special loggers are: + - "silent": no message will be printed. + - other str: the logger obtained with `get_root_logger(logger)`. + - None: The `print()` method will be used to print log messages. + level (int): Logging level. Only available when `logger` is a Logger + object or "root". + """ + if logger is None: + print(msg) + elif isinstance(logger, logging.Logger): + logger.log(level, msg) + elif logger == "silent": + pass + elif isinstance(logger, str): + _logger = get_logger(logger) + _logger.log(level, msg) + else: + raise TypeError( + "logger should be either a logging.Logger object, str, " + f'"silent" or None, but got {type(logger)}' + ) + + +def get_root_logger(log_file=None, log_level=logging.INFO, file_mode="a"): + """Get the root logger. + + The logger will be initialized if it has not been initialized. By default a + StreamHandler will be added. If `log_file` is specified, a FileHandler will + also be added. The name of the root logger is the top-level package name. + + Args: + log_file (str | None): The log filename. If specified, a FileHandler + will be added to the root logger. + log_level (int): The root logger level. Note that only the process of + rank 0 is affected, while other processes will set the level to + "Error" and be silent most of the time. + file_mode (str): File Mode of logger. (w or a) + + Returns: + logging.Logger: The root logger. + """ + logger = get_logger( + name="pointcept", log_file=log_file, log_level=log_level, file_mode=file_mode + ) + return logger + + +def _log_api_usage(identifier: str): + """ + Internal function used to log the usage of different detectron2 components + inside facebook's infra. + """ + torch._C._log_api_usage_once("pointcept." + identifier) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/utils/misc.py b/submodules/PointTransformerV3/Pointcept/pointcept/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..3177bae3882ccad347002165d2b34d5dc2540359 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/utils/misc.py @@ -0,0 +1,164 @@ +""" +Misc + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import warnings +from collections import abc +import numpy as np +import torch +from importlib import import_module + + +class AverageMeter(object): + """Computes and stores the average and current value""" + + def __init__(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + +def intersection_and_union(output, target, K, ignore_index=-1): + # 'K' classes, output and target sizes are N or N * L or N * H * W, each value in range 0 to K - 1. + assert output.ndim in [1, 2, 3] + assert output.shape == target.shape + output = output.reshape(output.size).copy() + target = target.reshape(target.size) + output[np.where(target == ignore_index)[0]] = ignore_index + intersection = output[np.where(output == target)[0]] + area_intersection, _ = np.histogram(intersection, bins=np.arange(K + 1)) + area_output, _ = np.histogram(output, bins=np.arange(K + 1)) + area_target, _ = np.histogram(target, bins=np.arange(K + 1)) + area_union = area_output + area_target - area_intersection + return area_intersection, area_union, area_target + + +def intersection_and_union_gpu(output, target, k, ignore_index=-1): + # 'K' classes, output and target sizes are N or N * L or N * H * W, each value in range 0 to K - 1. + assert output.dim() in [1, 2, 3] + assert output.shape == target.shape + output = output.view(-1) + target = target.view(-1) + output[target == ignore_index] = ignore_index + intersection = output[output == target] + area_intersection = torch.histc(intersection, bins=k, min=0, max=k - 1) + area_output = torch.histc(output, bins=k, min=0, max=k - 1) + area_target = torch.histc(target, bins=k, min=0, max=k - 1) + area_union = area_output + area_target - area_intersection + return area_intersection, area_union, area_target + + +def make_dirs(dir_name): + if not os.path.exists(dir_name): + os.makedirs(dir_name, exist_ok=True) + + +def find_free_port(): + import socket + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Binding to port 0 will cause the OS to find an available port for us + sock.bind(("", 0)) + port = sock.getsockname()[1] + sock.close() + # NOTE: there is still a chance the port could be taken by other processes. + return port + + +def is_seq_of(seq, expected_type, seq_type=None): + """Check whether it is a sequence of some type. + + Args: + seq (Sequence): The sequence to be checked. + expected_type (type): Expected type of sequence items. + seq_type (type, optional): Expected sequence type. + + Returns: + bool: Whether the sequence is valid. + """ + if seq_type is None: + exp_seq_type = abc.Sequence + else: + assert isinstance(seq_type, type) + exp_seq_type = seq_type + if not isinstance(seq, exp_seq_type): + return False + for item in seq: + if not isinstance(item, expected_type): + return False + return True + + +def is_str(x): + """Whether the input is an string instance. + + Note: This method is deprecated since python 2 is no longer supported. + """ + return isinstance(x, str) + + +def import_modules_from_strings(imports, allow_failed_imports=False): + """Import modules from the given list of strings. + + Args: + imports (list | str | None): The given module names to be imported. + allow_failed_imports (bool): If True, the failed imports will return + None. Otherwise, an ImportError is raise. Default: False. + + Returns: + list[module] | module | None: The imported modules. + + Examples: + >>> osp, sys = import_modules_from_strings( + ... ['os.path', 'sys']) + >>> import os.path as osp_ + >>> import sys as sys_ + >>> assert osp == osp_ + >>> assert sys == sys_ + """ + if not imports: + return + single_import = False + if isinstance(imports, str): + single_import = True + imports = [imports] + if not isinstance(imports, list): + raise TypeError(f"custom_imports must be a list but got type {type(imports)}") + imported = [] + for imp in imports: + if not isinstance(imp, str): + raise TypeError(f"{imp} is of type {type(imp)} and cannot be imported.") + try: + imported_tmp = import_module(imp) + except ImportError: + if allow_failed_imports: + warnings.warn(f"{imp} failed to import and is ignored.", UserWarning) + imported_tmp = None + else: + raise ImportError + imported.append(imported_tmp) + if single_import: + imported = imported[0] + return imported + + +class DummyClass: + def __init__(self): + pass diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/utils/optimizer.py b/submodules/PointTransformerV3/Pointcept/pointcept/utils/optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..355ec8916ad041ca02b404029983b0f59933fb8c --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/utils/optimizer.py @@ -0,0 +1,55 @@ +""" +Optimizer + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import torch +from pointcept.utils.logger import get_root_logger +from pointcept.utils.registry import Registry + +OPTIMIZERS = Registry("optimizers") + + +OPTIMIZERS.register_module(module=torch.optim.SGD, name="SGD") +OPTIMIZERS.register_module(module=torch.optim.Adam, name="Adam") +OPTIMIZERS.register_module(module=torch.optim.AdamW, name="AdamW") + + +def build_optimizer(cfg, model, param_dicts=None): + if param_dicts is None: + cfg.params = model.parameters() + else: + cfg.params = [dict(names=[], params=[], lr=cfg.lr)] + for i in range(len(param_dicts)): + param_group = dict(names=[], params=[]) + if "lr" in param_dicts[i].keys(): + param_group["lr"] = param_dicts[i].lr + if "momentum" in param_dicts[i].keys(): + param_group["momentum"] = param_dicts[i].momentum + if "weight_decay" in param_dicts[i].keys(): + param_group["weight_decay"] = param_dicts[i].weight_decay + cfg.params.append(param_group) + + for n, p in model.named_parameters(): + flag = False + for i in range(len(param_dicts)): + if param_dicts[i].keyword in n: + cfg.params[i + 1]["names"].append(n) + cfg.params[i + 1]["params"].append(p) + flag = True + break + if not flag: + cfg.params[0]["names"].append(n) + cfg.params[0]["params"].append(p) + + logger = get_root_logger() + for i in range(len(cfg.params)): + param_names = cfg.params[i].pop("names") + message = "" + for key in cfg.params[i].keys(): + if key != "params": + message += f" {key}: {cfg.params[i][key]};" + logger.info(f"Params Group {i+1} -{message} Params: {param_names}.") + return OPTIMIZERS.build(cfg=cfg) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/utils/path.py b/submodules/PointTransformerV3/Pointcept/pointcept/utils/path.py new file mode 100644 index 0000000000000000000000000000000000000000..ce98fa5fd0dfbf6e1d61e833ecc35fea4ab2782b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/utils/path.py @@ -0,0 +1,103 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os +import os.path as osp +from pathlib import Path + +from .misc import is_str + + +def is_filepath(x): + return is_str(x) or isinstance(x, Path) + + +def fopen(filepath, *args, **kwargs): + if is_str(filepath): + return open(filepath, *args, **kwargs) + elif isinstance(filepath, Path): + return filepath.open(*args, **kwargs) + raise ValueError("`filepath` should be a string or a Path") + + +def check_file_exist(filename, msg_tmpl='file "{}" does not exist'): + if not osp.isfile(filename): + raise FileNotFoundError(msg_tmpl.format(filename)) + + +def mkdir_or_exist(dir_name, mode=0o777): + if dir_name == "": + return + dir_name = osp.expanduser(dir_name) + os.makedirs(dir_name, mode=mode, exist_ok=True) + + +def symlink(src, dst, overwrite=True, **kwargs): + if os.path.lexists(dst) and overwrite: + os.remove(dst) + os.symlink(src, dst, **kwargs) + + +def scandir(dir_path, suffix=None, recursive=False, case_sensitive=True): + """Scan a directory to find the interested files. + + Args: + dir_path (str | obj:`Path`): Path of the directory. + suffix (str | tuple(str), optional): File suffix that we are + interested in. Default: None. + recursive (bool, optional): If set to True, recursively scan the + directory. Default: False. + case_sensitive (bool, optional) : If set to False, ignore the case of + suffix. Default: True. + + Returns: + A generator for all the interested files with relative paths. + """ + if isinstance(dir_path, (str, Path)): + dir_path = str(dir_path) + else: + raise TypeError('"dir_path" must be a string or Path object') + + if (suffix is not None) and not isinstance(suffix, (str, tuple)): + raise TypeError('"suffix" must be a string or tuple of strings') + + if suffix is not None and not case_sensitive: + suffix = ( + suffix.lower() + if isinstance(suffix, str) + else tuple(item.lower() for item in suffix) + ) + + root = dir_path + + def _scandir(dir_path, suffix, recursive, case_sensitive): + for entry in os.scandir(dir_path): + if not entry.name.startswith(".") and entry.is_file(): + rel_path = osp.relpath(entry.path, root) + _rel_path = rel_path if case_sensitive else rel_path.lower() + if suffix is None or _rel_path.endswith(suffix): + yield rel_path + elif recursive and os.path.isdir(entry.path): + # scan recursively if entry.path is a directory + yield from _scandir(entry.path, suffix, recursive, case_sensitive) + + return _scandir(dir_path, suffix, recursive, case_sensitive) + + +def find_vcs_root(path, markers=(".git",)): + """Finds the root directory (including itself) of specified markers. + + Args: + path (str): Path of directory or file. + markers (list[str], optional): List of file or directory names. + + Returns: + The directory contained one of the markers or None if not found. + """ + if osp.isfile(path): + path = osp.dirname(path) + + prev, cur = None, osp.abspath(osp.expanduser(path)) + while cur != prev: + if any(osp.exists(osp.join(cur, marker)) for marker in markers): + return cur + prev, cur = cur, osp.split(cur)[0] + return None diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/utils/registry.py b/submodules/PointTransformerV3/Pointcept/pointcept/utils/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..7ac308a87d38ff61da14d6b4d5c73b4c68c15a58 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/utils/registry.py @@ -0,0 +1,316 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import inspect +import warnings +from functools import partial + +from .misc import is_seq_of + + +def build_from_cfg(cfg, registry, default_args=None): + """Build a module from configs dict. + + Args: + cfg (dict): Config dict. It should at least contain the key "type". + registry (:obj:`Registry`): The registry to search the type from. + default_args (dict, optional): Default initialization arguments. + + Returns: + object: The constructed object. + """ + if not isinstance(cfg, dict): + raise TypeError(f"cfg must be a dict, but got {type(cfg)}") + if "type" not in cfg: + if default_args is None or "type" not in default_args: + raise KeyError( + '`cfg` or `default_args` must contain the key "type", ' + f"but got {cfg}\n{default_args}" + ) + if not isinstance(registry, Registry): + raise TypeError( + "registry must be an mmcv.Registry object, " f"but got {type(registry)}" + ) + if not (isinstance(default_args, dict) or default_args is None): + raise TypeError( + "default_args must be a dict or None, " f"but got {type(default_args)}" + ) + + args = cfg.copy() + + if default_args is not None: + for name, value in default_args.items(): + args.setdefault(name, value) + + obj_type = args.pop("type") + if isinstance(obj_type, str): + obj_cls = registry.get(obj_type) + if obj_cls is None: + raise KeyError(f"{obj_type} is not in the {registry.name} registry") + elif inspect.isclass(obj_type): + obj_cls = obj_type + else: + raise TypeError(f"type must be a str or valid type, but got {type(obj_type)}") + try: + return obj_cls(**args) + except Exception as e: + # Normal TypeError does not print class name. + raise type(e)(f"{obj_cls.__name__}: {e}") + + +class Registry: + """A registry to map strings to classes. + + Registered object could be built from registry. + Example: + >>> MODELS = Registry('models') + >>> @MODELS.register_module() + >>> class ResNet: + >>> pass + >>> resnet = MODELS.build(dict(type='ResNet')) + + Please refer to + https://mmcv.readthedocs.io/en/latest/understand_mmcv/registry.html for + advanced usage. + + Args: + name (str): Registry name. + build_func(func, optional): Build function to construct instance from + Registry, func:`build_from_cfg` is used if neither ``parent`` or + ``build_func`` is specified. If ``parent`` is specified and + ``build_func`` is not given, ``build_func`` will be inherited + from ``parent``. Default: None. + parent (Registry, optional): Parent registry. The class registered in + children registry could be built from parent. Default: None. + scope (str, optional): The scope of registry. It is the key to search + for children registry. If not specified, scope will be the name of + the package where class is defined, e.g. mmdet, mmcls, mmseg. + Default: None. + """ + + def __init__(self, name, build_func=None, parent=None, scope=None): + self._name = name + self._module_dict = dict() + self._children = dict() + self._scope = self.infer_scope() if scope is None else scope + + # self.build_func will be set with the following priority: + # 1. build_func + # 2. parent.build_func + # 3. build_from_cfg + if build_func is None: + if parent is not None: + self.build_func = parent.build_func + else: + self.build_func = build_from_cfg + else: + self.build_func = build_func + if parent is not None: + assert isinstance(parent, Registry) + parent._add_children(self) + self.parent = parent + else: + self.parent = None + + def __len__(self): + return len(self._module_dict) + + def __contains__(self, key): + return self.get(key) is not None + + def __repr__(self): + format_str = ( + self.__class__.__name__ + f"(name={self._name}, " + f"items={self._module_dict})" + ) + return format_str + + @staticmethod + def infer_scope(): + """Infer the scope of registry. + + The name of the package where registry is defined will be returned. + + Example: + # in mmdet/models/backbone/resnet.py + >>> MODELS = Registry('models') + >>> @MODELS.register_module() + >>> class ResNet: + >>> pass + The scope of ``ResNet`` will be ``mmdet``. + + + Returns: + scope (str): The inferred scope name. + """ + # inspect.stack() trace where this function is called, the index-2 + # indicates the frame where `infer_scope()` is called + filename = inspect.getmodule(inspect.stack()[2][0]).__name__ + split_filename = filename.split(".") + return split_filename[0] + + @staticmethod + def split_scope_key(key): + """Split scope and key. + + The first scope will be split from key. + + Examples: + >>> Registry.split_scope_key('mmdet.ResNet') + 'mmdet', 'ResNet' + >>> Registry.split_scope_key('ResNet') + None, 'ResNet' + + Return: + scope (str, None): The first scope. + key (str): The remaining key. + """ + split_index = key.find(".") + if split_index != -1: + return key[:split_index], key[split_index + 1 :] + else: + return None, key + + @property + def name(self): + return self._name + + @property + def scope(self): + return self._scope + + @property + def module_dict(self): + return self._module_dict + + @property + def children(self): + return self._children + + def get(self, key): + """Get the registry record. + + Args: + key (str): The class name in string format. + + Returns: + class: The corresponding class. + """ + scope, real_key = self.split_scope_key(key) + if scope is None or scope == self._scope: + # get from self + if real_key in self._module_dict: + return self._module_dict[real_key] + else: + # get from self._children + if scope in self._children: + return self._children[scope].get(real_key) + else: + # goto root + parent = self.parent + while parent.parent is not None: + parent = parent.parent + return parent.get(key) + + def build(self, *args, **kwargs): + return self.build_func(*args, **kwargs, registry=self) + + def _add_children(self, registry): + """Add children for a registry. + + The ``registry`` will be added as children based on its scope. + The parent registry could build objects from children registry. + + Example: + >>> models = Registry('models') + >>> mmdet_models = Registry('models', parent=models) + >>> @mmdet_models.register_module() + >>> class ResNet: + >>> pass + >>> resnet = models.build(dict(type='mmdet.ResNet')) + """ + + assert isinstance(registry, Registry) + assert registry.scope is not None + assert ( + registry.scope not in self.children + ), f"scope {registry.scope} exists in {self.name} registry" + self.children[registry.scope] = registry + + def _register_module(self, module_class, module_name=None, force=False): + if not inspect.isclass(module_class): + raise TypeError("module must be a class, " f"but got {type(module_class)}") + + if module_name is None: + module_name = module_class.__name__ + if isinstance(module_name, str): + module_name = [module_name] + for name in module_name: + if not force and name in self._module_dict: + raise KeyError(f"{name} is already registered " f"in {self.name}") + self._module_dict[name] = module_class + + def deprecated_register_module(self, cls=None, force=False): + warnings.warn( + "The old API of register_module(module, force=False) " + "is deprecated and will be removed, please use the new API " + "register_module(name=None, force=False, module=None) instead." + ) + if cls is None: + return partial(self.deprecated_register_module, force=force) + self._register_module(cls, force=force) + return cls + + def register_module(self, name=None, force=False, module=None): + """Register a module. + + A record will be added to `self._module_dict`, whose key is the class + name or the specified name, and value is the class itself. + It can be used as a decorator or a normal function. + + Example: + >>> backbones = Registry('backbone') + >>> @backbones.register_module() + >>> class ResNet: + >>> pass + + >>> backbones = Registry('backbone') + >>> @backbones.register_module(name='mnet') + >>> class MobileNet: + >>> pass + + >>> backbones = Registry('backbone') + >>> class ResNet: + >>> pass + >>> backbones.register_module(ResNet) + + Args: + name (str | None): The module name to be registered. If not + specified, the class name will be used. + force (bool, optional): Whether to override an existing class with + the same name. Default: False. + module (type): Module class to be registered. + """ + if not isinstance(force, bool): + raise TypeError(f"force must be a boolean, but got {type(force)}") + # NOTE: This is a walkaround to be compatible with the old api, + # while it may introduce unexpected bugs. + if isinstance(name, type): + return self.deprecated_register_module(name, force=force) + + # raise the error ahead of time + if not (name is None or isinstance(name, str) or is_seq_of(name, str)): + raise TypeError( + "name must be either of None, an instance of str or a sequence" + f" of str, but got {type(name)}" + ) + + # use it as a normal method: x.register_module(module=SomeClass) + if module is not None: + self._register_module(module_class=module, module_name=name, force=force) + return module + + # use it as a decorator: @x.register_module() + def _register(cls): + self._register_module(module_class=cls, module_name=name, force=force) + return cls + + return _register diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/utils/scheduler.py b/submodules/PointTransformerV3/Pointcept/pointcept/utils/scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..3e2e29fdde2e2668c023af36afdb89e73fb9ce53 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/utils/scheduler.py @@ -0,0 +1,147 @@ +""" +Scheduler + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import torch.optim.lr_scheduler as lr_scheduler +from .registry import Registry + +SCHEDULERS = Registry("schedulers") + + +@SCHEDULERS.register_module() +class MultiStepLR(lr_scheduler.MultiStepLR): + def __init__( + self, + optimizer, + milestones, + total_steps, + gamma=0.1, + last_epoch=-1, + verbose=False, + ): + super().__init__( + optimizer=optimizer, + milestones=[rate * total_steps for rate in milestones], + gamma=gamma, + last_epoch=last_epoch, + verbose=verbose, + ) + + +@SCHEDULERS.register_module() +class MultiStepWithWarmupLR(lr_scheduler.LambdaLR): + def __init__( + self, + optimizer, + milestones, + total_steps, + gamma=0.1, + warmup_rate=0.05, + warmup_scale=1e-6, + last_epoch=-1, + verbose=False, + ): + milestones = [rate * total_steps for rate in milestones] + + def multi_step_with_warmup(s): + factor = 1.0 + for i in range(len(milestones)): + if s < milestones[i]: + break + factor *= gamma + + if s <= warmup_rate * total_steps: + warmup_coefficient = 1 - (1 - s / warmup_rate / total_steps) * ( + 1 - warmup_scale + ) + else: + warmup_coefficient = 1.0 + return warmup_coefficient * factor + + super().__init__( + optimizer=optimizer, + lr_lambda=multi_step_with_warmup, + last_epoch=last_epoch, + verbose=verbose, + ) + + +@SCHEDULERS.register_module() +class PolyLR(lr_scheduler.LambdaLR): + def __init__(self, optimizer, total_steps, power=0.9, last_epoch=-1, verbose=False): + super().__init__( + optimizer=optimizer, + lr_lambda=lambda s: (1 - s / (total_steps + 1)) ** power, + last_epoch=last_epoch, + verbose=verbose, + ) + + +@SCHEDULERS.register_module() +class ExpLR(lr_scheduler.LambdaLR): + def __init__(self, optimizer, total_steps, gamma=0.9, last_epoch=-1, verbose=False): + super().__init__( + optimizer=optimizer, + lr_lambda=lambda s: gamma ** (s / total_steps), + last_epoch=last_epoch, + verbose=verbose, + ) + + +@SCHEDULERS.register_module() +class CosineAnnealingLR(lr_scheduler.CosineAnnealingLR): + def __init__(self, optimizer, total_steps, eta_min=0, last_epoch=-1, verbose=False): + super().__init__( + optimizer=optimizer, + T_max=total_steps, + eta_min=eta_min, + last_epoch=last_epoch, + verbose=verbose, + ) + + +@SCHEDULERS.register_module() +class OneCycleLR(lr_scheduler.OneCycleLR): + r""" + torch.optim.lr_scheduler.OneCycleLR, Block total_steps + """ + + def __init__( + self, + optimizer, + max_lr, + total_steps=None, + pct_start=0.3, + anneal_strategy="cos", + cycle_momentum=True, + base_momentum=0.85, + max_momentum=0.95, + div_factor=25.0, + final_div_factor=1e4, + three_phase=False, + last_epoch=-1, + verbose=False, + ): + super().__init__( + optimizer=optimizer, + max_lr=max_lr, + total_steps=total_steps, + pct_start=pct_start, + anneal_strategy=anneal_strategy, + cycle_momentum=cycle_momentum, + base_momentum=base_momentum, + max_momentum=max_momentum, + div_factor=div_factor, + final_div_factor=final_div_factor, + three_phase=three_phase, + last_epoch=last_epoch, + verbose=verbose, + ) + + +def build_scheduler(cfg, optimizer): + cfg.optimizer = optimizer + return SCHEDULERS.build(cfg=cfg) diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/utils/timer.py b/submodules/PointTransformerV3/Pointcept/pointcept/utils/timer.py new file mode 100644 index 0000000000000000000000000000000000000000..3de4a16e33c43fe61ea3088f82216fd62eb6e959 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/utils/timer.py @@ -0,0 +1,70 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# -*- coding: utf-8 -*- + +from time import perf_counter +from typing import Optional + + +class Timer: + """ + A timer which computes the time elapsed since the start/reset of the timer. + """ + + def __init__(self) -> None: + self.reset() + + def reset(self) -> None: + """ + Reset the timer. + """ + self._start = perf_counter() + self._paused: Optional[float] = None + self._total_paused = 0 + self._count_start = 1 + + def pause(self) -> None: + """ + Pause the timer. + """ + if self._paused is not None: + raise ValueError("Trying to pause a Timer that is already paused!") + self._paused = perf_counter() + + def is_paused(self) -> bool: + """ + Returns: + bool: whether the timer is currently paused + """ + return self._paused is not None + + def resume(self) -> None: + """ + Resume the timer. + """ + if self._paused is None: + raise ValueError("Trying to resume a Timer that is not paused!") + # pyre-fixme[58]: `-` is not supported for operand types `float` and + # `Optional[float]`. + self._total_paused += perf_counter() - self._paused + self._paused = None + self._count_start += 1 + + def seconds(self) -> float: + """ + Returns: + (float): the total number of seconds since the start/reset of the + timer, excluding the time when the timer is paused. + """ + if self._paused is not None: + end_time: float = self._paused # type: ignore + else: + end_time = perf_counter() + return end_time - self._start - self._total_paused + + def avg_seconds(self) -> float: + """ + Returns: + (float): the average number of seconds between every start/reset and + pause. + """ + return self.seconds() / self._count_start diff --git a/submodules/PointTransformerV3/Pointcept/pointcept/utils/visualization.py b/submodules/PointTransformerV3/Pointcept/pointcept/utils/visualization.py new file mode 100644 index 0000000000000000000000000000000000000000..7a010dd8289f60119d1bfbccdff65edb908e24f6 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/pointcept/utils/visualization.py @@ -0,0 +1,89 @@ +""" +Visualization Utils + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import open3d as o3d +import numpy as np +import torch + + +def to_numpy(x): + if isinstance(x, torch.Tensor): + x = x.clone().detach().cpu().numpy() + assert isinstance(x, np.ndarray) + return x + + +def save_point_cloud(coord, color=None, file_path="pc.ply", logger=None): + os.makedirs(os.path.dirname(file_path), exist_ok=True) + coord = to_numpy(coord) + if color is not None: + color = to_numpy(color) + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(coord) + pcd.colors = o3d.utility.Vector3dVector( + np.ones_like(coord) if color is None else color + ) + o3d.io.write_point_cloud(file_path, pcd) + if logger is not None: + logger.info(f"Save Point Cloud to: {file_path}") + + +def save_bounding_boxes( + bboxes_corners, color=(1.0, 0.0, 0.0), file_path="bbox.ply", logger=None +): + bboxes_corners = to_numpy(bboxes_corners) + # point list + points = bboxes_corners.reshape(-1, 3) + # line list + box_lines = np.array( + [ + [0, 1], + [1, 2], + [2, 3], + [3, 0], + [4, 5], + [5, 6], + [6, 7], + [7, 0], + [0, 4], + [1, 5], + [2, 6], + [3, 7], + ] + ) + lines = [] + for i, _ in enumerate(bboxes_corners): + lines.append(box_lines + i * 8) + lines = np.concatenate(lines) + # color list + color = np.array([color for _ in range(len(lines))]) + # generate line set + line_set = o3d.geometry.LineSet() + line_set.points = o3d.utility.Vector3dVector(points) + line_set.lines = o3d.utility.Vector2iVector(lines) + line_set.colors = o3d.utility.Vector3dVector(color) + o3d.io.write_line_set(file_path, line_set) + + if logger is not None: + logger.info(f"Save Boxes to: {file_path}") + + +def save_lines( + points, lines, color=(1.0, 0.0, 0.0), file_path="lines.ply", logger=None +): + points = to_numpy(points) + lines = to_numpy(lines) + colors = np.array([color for _ in range(len(lines))]) + line_set = o3d.geometry.LineSet() + line_set.points = o3d.utility.Vector3dVector(points) + line_set.lines = o3d.utility.Vector2iVector(lines) + line_set.colors = o3d.utility.Vector3dVector(colors) + o3d.io.write_line_set(file_path, line_set) + + if logger is not None: + logger.info(f"Save Lines to: {file_path}") diff --git a/submodules/PointTransformerV3/Pointcept/scripts/build_image.sh b/submodules/PointTransformerV3/Pointcept/scripts/build_image.sh new file mode 100644 index 0000000000000000000000000000000000000000..31a6a7fc23e57b3b738450d5c42fed4cc45b9b65 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/scripts/build_image.sh @@ -0,0 +1,83 @@ +TORCH_VERSION=2.0.1 +CUDA_VERSION=11.7 +CUDNN_VERSION=8 + +ARGS=`getopt -o t:c: -l torch:,cuda:,cudnn: -n "$0" -- "$@"` +[ $? != 0 ] && exit 1 +eval set -- "${ARGS}" +while true ; do + case "$1" in + -t | --torch) + TORCH_VERSION=$2 + shift 2 + ;; + -c | --cuda) + CUDA_VERSION=$2 + shift 2 + ;; + --cudnn) + CUDNN_VERSION=$2 + shift 2 + ;; + --) + break + ;; + *) + echo "Invalid option: $1" + exit 1 + ;; + esac +done + +CUDA_VERSION_NO_DOT=`echo ${CUDA_VERSION} | tr -d "."` +BASE_TORCH_TAG=${TORCH_VERSION}-cuda${CUDA_VERSION}-cudnn${CUDNN_VERSION}-devel +IMG_TAG=pointcept/pointcept:pytorch${BASE_TORCH_TAG} + +echo "TORCH VERSION: ${TORCH_VERSION}" +echo "CUDA VERSION: ${CUDA_VERSION}" +echo "CUDNN VERSION: ${CUDNN_VERSION}" + + +cat > ./Dockerfile <<- EOM +FROM pytorch/pytorch:${BASE_TORCH_TAG} + +# Fix nvidia-key error issue (NO_PUBKEY A4B469963BF863CC) +RUN rm /etc/apt/sources.list.d/*.list + +# Installing apt packages +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt -y update --no-install-recommends \ + && apt -y install --no-install-recommends \ + git wget tmux vim zsh build-essential cmake ninja-build libopenblas-dev libsparsehash-dev \ + && apt autoremove -y \ + && apt clean -y \ + && export DEBIAN_FRONTEND=dialog + +# Install Pointcept environment +RUN conda install h5py pyyaml -c anaconda -y +RUN conda install sharedarray tensorboard tensorboardx yapf addict einops scipy plyfile termcolor timm -c conda-forge -y +RUN conda install pytorch-cluster pytorch-scatter pytorch-sparse -c pyg -y + +RUN pip install --upgrade pip +RUN pip install torch-geometric +RUN pip install spconv-cu${CUDA_VERSION_NO_DOT} +RUN pip install open3d + +# Build MinkowskiEngine +RUN git clone https://github.com/NVIDIA/MinkowskiEngine.git +WORKDIR /workspace/MinkowskiEngine +RUN TORCH_CUDA_ARCH_LIST="5.2 6.0 6.1 7.0+PTX 8.0" python setup.py install --blas=openblas --force_cuda +WORKDIR /workspace + +# Build pointops +RUN git clone https://github.com/Pointcept/Pointcept.git +RUN TORCH_CUDA_ARCH_LIST="5.2 6.0 6.1 7.0+PTX 8.0" pip install Pointcept/libs/pointops -v + +# Build pointgroup_ops +RUN TORCH_CUDA_ARCH_LIST="5.2 6.0 6.1 7.0+PTX 8.0" pip install Pointcept/libs/pointgroup_ops -v + +# Build swin3d +RUN TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0+PTX 8.0" pip install -U git+https://github.com/microsoft/Swin3D.git -v +EOM + +docker build . -f ./Dockerfile -t $IMG_TAG \ No newline at end of file diff --git a/submodules/PointTransformerV3/Pointcept/scripts/create_tars.sh b/submodules/PointTransformerV3/Pointcept/scripts/create_tars.sh new file mode 100644 index 0000000000000000000000000000000000000000..8bd990b2fc6d3448202a04db63c2adb707c2652b --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/scripts/create_tars.sh @@ -0,0 +1,67 @@ +#!/bin/sh + +# Variables +SOURCE_DIR=$1 +DEST_DIR=$2 +MAX_SIZE=$(awk "BEGIN {printf \"%d\", $3 * 1024 * 1024}") # Convert GB to KB as an integer + +# Get the base name of the source directory to use as TAR_NAME +TAR_NAME=$(basename "$SOURCE_DIR") + +# Create destination directory if it doesn't exist +mkdir -p "$DEST_DIR" + +# Function to create a new tar file +create_tar() { + tar_number=$1 + file_list=$2 + tar_name=$(printf "%s/${TAR_NAME}_%0${width}d.tar.gz" "$DEST_DIR" "$tar_number") + tar -zcvf "$tar_name" -C "$SOURCE_DIR" -T "$file_list" +} + +# Initialize +tar_number=1 +current_size=0 +temp_dir=$(mktemp -d) +file_list="$temp_dir/file_list_$tar_number" +echo Start indexing "file_list_$tar_number" + +cd "$SOURCE_DIR" || exit 1 + +# Iterate over all files in the source directory +find . -type f | while IFS= read -r file; do + file_size=$(du -k "$file" | cut -f1) + + if [ $(( current_size + file_size )) -gt $MAX_SIZE ]; then + tar_number=$((tar_number + 1)) + file_list="$temp_dir/file_list_$tar_number" + echo Start indexing "file_list_$tar_number" + current_size=0 + fi + + echo "$file" >> "$file_list" + current_size=$((current_size + file_size)) +done + +# Determine the width for the tar file numbers +total_files=$(find "$temp_dir" -name 'file_list_*' | wc -l) +width=${#total_files} + +# Set PARALLEL_PROCESSES to the number of file lists if not provided +PARALLEL_PROCESSES=${4:-$total_files} + +# Debug information +echo "Total files: $total_files" +echo "Width: $width" +echo "Parallel processes: $PARALLEL_PROCESSES" + +# Run tar creation in parallel +find "$temp_dir" -name 'file_list_*' | xargs -P "$PARALLEL_PROCESSES" -I {} sh -c ' + file_list={} + tar_number=$(basename "$file_list" | cut -d_ -f3) + tar_name=$(printf "%s/'"$TAR_NAME"'_%0'"$width"'d.tar.gz" "'"$DEST_DIR"'" "$tar_number") + tar -zcvf "$tar_name" -C "'"$SOURCE_DIR"'" -T "$file_list" +' + +# Clean up +rm -rf "$temp_dir" \ No newline at end of file diff --git a/submodules/PointTransformerV3/Pointcept/scripts/test.sh b/submodules/PointTransformerV3/Pointcept/scripts/test.sh new file mode 100644 index 0000000000000000000000000000000000000000..a104f98e67873c7741711b63da6cdbd8c88b73f4 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/scripts/test.sh @@ -0,0 +1,74 @@ +#!/bin/sh + +cd $(dirname $(dirname "$0")) || exit +PYTHON=python + +TEST_CODE=test.py + +DATASET=scannet +CONFIG="None" +EXP_NAME=debug +WEIGHT=model_best +GPU=None + +while getopts "p:d:c:n:w:g:" opt; do + case $opt in + p) + PYTHON=$OPTARG + ;; + d) + DATASET=$OPTARG + ;; + c) + CONFIG=$OPTARG + ;; + n) + EXP_NAME=$OPTARG + ;; + w) + WEIGHT=$OPTARG + ;; + g) + GPU=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" + ;; + esac +done + +if [ "${NUM_GPU}" = 'None' ] +then + NUM_GPU=`$PYTHON -c 'import torch; print(torch.cuda.device_count())'` +fi + +echo "Experiment name: $EXP_NAME" +echo "Python interpreter dir: $PYTHON" +echo "Dataset: $DATASET" +echo "GPU Num: $GPU" + +EXP_DIR=exp/${DATASET}/${EXP_NAME} +MODEL_DIR=${EXP_DIR}/model +CODE_DIR=${EXP_DIR}/code +CONFIG_DIR=${EXP_DIR}/config.py + +if [ "${CONFIG}" = "None" ] +then + CONFIG_DIR=${EXP_DIR}/config.py +else + CONFIG_DIR=configs/${DATASET}/${CONFIG}.py +fi + +echo "Loading config in:" $CONFIG_DIR +#export PYTHONPATH=./$CODE_DIR +export PYTHONPATH=./ +echo "Running code in: $CODE_DIR" + + +echo " =========> RUN TASK <=========" + +#$PYTHON -u "$CODE_DIR"/tools/$TEST_CODE \ +$PYTHON -u tools/$TEST_CODE \ + --config-file "$CONFIG_DIR" \ + --num-gpus "$GPU" \ + --options save_path="$EXP_DIR" weight="${MODEL_DIR}"/"${WEIGHT}".pth diff --git a/submodules/PointTransformerV3/Pointcept/scripts/train.sh b/submodules/PointTransformerV3/Pointcept/scripts/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..2910ba1e92423ce8decf40eeeb4d5115da60b8b9 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/scripts/train.sh @@ -0,0 +1,92 @@ +#!/bin/sh + +cd $(dirname $(dirname "$0")) || exit +ROOT_DIR=$(pwd) +PYTHON=python + +TRAIN_CODE=train.py + +DATASET=scannet +CONFIG="None" +EXP_NAME=debug +WEIGHT="None" +RESUME=false +GPU=None + + +while getopts "p:d:c:n:w:g:r:" opt; do + case $opt in + p) + PYTHON=$OPTARG + ;; + d) + DATASET=$OPTARG + ;; + c) + CONFIG=$OPTARG + ;; + n) + EXP_NAME=$OPTARG + ;; + w) + WEIGHT=$OPTARG + ;; + r) + RESUME=$OPTARG + ;; + g) + GPU=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" + ;; + esac +done + +if [ "${NUM_GPU}" = 'None' ] +then + NUM_GPU=`$PYTHON -c 'import torch; print(torch.cuda.device_count())'` +fi + +echo "Experiment name: $EXP_NAME" +echo "Python interpreter dir: $PYTHON" +echo "Dataset: $DATASET" +echo "Config: $CONFIG" +echo "GPU Num: $GPU" + +EXP_DIR=exp/${DATASET}/${EXP_NAME} +MODEL_DIR=${EXP_DIR}/model +CODE_DIR=${EXP_DIR}/code +CONFIG_DIR=configs/${DATASET}/${CONFIG}.py + + +echo " =========> CREATE EXP DIR <=========" +echo "Experiment dir: $ROOT_DIR/$EXP_DIR" +if ${RESUME} +then + CONFIG_DIR=${EXP_DIR}/config.py + WEIGHT=$MODEL_DIR/model_last.pth +else + mkdir -p "$MODEL_DIR" "$CODE_DIR" + cp -r scripts tools pointcept "$CODE_DIR" +fi + +echo "Loading config in:" $CONFIG_DIR +export PYTHONPATH=./$CODE_DIR +echo "Running code in: $CODE_DIR" + + +echo " =========> RUN TASK <=========" + +if [ "${WEIGHT}" = "None" ] +then + $PYTHON "$CODE_DIR"/tools/$TRAIN_CODE \ + --config-file "$CONFIG_DIR" \ + --num-gpus "$GPU" \ + --options save_path="$EXP_DIR" +else + $PYTHON "$CODE_DIR"/tools/$TRAIN_CODE \ + --config-file "$CONFIG_DIR" \ + --num-gpus "$GPU" \ + --options save_path="$EXP_DIR" resume="$RESUME" weight="$WEIGHT" +fi \ No newline at end of file diff --git a/submodules/PointTransformerV3/Pointcept/tools/create_waymo_semseg_submission.py b/submodules/PointTransformerV3/Pointcept/tools/create_waymo_semseg_submission.py new file mode 100644 index 0000000000000000000000000000000000000000..ded9f68bde40015a1bc7d1b7197ae909ff5831fe --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/tools/create_waymo_semseg_submission.py @@ -0,0 +1,131 @@ +""" +Script for Creating Waymo Semantic Segmentation Submission + +The Waymo dataset toolkit relies on an old version of Tensorflow +which share a conflicting dependency with the Pointcept environment, +therefore we detach the submission generation from the test process +and the script require the following environment: + +```bash +conda create -n waymo python=3.8 -y +conda activate waymo +pip3 install waymo-open-dataset-tf-2-11-0 +``` + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import os +import tqdm +import argparse +import numpy as np +import zlib +import waymo_open_dataset.dataset_pb2 as open_dataset +from waymo_open_dataset.protos import segmentation_metrics_pb2 +from waymo_open_dataset.protos import segmentation_submission_pb2 + + +def compress_array(array: np.ndarray, is_int32: bool = False): + """Compress a numpy array to ZLIP compressed serialized MatrixFloat/Int32. + + Args: + array: A numpy array. + is_int32: If true, use MatrixInt32, otherwise use MatrixFloat. + + Returns: + The compressed bytes. + """ + if is_int32: + m = open_dataset.MatrixInt32() + else: + m = open_dataset.MatrixFloat() + m.shape.dims.extend(list(array.shape)) + m.data.extend(array.reshape([-1]).tolist()) + return zlib.compress(m.SerializeToString()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--record_path", + required=True, + help="Path to the prediction result folder of Waymo dataset", + ) + parser.add_argument( + "--dataset_path", + required=True, + help="Path to the processed Waymo dataset", + ) + parser.add_argument( + "--split", + required=True, + choices=["validation", "testing"], + help="Split of the prediction ([training, validation, testing]).", + ) + args = parser.parse_args() + file_list = [file for file in os.listdir(args.record_path) if file.endswith(".npy")] + submission = segmentation_submission_pb2.SemanticSegmentationSubmission() + frames = segmentation_metrics_pb2.SegmentationFrameList() + bar = tqdm.tqdm(file_list) + for file in bar: + bar.set_postfix(file=file) + context_name, frame_timestamp_micros = file.strip("segment-*_pred.npy").split( + "_with_camera_labels_" + ) + # Load prediction. + # In Pointcept waymo dataset, we minus 1 to label to ignore UNLABELLED class (0 -> -1) + pred = np.load(os.path.join(args.record_path, file)) + 1 + masks = np.load( + os.path.join( + args.dataset_path, + args.split, + f"segment-{context_name}_with_camera_labels", + frame_timestamp_micros, + "mask.npy", + ), + allow_pickle=True, + ) + offset = np.cumsum([mask.sum() for mask in masks.reshape(-1)]) + pred = np.split(pred[: offset[-1]], offset[:-1]) + pred_ri1 = pred[0] + pred_ri2 = pred[5] + mask_ri1 = np.expand_dims(masks[0, 0], -1) + mask_ri2 = np.expand_dims(masks[1, 0], -1) + range_dummy = np.zeros_like(mask_ri1, dtype=np.int32) + range_pred_ri1 = np.zeros_like(mask_ri1, dtype=np.int32) + range_pred_ri1[mask_ri1] = pred_ri1 + range_pred_ri1 = np.concatenate([range_dummy, range_pred_ri1], axis=-1) + range_pred_ri2 = np.zeros_like(mask_ri2, dtype=np.int32) + range_pred_ri2[mask_ri2] = pred_ri2 + range_pred_ri2 = np.concatenate([range_dummy, range_pred_ri2], axis=-1) + + # generate frame submission + segmentation_label = open_dataset.Laser() + segmentation_label.name = open_dataset.LaserName.TOP + segmentation_label.ri_return1.segmentation_label_compressed = compress_array( + range_pred_ri1, is_int32=True + ) + segmentation_label.ri_return2.segmentation_label_compressed = compress_array( + range_pred_ri2, is_int32=True + ) + frame = segmentation_metrics_pb2.SegmentationFrame() + frame.segmentation_labels.append(segmentation_label) + frame.context_name = context_name + frame.frame_timestamp_micros = int(frame_timestamp_micros) + frames.frames.append(frame) + submission.account_name = "***" + submission.unique_method_name = "***" + submission.authors.append("***") + submission.affiliation = "***" + submission.method_link = "***" + submission.sensor_type = ( + segmentation_submission_pb2.SemanticSegmentationSubmission.LIDAR_ALL + ) + submission.number_past_frames_exclude_current = 0 + submission.number_future_frames_exclude_current = 0 + submission.inference_results.CopyFrom(frames) + output_filename = os.path.join(args.record_path, "submission.bin") + f = open(output_filename, "wb") + f.write(submission.SerializeToString()) + f.close() diff --git a/submodules/PointTransformerV3/Pointcept/tools/test.py b/submodules/PointTransformerV3/Pointcept/tools/test.py new file mode 100644 index 0000000000000000000000000000000000000000..c66708d417082451f23cb635bf4dd1c59082f625 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/tools/test.py @@ -0,0 +1,38 @@ +""" +Main Testing Script + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from pointcept.engines.defaults import ( + default_argument_parser, + default_config_parser, + default_setup, +) +from pointcept.engines.test import TESTERS +from pointcept.engines.launch import launch + + +def main_worker(cfg): + cfg = default_setup(cfg) + tester = TESTERS.build(dict(type=cfg.test.type, cfg=cfg)) + tester.test() + + +def main(): + args = default_argument_parser().parse_args() + cfg = default_config_parser(args.config_file, args.options) + + launch( + main_worker, + num_gpus_per_machine=args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + cfg=(cfg,), + ) + + +if __name__ == "__main__": + main() diff --git a/submodules/PointTransformerV3/Pointcept/tools/test_s3dis_6fold.py b/submodules/PointTransformerV3/Pointcept/tools/test_s3dis_6fold.py new file mode 100644 index 0000000000000000000000000000000000000000..711ad42c956412cb9cb68adf596b679e25f48d19 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/tools/test_s3dis_6fold.py @@ -0,0 +1,102 @@ +""" +Test script for S3DIS 6-fold cross validation + +Gathering Area_X.pth from result folder of experiment record of each area as follows: +|- RECORDS_PATH + |- Area_1.pth + |- Area_2.pth + |- Area_3.pth + |- Area_4.pth + |- Area_5.pth + |- Area_6.pth + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import argparse +import os + +import torch +import numpy as np +import glob +from pointcept.utils.logger import get_root_logger + +CLASS_NAMES = [ + "ceiling", + "floor", + "wall", + "beam", + "column", + "window", + "door", + "table", + "chair", + "sofa", + "bookcase", + "board", + "clutter", +] + + +def evaluation(intersection, union, target, logger=None): + iou_class = intersection / (union + 1e-10) + accuracy_class = intersection / (target + 1e-10) + mIoU = np.mean(iou_class) + mAcc = np.mean(accuracy_class) + allAcc = sum(intersection) / (sum(target) + 1e-10) + + if logger is not None: + logger.info( + "Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}".format( + mIoU, mAcc, allAcc + ) + ) + for i in range(len(CLASS_NAMES)): + logger.info( + "Class_{idx} - {name} Result: iou/accuracy {iou:.4f}/{accuracy:.4f}".format( + idx=i, + name=CLASS_NAMES[i], + iou=iou_class[i], + accuracy=accuracy_class[i], + ) + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--record_root", + required=True, + help="Path to the S3DIS record of each split", + ) + config = parser.parse_args() + logger = get_root_logger( + log_file=os.path.join(config.record_root, "6-fold.log"), + file_mode="w", + ) + + records = sorted(glob.glob(os.path.join(config.record_root, "Area_*.pth"))) + assert len(records) == 6 + intersection_ = np.zeros(len(CLASS_NAMES), dtype=int) + union_ = np.zeros(len(CLASS_NAMES), dtype=int) + target_ = np.zeros(len(CLASS_NAMES), dtype=int) + + for record in records: + area = os.path.basename(record).split(".")[0] + info = torch.load(record) + logger.info(f"<<<<<<<<<<<<<<<<< Parsing {area} <<<<<<<<<<<<<<<<<") + intersection = info["intersection"] + union = info["union"] + target = info["target"] + evaluation(intersection, union, target, logger=logger) + intersection_ += intersection + union_ += union + target_ += target + + logger.info(f"<<<<<<<<<<<<<<<<< Parsing 6-fold <<<<<<<<<<<<<<<<<") + evaluation(intersection_, union_, target_, logger=logger) + + +if __name__ == "__main__": + main() diff --git a/submodules/PointTransformerV3/Pointcept/tools/train.py b/submodules/PointTransformerV3/Pointcept/tools/train.py new file mode 100644 index 0000000000000000000000000000000000000000..e3ed749c4d0bae2c3ad26487d92c46c5695341a2 --- /dev/null +++ b/submodules/PointTransformerV3/Pointcept/tools/train.py @@ -0,0 +1,38 @@ +""" +Main Training Script + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +from pointcept.engines.defaults import ( + default_argument_parser, + default_config_parser, + default_setup, +) +from pointcept.engines.train import TRAINERS +from pointcept.engines.launch import launch + + +def main_worker(cfg): + cfg = default_setup(cfg) + trainer = TRAINERS.build(dict(type=cfg.train.type, cfg=cfg)) + trainer.train() + + +def main(): + args = default_argument_parser().parse_args() + cfg = default_config_parser(args.config_file, args.options) + + launch( + main_worker, + num_gpus_per_machine=args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + cfg=(cfg,), + ) + + +if __name__ == "__main__": + main() diff --git a/submodules/PointTransformerV3/README.md b/submodules/PointTransformerV3/README.md new file mode 100644 index 0000000000000000000000000000000000000000..39fdf451720f1d71e403224e2a17979034d4bb6b --- /dev/null +++ b/submodules/PointTransformerV3/README.md @@ -0,0 +1,244 @@ +# Point Transformer V3 +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/point-transformer-v3-simpler-faster-stronger/lidar-semantic-segmentation-on-nuscenes)](https://paperswithcode.com/sota/lidar-semantic-segmentation-on-nuscenes?p=point-transformer-v3-simpler-faster-stronger) +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/point-transformer-v3-simpler-faster-stronger/semantic-segmentation-on-s3dis)](https://paperswithcode.com/sota/semantic-segmentation-on-s3dis?p=point-transformer-v3-simpler-faster-stronger) +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/point-transformer-v3-simpler-faster-stronger/semantic-segmentation-on-scannet)](https://paperswithcode.com/sota/semantic-segmentation-on-scannet?p=point-transformer-v3-simpler-faster-stronger) +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/point-transformer-v3-simpler-faster-stronger/3d-semantic-segmentation-on-scannet200)](https://paperswithcode.com/sota/3d-semantic-segmentation-on-scannet200?p=point-transformer-v3-simpler-faster-stronger) +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/point-transformer-v3-simpler-faster-stronger/3d-semantic-segmentation-on-semantickitti)](https://paperswithcode.com/sota/3d-semantic-segmentation-on-semantickitti?p=point-transformer-v3-simpler-faster-stronger) + +This repo is the official project repository of the paper **_Point Transformer V3: Simpler, Faster, Stronger_** and is mainly used for releasing schedules, updating instructions, sharing experiment records (containing model weight), and handling issues. The code will be updated in _[Pointcept](https://github.com/Pointcept/Pointcept) v1.5_. +[ Backbone ] [PTv3] - [ [arXiv](https://arxiv.org/abs/2312.10035) ] [ [Bib](https://xywu.me/research/ptv3/bib.txt) ] [ [Code](https://github.com/Pointcept/Pointcept) ] + +
+teaser +
+ +## Highlights +- *Apr 05, 2024*: PTv3 is selected as one of the 90 **Oral** presentations (3.3% accepted paper, 0.78% submissions) by CVPR'24! +- *Feb 28, 2024*: PTv3 is accepted by CVPR'24 🎉🎉🎉. +- *Dec 31, 2023*: We released the model code of PTv3, experiment records for scratched ScanNet and ScanNet200 are now available. More will be available soon. +- *Dec 19, 2023*: We released our project repo for PTv3, if you have any questions related to our work, please feel free to open an issue. Subscribe to our updates by filling out the [form](https://forms.gle/jHoBNqfhqK94WG678) and the subscription can be canceled by editing the form. + + +## Overview +- [Schedule](#schedule) +- [Citation](#citation) +- [Installation](#installation) +- [Data Preparation](#data-preparation) +- [Quick Start](#quick-start) +- [Model Zoo](#model-zoo) + + +## Schedule +To make our polished code and reproduced experiments available as soon as possible, this time we will release what we already finished immediately after a validation instead of releasing them together after all work is done. We list a task list as follows: + +- [x] Release model code of PTv3; +- [x] Release scratched config and record of indoor semantic segmentation; + - [x] ScanNet + - [x] ScanNet200 + - [x] S3DIS + - [x] S3DIS 6-Fold (with cross-validation script) +- [ ] Release pre-trained config and record of indoor semantic segmentation; + - [x] ScanNet (ScanNet + S3DIS + Structured3D) + - [ ] ScanNet200 (Fine-tuned from above) + - [x] S3DIS (ScanNet + S3DIS + Structured3D) + - [x] S3DIS 6-Fold (Fine-tuned from ScanNet + Structured3D) +- [ ] Release scratched config and record of outdoor semantic segmentation; + - [x] NuScenes + - [ ] SemanticKITTI + - [x] Waymo +- [ ] Release pre-trained config and record of outdoor semantic segmentation; + - [ ] NuScenes (NuScenes + SemanticKITTI + Waymo) + - [ ] SemanticKITTI (NuScenes + SemanticKITTI + Waymo) + - [ ] Waymo (NuScenes + SemanticKITTI + Waymo) +- [ ] Release config and record of indoor instance segmentation; + - [ ] ScanNet (Scratch and Fine-tuned from PPT pre-trained PTv3) + - [ ] ScanNet200 (Scratch and Fine-tuned from PPT pre-trained PTv3) +- [ ] Release config and record of ScanNet data efficient benchmark; +- [ ] Release config and record of Waymo Object Detection benchmark; +- [ ] Release config and record of ImageNet classification; + - [ ] ImageClassifier (making all 3D backbones in Pointcept support image classification) + - [ ] Config and Record (PTv3 + SparseUNet) + +## Citation +If you find _PTv3_ useful to your research, please cite our work as an acknowledgment. (੭ˊ꒳​ˋ)੭✧ +```bib +@inproceedings{wu2024ptv3, + title={Point Transformer V3: Simpler, Faster, Stronger}, + author={Wu, Xiaoyang and Jiang, Li and Wang, Peng-Shuai and Liu, Zhijian and Liu, Xihui and Qiao, Yu and Ouyang, Wanli and He, Tong and Zhao, Hengshuang}, + booktitle={CVPR}, + year={2024} +} + +@inproceedings{wu2024ppt, + title={Towards Large-scale 3D Representation Learning with Multi-dataset Point Prompt Training}, + author={Wu, Xiaoyang and Tian, Zhuotao and Wen, Xin and Peng, Bohao and Liu, Xihui and Yu, Kaicheng and Zhao, Hengshuang}, + booktitle={CVPR}, + year={2024} +} + +@inproceedings{wu2022ptv2, + title={Point transformer V2: Grouped Vector Attention and Partition-based Pooling}, + author={Wu, Xiaoyang and Lao, Yixing and Jiang, Li and Liu, Xihui and Zhao, Hengshuang}, + booktitle={NeurIPS}, + year={2022} +} + +@misc{pointcept2023, + title={Pointcept: A Codebase for Point Cloud Perception Research}, + author={Pointcept Contributors}, + howpublished={\url{https://github.com/Pointcept/Pointcept}}, + year={2023} +} +``` + +## Installation + +### Requirements + PTv3 relies on FlashAttention, while FlashAttention relies on the following requirement, make sure your local Pointcept environment satisfies the requirements: + +(Recommendation) +- Ubuntu: 20.04 and above +- CUDA: 11.6 and above +- PyTorch: 1.12.0 and above + +If you can not upgrade your local environment to satisfy the above-recommended requirements, the following requirement is the minimum to run PTv3 with Pointcept, and you need to disable Flash Attention to enable PTv3: + +(Minimum) +- Ubuntu: 18.04 and above +- CUDA: 11.3 and above +- PyTorch: 1.10.0 and above + +### Environment + +- Base environment +```bash +conda create -n pointcept python=3.8 -y +conda activate pointcept +conda install ninja -y +# Choose version you want here: https://pytorch.org/get-started/previous-versions/ +# We use CUDA 11.8 and PyTorch 2.1.0 for our development of PTv3 +conda install pytorch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0 pytorch-cuda=11.8 -c pytorch -c nvidia +conda install h5py pyyaml -c anaconda -y +conda install sharedarray tensorboard tensorboardx yapf addict einops scipy plyfile termcolor timm -c conda-forge -y +conda install pytorch-cluster pytorch-scatter pytorch-sparse -c pyg -y +pip install torch-geometric + +cd libs/pointops +python setup.py install +cd ../.. + +# spconv (SparseUNet) +# refer https://github.com/traveller59/spconv +pip install spconv-cu118 # choose version match your local cuda version + +# Open3D (visualization, optional) +pip install open3d +``` + +- Flash Attention + +Following [README](https://github.com/Dao-AILab/flash-attention?tab=readme-ov-file#installation-and-features) in Flash Attention repo and install Flash Attention for PTv3. This installation is optional, but we recommend enabling Flash Attention for PTv3. + + +## Data Preparation +Please further refer Pointcept readme [Data Preparation](https://github.com/Pointcept/Pointcept#data-preparation) section. + +## Quick Start +### Two running scenarios +We provide two running scenarios for PTv3, Pointcept-driven and custom-framework-driven. For the former one, you only need to clone the code of Pointcept to your local and follow the [Quick Start](https://github.com/Pointcept/Pointcept#quick-start) in Pointcept to run PTv3: +```bash +git clone https://github.com/Pointcept/Pointcept.git +sh scripts/train.sh -p ${INTERPRETER_PATH} -g ${NUM_GPU} -d ${DATASET_NAME} -c ${CONFIG_NAME} -n ${EXP_NAME} +``` +For the latter scenario, we offer a distinct instance of PTv3, disassociated from our Pointcept framework. To incorporate this code into your project, clone the project repo and copy the following file/folder to your project: +```bash +git clone https://github.com/Pointcept/PointTransformerV3.git +cp model.py ${PATH_TO_YOUR_PROJECT} +cp -r serialization ${PATH_TO_YOUR_PROJECT} +``` +Align the input dictionary defined in our [model](https://github.com/Pointcept/PointTransformerV3/blob/dev/model.py#L968) file and the model will return the encoded feature of the given batch point cloud. + +### Flash Attention +The full PTv3 relies on Flash Attention, while Flash Attention relies on CUDA 11.6 and above, make sure your local Pointcept environment satisfies the requirements. + +If you can not upgrade your local environment to satisfy the requirements (CUDA >= 11.6), then you can disable FlashAttention by setting the model parameter `enable_flash` to `false` and reducing the `enc_patch_size` and `dec_patch_size` to a level (e.g. 128). + +FlashAttention force disables RPE and forces the accuracy reduced to fp16. If you require these features, please disable `enable_flash` and adjust `enable_rpe`, `upcast_attention` and`upcast_softmax`. + + +## Model Zoo +### 1. Indoor semantic segmentation +| Model | Benchmark | Additional Data | Num GPUs | Val mIoU | Config | Tensorboard | Exp Record | +| :---: | :---: |:---------------:| :---: | :---: | :---: | :---: | :---: | +| PTv3 | ScanNet | ✗ | 4 | 77.6% | [link](https://github.com/Pointcept/Pointcept/blob/main/configs/scannet/semseg-pt-v3m1-0-base.py) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tensorboard) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tree/main/scannet-semseg-pt-v3m1-0-base) | +| PTv3 + PPT | ScanNet | ✓ | 8 | 78.5% | [link](https://github.com/Pointcept/Pointcept/blob/main/configs/scannet/semseg-pt-v3m1-1-ppt-extreme.py) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tensorboard) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tree/main/scannet-semseg-pt-v3m1-1-ppt-extreme) | +| PTv3 | ScanNet200 | ✗ | 4 | 35.3% | [link](https://github.com/Pointcept/Pointcept/blob/main/configs/scannet200/semseg-pt-v3m1-0-base.py) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tensorboard) |[link](https://huggingface.co/Pointcept/PointTransformerV3/tree/main/scannet200-semseg-pt-v3m1-0-base)| +| PTv3 + PPT | ScanNet200 | ✓ (f.t.) | 4 | | | | | +| PTv3 | S3DIS (Area5) | ✗ | 4 | 73.6% | [link](https://github.com/Pointcept/Pointcept/blob/main/configs/s3dis/semseg-pt-v3m1-0-rpe.py) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tensorboard) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tree/main/s3dis-semseg-pt-v3m1-0-rpe) | +| PTv3 + PPT | S3DIS (Area5) | ✓ | 8 | 75.4% | [link](https://github.com/Pointcept/Pointcept/blob/main/configs/s3dis/semseg-pt-v3m1-1-ppt-extreme.py) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tensorboard) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tree/main/s3dis-semseg-pt-v3m1-1-ppt-extreme) | + +_**\*Released model weights are temporarily invalid as the model structure of PTv3 is adjusted.**_ + +Example running scripts are as follows: +```bash +# Scratched ScanNet +sh scripts/train.sh -g 4 -d scannet -c semseg-pt-v3m1-0-base -n semseg-pt-v3m1-0-base +# PPT joint training (ScanNet + Structured3D) and evaluate in ScanNet +sh scripts/train.sh -g 8 -d scannet -c semseg-pt-v3m1-1-ppt-extreme -n semseg-pt-v3m1-1-ppt-extreme + +# Scratched ScanNet200 +sh scripts/train.sh -g 4 -d scannet200 -c semseg-pt-v3m1-0-base -n semseg-pt-v3m1-0-base +# Fine-tuning from PPT joint training (ScanNet + Structured3D) with ScanNet200 +# TODO + +# Scratched S3DIS, S3DIS rely on RPE, also an example for disable flash attention +sh scripts/train.sh -g 4 -d s3dis -c semseg-pt-v3m1-0-rpe -n semseg-pt-v3m1-0-rpe +# PPT joint training (ScanNet + S3DIS + Structured3D) and evaluate in ScanNet +sh scripts/train.sh -g 8 -d s3dis -c semseg-pt-v3m1-1-ppt-extreme -n semseg-pt-v3m1-1-ppt-extreme + +# More configs and exp records for PTv3 will be available soon. +``` + +### 2.Outdoor semantic segmentation +| Model | Benchmark | Additional Data | Num GPUs | Val mIoU | Config | Tensorboard | Exp Record | +| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| PTv3 | nuScenes | ✗ | 4 | 80.3 | [link](https://github.com/Pointcept/Pointcept/blob/main/configs/nuscenes/semseg-pt-v3m1-0-base.py) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tensorboard)|[link](https://huggingface.co/Pointcept/PointTransformerV3/tree/main/nuscenes-semseg-pt-v3m1-0-base) | +| PTv3 + PPT | nuScenes | ✓ | 8 | | | | | +| PTv3 | SemanticKITTI | ✗ | 4 | | | | | +| PTv3 + PPT | SemanticKITTI | ✓ | 8 | | | | | +| PTv3 | Waymo | ✗ | 4 | 71.2 | [link](https://github.com/Pointcept/Pointcept/blob/main/configs/waymo/semseg-pt-v3m1-0-base.py) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tensorboard) | [link](https://huggingface.co/Pointcept/PointTransformerV3/tree/main/waymo-semseg-pt-v3m1-0-base) (log only) | +| PTv3 + PPT | Waymo | ✓ | 8 | | | | | + +_**\*Released model weights are temporarily invalid as the model structure of PTv3 is adjusted.**_ +_**\*Model weights trained with Waymo Open Dataset cannot be released due to the regulations.**_ + +Example running scripts are as follows: +```bash +# Scratched ScanNet +sh scripts/train.sh -g 4 -d scannet -c semseg-pt-v3m1-0-base -n semseg-pt-v3m1-0-base +# PPT joint training (ScanNet + Structured3D) and evaluate in ScanNet +sh scripts/train.sh -g 8 -d scannet -c semseg-pt-v3m1-1-ppt-extreme -n semseg-pt-v3m1-1-ppt-extreme + +# Scratched ScanNet200 +sh scripts/train.sh -g 4 -d scannet200 -c semseg-pt-v3m1-0-base -n semseg-pt-v3m1-0-base +# Fine-tuning from PPT joint training (ScanNet + Structured3D) with ScanNet200 +# TODO + +# Scratched S3DIS, S3DIS rely on RPE, also an example for disable flash attention +sh scripts/train.sh -g 4 -d s3dis -c semseg-pt-v3m1-0-rpe -n semseg-pt-v3m1-0-rpe +# PPT joint training (ScanNet + S3DIS + Structured3D) and evaluate in ScanNet +sh scripts/train.sh -g 8 -d s3dis -c semseg-pt-v3m1-1-ppt-extreme -n semseg-pt-v3m1-1-ppt-extreme +# S3DIS 6-fold cross validation +# 1. The default configs are evaluated on Area_5, modify the "data.train.split", "data.val.split", and "data.test.split" to make the config evaluated on Area_1 ~ Area_6 respectively. +# 2. Train and evaluate the model on each split of areas and gather result files located in "exp/s3dis/EXP_NAME/result/Area_x.pth" in one single folder, noted as RECORD_FOLDER. +# 3. Run the following script to get S3DIS 6-fold cross validation performance: +export PYTHONPATH=./ +python tools/test_s3dis_6fold.py --record_root ${RECORD_FOLDER} + +# Scratched nuScenes +sh scripts/train.sh -g 4 -d nuscenes -c semseg-pt-v3m1-0-base -n semseg-pt-v3m1-0-base +# Scratched Waymo +sh scripts/train.sh -g 4 -d waymo -c semseg-pt-v3m1-0-base -n semseg-pt-v3m1-0-base + +# More configs and exp records for PTv3 will be available soon. +``` diff --git a/submodules/PointTransformerV3/assets/teaser.png b/submodules/PointTransformerV3/assets/teaser.png new file mode 100644 index 0000000000000000000000000000000000000000..10e6633ae876ee0798b7865ad1b635241119e5b2 Binary files /dev/null and b/submodules/PointTransformerV3/assets/teaser.png differ diff --git a/submodules/PointTransformerV3/model.py b/submodules/PointTransformerV3/model.py new file mode 100644 index 0000000000000000000000000000000000000000..4d07738fd8dfdbee5d9d22629c1f25f8c388d6d4 --- /dev/null +++ b/submodules/PointTransformerV3/model.py @@ -0,0 +1,984 @@ +""" +Point Transformer - V3 Mode1 +Pointcept detached version + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com) +Please cite our work if the code is helpful to you. +""" + +import sys +from functools import partial +from addict import Dict +import math +import torch +import torch.nn as nn +import spconv.pytorch as spconv +import torch_scatter +from timm.models.layers import DropPath +from collections import OrderedDict + +try: + import flash_attn +except ImportError: + flash_attn = None + +from .serialization import encode + + +@torch.inference_mode() +def offset2bincount(offset): + return torch.diff( + offset, prepend=torch.tensor([0], device=offset.device, dtype=torch.long) + ) + + +@torch.inference_mode() +def offset2batch(offset): + bincount = offset2bincount(offset) + return torch.arange( + len(bincount), device=offset.device, dtype=torch.long + ).repeat_interleave(bincount) + + +@torch.inference_mode() +def batch2offset(batch): + return torch.cumsum(batch.bincount(), dim=0).long() + + +class Point(Dict): + """ + Point Structure of Pointcept + + A Point (point cloud) in Pointcept is a dictionary that contains various properties of + a batched point cloud. The property with the following names have a specific definition + as follows: + + - "coord": original coordinate of point cloud; + - "grid_coord": grid coordinate for specific grid size (related to GridSampling); + Point also support the following optional attributes: + - "offset": if not exist, initialized as batch size is 1; + - "batch": if not exist, initialized as batch size is 1; + - "feat": feature of point cloud, default input of model; + - "grid_size": Grid size of point cloud (related to GridSampling); + (related to Serialization) + - "serialized_depth": depth of serialization, 2 ** depth * grid_size describe the maximum of point cloud range; + - "serialized_code": a list of serialization codes; + - "serialized_order": a list of serialization order determined by code; + - "serialized_inverse": a list of inverse mapping determined by code; + (related to Sparsify: SpConv) + - "sparse_shape": Sparse shape for Sparse Conv Tensor; + - "sparse_conv_feat": SparseConvTensor init with information provide by Point; + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # If one of "offset" or "batch" do not exist, generate by the existing one + if "batch" not in self.keys() and "offset" in self.keys(): + self["batch"] = offset2batch(self.offset) + elif "offset" not in self.keys() and "batch" in self.keys(): + self["offset"] = batch2offset(self.batch) + + def serialization(self, order="z", depth=None, shuffle_orders=False): + """ + Point Cloud Serialization + + relay on ["grid_coord" or "coord" + "grid_size", "batch", "feat"] + """ + assert "batch" in self.keys() + if "grid_coord" not in self.keys(): + # if you don't want to operate GridSampling in data augmentation, + # please add the following augmentation into your pipline: + # dict(type="Copy", keys_dict={"grid_size": 0.01}), + # (adjust `grid_size` to what your want) + assert {"grid_size", "coord"}.issubset(self.keys()) + self["grid_coord"] = torch.div( + self.coord - self.coord.min(0)[0], self.grid_size, rounding_mode="trunc" + ).int() + + if depth is None: + # Adaptive measure the depth of serialization cube (length = 2 ^ depth) + depth = int(self.grid_coord.max()).bit_length() + self["serialized_depth"] = depth + # Maximum bit length for serialization code is 63 (int64) + assert depth * 3 + len(self.offset).bit_length() <= 63 + # Here we follow OCNN and set the depth limitation to 16 (48bit) for the point position. + # Although depth is limited to less than 16, we can encode a 655.36^3 (2^16 * 0.01) meter^3 + # cube with a grid size of 0.01 meter. We consider it is enough for the current stage. + # We can unlock the limitation by optimizing the z-order encoding function if necessary. + assert depth <= 16 + + # The serialization codes are arranged as following structures: + # [Order1 ([n]), + # Order2 ([n]), + # ... + # OrderN ([n])] (k, n) + code = [ + encode(self.grid_coord, self.batch, depth, order=order_) for order_ in order + ] + code = torch.stack(code) + order = torch.argsort(code) + inverse = torch.zeros_like(order).scatter_( + dim=1, + index=order, + src=torch.arange(0, code.shape[1], device=order.device).repeat( + code.shape[0], 1 + ), + ) + + if shuffle_orders: + perm = torch.randperm(code.shape[0]) + code = code[perm] + order = order[perm] + inverse = inverse[perm] + + self["serialized_code"] = code + self["serialized_order"] = order + self["serialized_inverse"] = inverse + + def sparsify(self, pad=96): + """ + Point Cloud Serialization + + Point cloud is sparse, here we use "sparsify" to specifically refer to + preparing "spconv.SparseConvTensor" for SpConv. + + relay on ["grid_coord" or "coord" + "grid_size", "batch", "feat"] + + pad: padding sparse for sparse shape. + """ + assert {"feat", "batch"}.issubset(self.keys()) + if "grid_coord" not in self.keys(): + # if you don't want to operate GridSampling in data augmentation, + # please add the following augmentation into your pipline: + # dict(type="Copy", keys_dict={"grid_size": 0.01}), + # (adjust `grid_size` to what your want) + assert {"grid_size", "coord"}.issubset(self.keys()) + self["grid_coord"] = torch.div( + self.coord - self.coord.min(0)[0], self.grid_size, rounding_mode="trunc" + ).int() + if "sparse_shape" in self.keys(): + sparse_shape = self.sparse_shape + else: + sparse_shape = torch.add( + torch.max(self.grid_coord, dim=0).values, pad + ).tolist() + sparse_conv_feat = spconv.SparseConvTensor( + features=self.feat, + indices=torch.cat( + [self.batch.unsqueeze(-1).int(), self.grid_coord.int()], dim=1 + ).contiguous(), + spatial_shape=sparse_shape, + batch_size=self.batch[-1].tolist() + 1, + ) + self["sparse_shape"] = sparse_shape + self["sparse_conv_feat"] = sparse_conv_feat + + +class PointModule(nn.Module): + r"""PointModule + placeholder, all module subclass from this will take Point in PointSequential. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class PointSequential(PointModule): + r"""A sequential container. + Modules will be added to it in the order they are passed in the constructor. + Alternatively, an ordered dict of modules can also be passed in. + """ + + def __init__(self, *args, **kwargs): + super().__init__() + if len(args) == 1 and isinstance(args[0], OrderedDict): + for key, module in args[0].items(): + self.add_module(key, module) + else: + for idx, module in enumerate(args): + self.add_module(str(idx), module) + for name, module in kwargs.items(): + if sys.version_info < (3, 6): + raise ValueError("kwargs only supported in py36+") + if name in self._modules: + raise ValueError("name exists.") + self.add_module(name, module) + + def __getitem__(self, idx): + if not (-len(self) <= idx < len(self)): + raise IndexError("index {} is out of range".format(idx)) + if idx < 0: + idx += len(self) + it = iter(self._modules.values()) + for i in range(idx): + next(it) + return next(it) + + def __len__(self): + return len(self._modules) + + def add(self, module, name=None): + if name is None: + name = str(len(self._modules)) + if name in self._modules: + raise KeyError("name exists") + self.add_module(name, module) + + def forward(self, input): + for k, module in self._modules.items(): + # Point module + if isinstance(module, PointModule): + input = module(input) + # Spconv module + elif spconv.modules.is_spconv_module(module): + if isinstance(input, Point): + input.sparse_conv_feat = module(input.sparse_conv_feat) + input.feat = input.sparse_conv_feat.features + else: + input = module(input) + # PyTorch module + else: + if isinstance(input, Point): + input.feat = module(input.feat) + if "sparse_conv_feat" in input.keys(): + input.sparse_conv_feat = input.sparse_conv_feat.replace_feature( + input.feat + ) + elif isinstance(input, spconv.SparseConvTensor): + if input.indices.shape[0] != 0: + input = input.replace_feature(module(input.features)) + else: + input = module(input) + return input + + +class PDNorm(PointModule): + def __init__( + self, + num_features, + norm_layer, + context_channels=256, + conditions=("ScanNet", "S3DIS", "Structured3D"), + decouple=True, + adaptive=False, + ): + super().__init__() + self.conditions = conditions + self.decouple = decouple + self.adaptive = adaptive + if self.decouple: + self.norm = nn.ModuleList([norm_layer(num_features) for _ in conditions]) + else: + self.norm = norm_layer + if self.adaptive: + self.modulation = nn.Sequential( + nn.SiLU(), nn.Linear(context_channels, 2 * num_features, bias=True) + ) + + def forward(self, point): + assert {"feat", "condition"}.issubset(point.keys()) + if isinstance(point.condition, str): + condition = point.condition + else: + condition = point.condition[0] + if self.decouple: + assert condition in self.conditions + norm = self.norm[self.conditions.index(condition)] + else: + norm = self.norm + point.feat = norm(point.feat) + if self.adaptive: + assert "context" in point.keys() + shift, scale = self.modulation(point.context).chunk(2, dim=1) + point.feat = point.feat * (1.0 + scale) + shift + return point + + +class RPE(torch.nn.Module): + def __init__(self, patch_size, num_heads): + super().__init__() + self.patch_size = patch_size + self.num_heads = num_heads + self.pos_bnd = int((4 * patch_size) ** (1 / 3) * 2) + self.rpe_num = 2 * self.pos_bnd + 1 + self.rpe_table = torch.nn.Parameter(torch.zeros(3 * self.rpe_num, num_heads)) + torch.nn.init.trunc_normal_(self.rpe_table, std=0.02) + + def forward(self, coord): + idx = ( + coord.clamp(-self.pos_bnd, self.pos_bnd) # clamp into bnd + + self.pos_bnd # relative position to positive index + + torch.arange(3, device=coord.device) * self.rpe_num # x, y, z stride + ) + out = self.rpe_table.index_select(0, idx.reshape(-1)) + out = out.view(idx.shape + (-1,)).sum(3) + out = out.permute(0, 3, 1, 2) # (N, K, K, H) -> (N, H, K, K) + return out + + +class SerializedAttention(PointModule): + def __init__( + self, + channels, + num_heads, + patch_size, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + order_index=0, + enable_rpe=False, + enable_flash=True, + upcast_attention=True, + upcast_softmax=True, + ): + super().__init__() + assert channels % num_heads == 0 + self.channels = channels + self.num_heads = num_heads + self.scale = qk_scale or (channels // num_heads) ** -0.5 + self.order_index = order_index + self.upcast_attention = upcast_attention + self.upcast_softmax = upcast_softmax + self.enable_rpe = enable_rpe + self.enable_flash = enable_flash + if enable_flash: + assert ( + enable_rpe is False + ), "Set enable_rpe to False when enable Flash Attention" + assert ( + upcast_attention is False + ), "Set upcast_attention to False when enable Flash Attention" + assert ( + upcast_softmax is False + ), "Set upcast_softmax to False when enable Flash Attention" + assert flash_attn is not None, "Make sure flash_attn is installed." + self.patch_size = patch_size + self.attn_drop = attn_drop + else: + # when disable flash attention, we still don't want to use mask + # consequently, patch size will auto set to the + # min number of patch_size_max and number of points + self.patch_size_max = patch_size + self.patch_size = 0 + self.attn_drop = torch.nn.Dropout(attn_drop) + + self.qkv = torch.nn.Linear(channels, channels * 3, bias=qkv_bias) + self.proj = torch.nn.Linear(channels, channels) + self.proj_drop = torch.nn.Dropout(proj_drop) + self.softmax = torch.nn.Softmax(dim=-1) + self.rpe = RPE(patch_size, num_heads) if self.enable_rpe else None + + @torch.no_grad() + def get_rel_pos(self, point, order): + K = self.patch_size + rel_pos_key = f"rel_pos_{self.order_index}" + if rel_pos_key not in point.keys(): + grid_coord = point.grid_coord[order] + grid_coord = grid_coord.reshape(-1, K, 3) + point[rel_pos_key] = grid_coord.unsqueeze(2) - grid_coord.unsqueeze(1) + return point[rel_pos_key] + + @torch.no_grad() + def get_padding_and_inverse(self, point): + pad_key = "pad" + unpad_key = "unpad" + cu_seqlens_key = "cu_seqlens_key" + if ( + pad_key not in point.keys() + or unpad_key not in point.keys() + or cu_seqlens_key not in point.keys() + ): + offset = point.offset + bincount = offset2bincount(offset) + bincount_pad = ( + torch.div( + bincount + self.patch_size - 1, + self.patch_size, + rounding_mode="trunc", + ) + * self.patch_size + ) + # only pad point when num of points larger than patch_size + mask_pad = bincount > self.patch_size + bincount_pad = ~mask_pad * bincount + mask_pad * bincount_pad + _offset = nn.functional.pad(offset, (1, 0)) + _offset_pad = nn.functional.pad(torch.cumsum(bincount_pad, dim=0), (1, 0)) + pad = torch.arange(_offset_pad[-1], device=offset.device) + unpad = torch.arange(_offset[-1], device=offset.device) + cu_seqlens = [] + for i in range(len(offset)): + unpad[_offset[i] : _offset[i + 1]] += _offset_pad[i] - _offset[i] + if bincount[i] != bincount_pad[i]: + pad[ + _offset_pad[i + 1] + - self.patch_size + + (bincount[i] % self.patch_size) : _offset_pad[i + 1] + ] = pad[ + _offset_pad[i + 1] + - 2 * self.patch_size + + (bincount[i] % self.patch_size) : _offset_pad[i + 1] + - self.patch_size + ] + pad[_offset_pad[i] : _offset_pad[i + 1]] -= _offset_pad[i] - _offset[i] + cu_seqlens.append( + torch.arange( + _offset_pad[i], + _offset_pad[i + 1], + step=self.patch_size, + dtype=torch.int32, + device=offset.device, + ) + ) + point[pad_key] = pad + point[unpad_key] = unpad + point[cu_seqlens_key] = nn.functional.pad( + torch.concat(cu_seqlens), (0, 1), value=_offset_pad[-1] + ) + return point[pad_key], point[unpad_key], point[cu_seqlens_key] + + def forward(self, point): + if not self.enable_flash: + self.patch_size = min( + offset2bincount(point.offset).min().tolist(), self.patch_size_max + ) + + H = self.num_heads + K = self.patch_size + C = self.channels + + pad, unpad, cu_seqlens = self.get_padding_and_inverse(point) + + order = point.serialized_order[self.order_index][pad] + inverse = unpad[point.serialized_inverse[self.order_index]] + + # padding and reshape feat and batch for serialized point patch + qkv = self.qkv(point.feat)[order] + + if not self.enable_flash: + # encode and reshape qkv: (N', K, 3, H, C') => (3, N', H, K, C') + q, k, v = ( + qkv.reshape(-1, K, 3, H, C // H).permute(2, 0, 3, 1, 4).unbind(dim=0) + ) + # attn + if self.upcast_attention: + q = q.float() + k = k.float() + attn = (q * self.scale) @ k.transpose(-2, -1) # (N', H, K, K) + if self.enable_rpe: + attn = attn + self.rpe(self.get_rel_pos(point, order)) + if self.upcast_softmax: + attn = attn.float() + attn = self.softmax(attn) + attn = self.attn_drop(attn).to(qkv.dtype) + feat = (attn @ v).transpose(1, 2).reshape(-1, C) + else: + feat = flash_attn.flash_attn_varlen_qkvpacked_func( + qkv.half().reshape(-1, 3, H, C // H), + cu_seqlens, + max_seqlen=self.patch_size, + dropout_p=self.attn_drop if self.training else 0, + softmax_scale=self.scale, + ).reshape(-1, C) + feat = feat.to(qkv.dtype) + feat = feat[inverse] + + # ffn + feat = self.proj(feat) + feat = self.proj_drop(feat) + point.feat = feat + return point + + +class MLP(nn.Module): + def __init__( + self, + in_channels, + hidden_channels=None, + out_channels=None, + act_layer=nn.GELU, + drop=0.0, + ): + super().__init__() + out_channels = out_channels or in_channels + hidden_channels = hidden_channels or in_channels + self.fc1 = nn.Linear(in_channels, hidden_channels) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_channels, out_channels) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class Block(PointModule): + def __init__( + self, + channels, + num_heads, + patch_size=48, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.0, + norm_layer=nn.LayerNorm, + act_layer=nn.GELU, + pre_norm=True, + order_index=0, + cpe_indice_key=None, + enable_rpe=False, + enable_flash=True, + upcast_attention=True, + upcast_softmax=True, + ): + super().__init__() + self.channels = channels + self.pre_norm = pre_norm + + self.cpe = PointSequential( + spconv.SubMConv3d( + channels, + channels, + kernel_size=3, + bias=True, + indice_key=cpe_indice_key, + ), + nn.Linear(channels, channels), + norm_layer(channels), + ) + + self.norm1 = PointSequential(norm_layer(channels)) + self.attn = SerializedAttention( + channels=channels, + patch_size=patch_size, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=proj_drop, + order_index=order_index, + enable_rpe=enable_rpe, + enable_flash=enable_flash, + upcast_attention=upcast_attention, + upcast_softmax=upcast_softmax, + ) + self.norm2 = PointSequential(norm_layer(channels)) + self.mlp = PointSequential( + MLP( + in_channels=channels, + hidden_channels=int(channels * mlp_ratio), + out_channels=channels, + act_layer=act_layer, + drop=proj_drop, + ) + ) + self.drop_path = PointSequential( + DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + ) + + def forward(self, point: Point): + shortcut = point.feat + point = self.cpe(point) + point.feat = shortcut + point.feat + shortcut = point.feat + if self.pre_norm: + point = self.norm1(point) + point = self.drop_path(self.attn(point)) + point.feat = shortcut + point.feat + if not self.pre_norm: + point = self.norm1(point) + + shortcut = point.feat + if self.pre_norm: + point = self.norm2(point) + point = self.drop_path(self.mlp(point)) + point.feat = shortcut + point.feat + if not self.pre_norm: + point = self.norm2(point) + point.sparse_conv_feat = point.sparse_conv_feat.replace_feature(point.feat) + return point + + +class SerializedPooling(PointModule): + def __init__( + self, + in_channels, + out_channels, + stride=2, + norm_layer=None, + act_layer=None, + reduce="max", + shuffle_orders=True, + traceable=True, # record parent and cluster + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + + assert stride == 2 ** (math.ceil(stride) - 1).bit_length() # 2, 4, 8 + # TODO: add support to grid pool (any stride) + self.stride = stride + assert reduce in ["sum", "mean", "min", "max"] + self.reduce = reduce + self.shuffle_orders = shuffle_orders + self.traceable = traceable + + self.proj = nn.Linear(in_channels, out_channels) + if norm_layer is not None: + self.norm = PointSequential(norm_layer(out_channels)) + if act_layer is not None: + self.act = PointSequential(act_layer()) + + def forward(self, point: Point): + pooling_depth = (math.ceil(self.stride) - 1).bit_length() + if pooling_depth > point.serialized_depth: + pooling_depth = 0 + assert { + "serialized_code", + "serialized_order", + "serialized_inverse", + "serialized_depth", + }.issubset( + point.keys() + ), "Run point.serialization() point cloud before SerializedPooling" + + code = point.serialized_code >> pooling_depth * 3 + code_, cluster, counts = torch.unique( + code[0], + sorted=True, + return_inverse=True, + return_counts=True, + ) + # indices of point sorted by cluster, for torch_scatter.segment_csr + _, indices = torch.sort(cluster) + # index pointer for sorted point, for torch_scatter.segment_csr + idx_ptr = torch.cat([counts.new_zeros(1), torch.cumsum(counts, dim=0)]) + # head_indices of each cluster, for reduce attr e.g. code, batch + head_indices = indices[idx_ptr[:-1]] + # generate down code, order, inverse + code = code[:, head_indices] + order = torch.argsort(code) + inverse = torch.zeros_like(order).scatter_( + dim=1, + index=order, + src=torch.arange(0, code.shape[1], device=order.device).repeat( + code.shape[0], 1 + ), + ) + + if self.shuffle_orders: + perm = torch.randperm(code.shape[0]) + code = code[perm] + order = order[perm] + inverse = inverse[perm] + + # collect information + point_dict = Dict( + feat=torch_scatter.segment_csr( + self.proj(point.feat)[indices], idx_ptr, reduce=self.reduce + ), + coord=torch_scatter.segment_csr( + point.coord[indices], idx_ptr, reduce="mean" + ), + grid_coord=point.grid_coord[head_indices] >> pooling_depth, + serialized_code=code, + serialized_order=order, + serialized_inverse=inverse, + serialized_depth=point.serialized_depth - pooling_depth, + batch=point.batch[head_indices], + ) + + if "condition" in point.keys(): + point_dict["condition"] = point.condition + if "context" in point.keys(): + point_dict["context"] = point.context + + if self.traceable: + point_dict["pooling_inverse"] = cluster + point_dict["pooling_parent"] = point + point = Point(point_dict) + if self.norm is not None: + point = self.norm(point) + if self.act is not None: + point = self.act(point) + point.sparsify() + return point + + +class SerializedUnpooling(PointModule): + def __init__( + self, + in_channels, + skip_channels, + out_channels, + norm_layer=None, + act_layer=None, + traceable=False, # record parent and cluster + ): + super().__init__() + self.proj = PointSequential(nn.Linear(in_channels, out_channels)) + self.proj_skip = PointSequential(nn.Linear(skip_channels, out_channels)) + + if norm_layer is not None: + self.proj.add(norm_layer(out_channels)) + self.proj_skip.add(norm_layer(out_channels)) + + if act_layer is not None: + self.proj.add(act_layer()) + self.proj_skip.add(act_layer()) + + self.traceable = traceable + + def forward(self, point): + assert "pooling_parent" in point.keys() + assert "pooling_inverse" in point.keys() + parent = point.pop("pooling_parent") + inverse = point.pop("pooling_inverse") + point = self.proj(point) + parent = self.proj_skip(parent) + parent.feat = parent.feat + point.feat[inverse] + # # no skip connection + # parent.feat = point.feat[inverse] + + if self.traceable: + parent["unpooling_parent"] = point + return parent + + +class Embedding(PointModule): + def __init__( + self, + in_channels, + embed_channels, + norm_layer=None, + act_layer=None, + ): + super().__init__() + self.in_channels = in_channels + self.embed_channels = embed_channels + + # TODO: check remove spconv + self.stem = PointSequential( + conv=spconv.SubMConv3d( + in_channels, + embed_channels, + kernel_size=5, + padding=1, + bias=False, + indice_key="stem", + ) + ) + if norm_layer is not None: + self.stem.add(norm_layer(embed_channels), name="norm") + if act_layer is not None: + self.stem.add(act_layer(), name="act") + + def forward(self, point: Point): + point = self.stem(point) + return point + + +class PointTransformerV3(PointModule): + def __init__( + self, + in_channels=6, + order=("z", "z-trans", "hilbert", "hilbert-trans"), + stride=(2, 2, 2, 2), + enc_depths=(2, 2, 2, 6, 2), + enc_channels=(32, 64, 128, 256, 512), + enc_num_head=(2, 4, 8, 16, 32), + enc_patch_size=(1024, 1024, 1024, 1024, 1024), + dec_depths=(2, 2, 2, 2), + dec_channels=(64, 64, 128, 256), + dec_num_head=(4, 4, 8, 16), + dec_patch_size=(1024, 1024, 1024, 1024), + mlp_ratio=4, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + drop_path=0.3, + pre_norm=True, + shuffle_orders=True, + enable_rpe=False, + enable_flash=True, + upcast_attention=False, + upcast_softmax=False, + cls_mode=False, + pdnorm_bn=False, + pdnorm_ln=False, + pdnorm_decouple=True, + pdnorm_adaptive=False, + pdnorm_affine=True, + pdnorm_conditions=("ScanNet", "S3DIS", "Structured3D"), + ): + super().__init__() + self.num_stages = len(enc_depths) + self.order = [order] if isinstance(order, str) else order + self.cls_mode = cls_mode + self.shuffle_orders = shuffle_orders + + assert self.num_stages == len(stride) + 1 + assert self.num_stages == len(enc_depths) + assert self.num_stages == len(enc_channels) + assert self.num_stages == len(enc_num_head) + assert self.num_stages == len(enc_patch_size) + assert self.cls_mode or self.num_stages == len(dec_depths) + 1 + assert self.cls_mode or self.num_stages == len(dec_channels) + 1 + assert self.cls_mode or self.num_stages == len(dec_num_head) + 1 + assert self.cls_mode or self.num_stages == len(dec_patch_size) + 1 + + # norm layers + if pdnorm_bn: + bn_layer = partial( + PDNorm, + norm_layer=partial( + nn.BatchNorm1d, eps=1e-3, momentum=0.01, affine=pdnorm_affine + ), + conditions=pdnorm_conditions, + decouple=pdnorm_decouple, + adaptive=pdnorm_adaptive, + ) + else: + bn_layer = partial(nn.BatchNorm1d, eps=1e-3, momentum=0.01) + if pdnorm_ln: + ln_layer = partial( + PDNorm, + norm_layer=partial(nn.LayerNorm, elementwise_affine=pdnorm_affine), + conditions=pdnorm_conditions, + decouple=pdnorm_decouple, + adaptive=pdnorm_adaptive, + ) + else: + ln_layer = nn.LayerNorm + # activation layers + act_layer = nn.GELU + + self.embedding = Embedding( + in_channels=in_channels, + embed_channels=enc_channels[0], + norm_layer=bn_layer, + act_layer=act_layer, + ) + + # encoder + enc_drop_path = [ + x.item() for x in torch.linspace(0, drop_path, sum(enc_depths)) + ] + self.enc = PointSequential() + for s in range(self.num_stages): + enc_drop_path_ = enc_drop_path[ + sum(enc_depths[:s]) : sum(enc_depths[: s + 1]) + ] + enc = PointSequential() + if s > 0: + enc.add( + SerializedPooling( + in_channels=enc_channels[s - 1], + out_channels=enc_channels[s], + stride=stride[s - 1], + norm_layer=bn_layer, + act_layer=act_layer, + ), + name="down", + ) + for i in range(enc_depths[s]): + enc.add( + Block( + channels=enc_channels[s], + num_heads=enc_num_head[s], + patch_size=enc_patch_size[s], + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=proj_drop, + drop_path=enc_drop_path_[i], + norm_layer=ln_layer, + act_layer=act_layer, + pre_norm=pre_norm, + order_index=i % len(self.order), + cpe_indice_key=f"stage{s}", + enable_rpe=enable_rpe, + enable_flash=enable_flash, + upcast_attention=upcast_attention, + upcast_softmax=upcast_softmax, + ), + name=f"block{i}", + ) + if len(enc) != 0: + self.enc.add(module=enc, name=f"enc{s}") + + # decoder + if not self.cls_mode: + dec_drop_path = [ + x.item() for x in torch.linspace(0, drop_path, sum(dec_depths)) + ] + self.dec = PointSequential() + dec_channels = list(dec_channels) + [enc_channels[-1]] + for s in reversed(range(self.num_stages - 1)): + dec_drop_path_ = dec_drop_path[ + sum(dec_depths[:s]) : sum(dec_depths[: s + 1]) + ] + dec_drop_path_.reverse() + dec = PointSequential() + dec.add( + SerializedUnpooling( + in_channels=dec_channels[s + 1], + skip_channels=enc_channels[s], + out_channels=dec_channels[s], + norm_layer=bn_layer, + act_layer=act_layer, + ), + name="up", + ) + for i in range(dec_depths[s]): + dec.add( + Block( + channels=dec_channels[s], + num_heads=dec_num_head[s], + patch_size=dec_patch_size[s], + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=proj_drop, + drop_path=dec_drop_path_[i], + norm_layer=ln_layer, + act_layer=act_layer, + pre_norm=pre_norm, + order_index=i % len(self.order), + cpe_indice_key=f"stage{s}", + enable_rpe=enable_rpe, + enable_flash=enable_flash, + upcast_attention=upcast_attention, + upcast_softmax=upcast_softmax, + ), + name=f"block{i}", + ) + self.dec.add(module=dec, name=f"dec{s}") + + def forward(self, data_dict): + """ + A data_dict is a dictionary containing properties of a batched point cloud. + It should contain the following properties for PTv3: + 1. "feat": feature of point cloud + 2. "grid_coord": discrete coordinate after grid sampling (voxelization) or "coord" + "grid_size" + 3. "offset" or "batch": https://github.com/Pointcept/Pointcept?tab=readme-ov-file#offset + """ + point = Point(data_dict) + point.serialization(order=self.order, shuffle_orders=self.shuffle_orders) + point.sparsify() + + point = self.embedding(point) + point = self.enc(point) + if not self.cls_mode: + point = self.dec(point) + return point diff --git a/submodules/PointTransformerV3/serialization/__init__.py b/submodules/PointTransformerV3/serialization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..86159d07ca3405e01de235dc8bf921a2752b8ce8 --- /dev/null +++ b/submodules/PointTransformerV3/serialization/__init__.py @@ -0,0 +1,8 @@ +from .default import ( + encode, + decode, + z_order_encode, + z_order_decode, + hilbert_encode, + hilbert_decode, +) diff --git a/submodules/PointTransformerV3/serialization/default.py b/submodules/PointTransformerV3/serialization/default.py new file mode 100644 index 0000000000000000000000000000000000000000..ea76c88e2eef3308e700230bc11ae3cab77c43e9 --- /dev/null +++ b/submodules/PointTransformerV3/serialization/default.py @@ -0,0 +1,59 @@ +import torch +from .z_order import xyz2key as z_order_encode_ +from .z_order import key2xyz as z_order_decode_ +from .hilbert import encode as hilbert_encode_ +from .hilbert import decode as hilbert_decode_ + + +@torch.inference_mode() +def encode(grid_coord, batch=None, depth=16, order="z"): + assert order in {"z", "z-trans", "hilbert", "hilbert-trans"} + if order == "z": + code = z_order_encode(grid_coord, depth=depth) + elif order == "z-trans": + code = z_order_encode(grid_coord[:, [1, 0, 2]], depth=depth) + elif order == "hilbert": + code = hilbert_encode(grid_coord, depth=depth) + elif order == "hilbert-trans": + code = hilbert_encode(grid_coord[:, [1, 0, 2]], depth=depth) + else: + raise NotImplementedError + if batch is not None: + batch = batch.long() + code = batch << depth * 3 | code + return code + + +@torch.inference_mode() +def decode(code, depth=16, order="z"): + assert order in {"z", "hilbert"} + batch = code >> depth * 3 + code = code & ((1 << depth * 3) - 1) + if order == "z": + grid_coord = z_order_decode(code, depth=depth) + elif order == "hilbert": + grid_coord = hilbert_decode(code, depth=depth) + else: + raise NotImplementedError + return grid_coord, batch + + +def z_order_encode(grid_coord: torch.Tensor, depth: int = 16): + x, y, z = grid_coord[:, 0].long(), grid_coord[:, 1].long(), grid_coord[:, 2].long() + # we block the support to batch, maintain batched code in Point class + code = z_order_encode_(x, y, z, b=None, depth=depth) + return code + + +def z_order_decode(code: torch.Tensor, depth): + x, y, z = z_order_decode_(code, depth=depth) + grid_coord = torch.stack([x, y, z], dim=-1) # (N, 3) + return grid_coord + + +def hilbert_encode(grid_coord: torch.Tensor, depth: int = 16): + return hilbert_encode_(grid_coord, num_dims=3, num_bits=depth) + + +def hilbert_decode(code: torch.Tensor, depth: int = 16): + return hilbert_decode_(code, num_dims=3, num_bits=depth) diff --git a/submodules/PointTransformerV3/serialization/hilbert.py b/submodules/PointTransformerV3/serialization/hilbert.py new file mode 100644 index 0000000000000000000000000000000000000000..682be19e296beaa26448f9485d0a226e1adc9f0b --- /dev/null +++ b/submodules/PointTransformerV3/serialization/hilbert.py @@ -0,0 +1,303 @@ +""" +Hilbert Order +Modified from https://github.com/PrincetonLIPS/numpy-hilbert-curve + +Author: Xiaoyang Wu (xiaoyang.wu.cs@gmail.com), Kaixin Xu +Please cite our work if the code is helpful to you. +""" + +import torch + + +def right_shift(binary, k=1, axis=-1): + """Right shift an array of binary values. + + Parameters: + ----------- + binary: An ndarray of binary values. + + k: The number of bits to shift. Default 1. + + axis: The axis along which to shift. Default -1. + + Returns: + -------- + Returns an ndarray with zero prepended and the ends truncated, along + whatever axis was specified.""" + + # If we're shifting the whole thing, just return zeros. + if binary.shape[axis] <= k: + return torch.zeros_like(binary) + + # Determine the padding pattern. + # padding = [(0,0)] * len(binary.shape) + # padding[axis] = (k,0) + + # Determine the slicing pattern to eliminate just the last one. + slicing = [slice(None)] * len(binary.shape) + slicing[axis] = slice(None, -k) + shifted = torch.nn.functional.pad( + binary[tuple(slicing)], (k, 0), mode="constant", value=0 + ) + + return shifted + + +def binary2gray(binary, axis=-1): + """Convert an array of binary values into Gray codes. + + This uses the classic X ^ (X >> 1) trick to compute the Gray code. + + Parameters: + ----------- + binary: An ndarray of binary values. + + axis: The axis along which to compute the gray code. Default=-1. + + Returns: + -------- + Returns an ndarray of Gray codes. + """ + shifted = right_shift(binary, axis=axis) + + # Do the X ^ (X >> 1) trick. + gray = torch.logical_xor(binary, shifted) + + return gray + + +def gray2binary(gray, axis=-1): + """Convert an array of Gray codes back into binary values. + + Parameters: + ----------- + gray: An ndarray of gray codes. + + axis: The axis along which to perform Gray decoding. Default=-1. + + Returns: + -------- + Returns an ndarray of binary values. + """ + + # Loop the log2(bits) number of times necessary, with shift and xor. + shift = 2 ** (torch.Tensor([gray.shape[axis]]).log2().ceil().int() - 1) + while shift > 0: + gray = torch.logical_xor(gray, right_shift(gray, shift)) + shift = torch.div(shift, 2, rounding_mode="floor") + return gray + + +def encode(locs, num_dims, num_bits): + """Decode an array of locations in a hypercube into a Hilbert integer. + + This is a vectorized-ish version of the Hilbert curve implementation by John + Skilling as described in: + + Skilling, J. (2004, April). Programming the Hilbert curve. In AIP Conference + Proceedings (Vol. 707, No. 1, pp. 381-387). American Institute of Physics. + + Params: + ------- + locs - An ndarray of locations in a hypercube of num_dims dimensions, in + which each dimension runs from 0 to 2**num_bits-1. The shape can + be arbitrary, as long as the last dimension of the same has size + num_dims. + + num_dims - The dimensionality of the hypercube. Integer. + + num_bits - The number of bits for each dimension. Integer. + + Returns: + -------- + The output is an ndarray of uint64 integers with the same shape as the + input, excluding the last dimension, which needs to be num_dims. + """ + + # Keep around the original shape for later. + orig_shape = locs.shape + bitpack_mask = 1 << torch.arange(0, 8).to(locs.device) + bitpack_mask_rev = bitpack_mask.flip(-1) + + if orig_shape[-1] != num_dims: + raise ValueError( + """ + The shape of locs was surprising in that the last dimension was of size + %d, but num_dims=%d. These need to be equal. + """ + % (orig_shape[-1], num_dims) + ) + + if num_dims * num_bits > 63: + raise ValueError( + """ + num_dims=%d and num_bits=%d for %d bits total, which can't be encoded + into a int64. Are you sure you need that many points on your Hilbert + curve? + """ + % (num_dims, num_bits, num_dims * num_bits) + ) + + # Treat the location integers as 64-bit unsigned and then split them up into + # a sequence of uint8s. Preserve the association by dimension. + locs_uint8 = locs.long().view(torch.uint8).reshape((-1, num_dims, 8)).flip(-1) + + # Now turn these into bits and truncate to num_bits. + gray = ( + locs_uint8.unsqueeze(-1) + .bitwise_and(bitpack_mask_rev) + .ne(0) + .byte() + .flatten(-2, -1)[..., -num_bits:] + ) + + # Run the decoding process the other way. + # Iterate forwards through the bits. + for bit in range(0, num_bits): + # Iterate forwards through the dimensions. + for dim in range(0, num_dims): + # Identify which ones have this bit active. + mask = gray[:, dim, bit] + + # Where this bit is on, invert the 0 dimension for lower bits. + gray[:, 0, bit + 1 :] = torch.logical_xor( + gray[:, 0, bit + 1 :], mask[:, None] + ) + + # Where the bit is off, exchange the lower bits with the 0 dimension. + to_flip = torch.logical_and( + torch.logical_not(mask[:, None]).repeat(1, gray.shape[2] - bit - 1), + torch.logical_xor(gray[:, 0, bit + 1 :], gray[:, dim, bit + 1 :]), + ) + gray[:, dim, bit + 1 :] = torch.logical_xor( + gray[:, dim, bit + 1 :], to_flip + ) + gray[:, 0, bit + 1 :] = torch.logical_xor(gray[:, 0, bit + 1 :], to_flip) + + # Now flatten out. + gray = gray.swapaxes(1, 2).reshape((-1, num_bits * num_dims)) + + # Convert Gray back to binary. + hh_bin = gray2binary(gray) + + # Pad back out to 64 bits. + extra_dims = 64 - num_bits * num_dims + padded = torch.nn.functional.pad(hh_bin, (extra_dims, 0), "constant", 0) + + # Convert binary values into uint8s. + hh_uint8 = ( + (padded.flip(-1).reshape((-1, 8, 8)) * bitpack_mask) + .sum(2) + .squeeze() + .type(torch.uint8) + ) + + # Convert uint8s into uint64s. + hh_uint64 = hh_uint8.view(torch.int64).squeeze() + + return hh_uint64 + + +def decode(hilberts, num_dims, num_bits): + """Decode an array of Hilbert integers into locations in a hypercube. + + This is a vectorized-ish version of the Hilbert curve implementation by John + Skilling as described in: + + Skilling, J. (2004, April). Programming the Hilbert curve. In AIP Conference + Proceedings (Vol. 707, No. 1, pp. 381-387). American Institute of Physics. + + Params: + ------- + hilberts - An ndarray of Hilbert integers. Must be an integer dtype and + cannot have fewer bits than num_dims * num_bits. + + num_dims - The dimensionality of the hypercube. Integer. + + num_bits - The number of bits for each dimension. Integer. + + Returns: + -------- + The output is an ndarray of unsigned integers with the same shape as hilberts + but with an additional dimension of size num_dims. + """ + + if num_dims * num_bits > 64: + raise ValueError( + """ + num_dims=%d and num_bits=%d for %d bits total, which can't be encoded + into a uint64. Are you sure you need that many points on your Hilbert + curve? + """ + % (num_dims, num_bits) + ) + + # Handle the case where we got handed a naked integer. + hilberts = torch.atleast_1d(hilberts) + + # Keep around the shape for later. + orig_shape = hilberts.shape + bitpack_mask = 2 ** torch.arange(0, 8).to(hilberts.device) + bitpack_mask_rev = bitpack_mask.flip(-1) + + # Treat each of the hilberts as a s equence of eight uint8. + # This treats all of the inputs as uint64 and makes things uniform. + hh_uint8 = ( + hilberts.ravel().type(torch.int64).view(torch.uint8).reshape((-1, 8)).flip(-1) + ) + + # Turn these lists of uints into lists of bits and then truncate to the size + # we actually need for using Skilling's procedure. + hh_bits = ( + hh_uint8.unsqueeze(-1) + .bitwise_and(bitpack_mask_rev) + .ne(0) + .byte() + .flatten(-2, -1)[:, -num_dims * num_bits :] + ) + + # Take the sequence of bits and Gray-code it. + gray = binary2gray(hh_bits) + + # There has got to be a better way to do this. + # I could index them differently, but the eventual packbits likes it this way. + gray = gray.reshape((-1, num_bits, num_dims)).swapaxes(1, 2) + + # Iterate backwards through the bits. + for bit in range(num_bits - 1, -1, -1): + # Iterate backwards through the dimensions. + for dim in range(num_dims - 1, -1, -1): + # Identify which ones have this bit active. + mask = gray[:, dim, bit] + + # Where this bit is on, invert the 0 dimension for lower bits. + gray[:, 0, bit + 1 :] = torch.logical_xor( + gray[:, 0, bit + 1 :], mask[:, None] + ) + + # Where the bit is off, exchange the lower bits with the 0 dimension. + to_flip = torch.logical_and( + torch.logical_not(mask[:, None]), + torch.logical_xor(gray[:, 0, bit + 1 :], gray[:, dim, bit + 1 :]), + ) + gray[:, dim, bit + 1 :] = torch.logical_xor( + gray[:, dim, bit + 1 :], to_flip + ) + gray[:, 0, bit + 1 :] = torch.logical_xor(gray[:, 0, bit + 1 :], to_flip) + + # Pad back out to 64 bits. + extra_dims = 64 - num_bits + padded = torch.nn.functional.pad(gray, (extra_dims, 0), "constant", 0) + + # Now chop these up into blocks of 8. + locs_chopped = padded.flip(-1).reshape((-1, num_dims, 8, 8)) + + # Take those blocks and turn them unto uint8s. + # from IPython import embed; embed() + locs_uint8 = (locs_chopped * bitpack_mask).sum(3).squeeze().type(torch.uint8) + + # Finally, treat these as uint64s. + flat_locs = locs_uint8.view(torch.int64) + + # Return them in the expected shape. + return flat_locs.reshape((*orig_shape, num_dims)) diff --git a/submodules/PointTransformerV3/serialization/z_order.py b/submodules/PointTransformerV3/serialization/z_order.py new file mode 100644 index 0000000000000000000000000000000000000000..2fa498fafc371365c5462f678adf35f88cae2c8b --- /dev/null +++ b/submodules/PointTransformerV3/serialization/z_order.py @@ -0,0 +1,126 @@ +# -------------------------------------------------------- +# Octree-based Sparse Convolutional Neural Networks +# Copyright (c) 2022 Peng-Shuai Wang +# Licensed under The MIT License [see LICENSE for details] +# Written by Peng-Shuai Wang +# -------------------------------------------------------- + +import torch +from typing import Optional, Union + + +class KeyLUT: + def __init__(self): + r256 = torch.arange(256, dtype=torch.int64) + r512 = torch.arange(512, dtype=torch.int64) + zero = torch.zeros(256, dtype=torch.int64) + device = torch.device("cpu") + + self._encode = { + device: ( + self.xyz2key(r256, zero, zero, 8), + self.xyz2key(zero, r256, zero, 8), + self.xyz2key(zero, zero, r256, 8), + ) + } + self._decode = {device: self.key2xyz(r512, 9)} + + def encode_lut(self, device=torch.device("cpu")): + if device not in self._encode: + cpu = torch.device("cpu") + self._encode[device] = tuple(e.to(device) for e in self._encode[cpu]) + return self._encode[device] + + def decode_lut(self, device=torch.device("cpu")): + if device not in self._decode: + cpu = torch.device("cpu") + self._decode[device] = tuple(e.to(device) for e in self._decode[cpu]) + return self._decode[device] + + def xyz2key(self, x, y, z, depth): + key = torch.zeros_like(x) + for i in range(depth): + mask = 1 << i + key = ( + key + | ((x & mask) << (2 * i + 2)) + | ((y & mask) << (2 * i + 1)) + | ((z & mask) << (2 * i + 0)) + ) + return key + + def key2xyz(self, key, depth): + x = torch.zeros_like(key) + y = torch.zeros_like(key) + z = torch.zeros_like(key) + for i in range(depth): + x = x | ((key & (1 << (3 * i + 2))) >> (2 * i + 2)) + y = y | ((key & (1 << (3 * i + 1))) >> (2 * i + 1)) + z = z | ((key & (1 << (3 * i + 0))) >> (2 * i + 0)) + return x, y, z + + +_key_lut = KeyLUT() + + +def xyz2key( + x: torch.Tensor, + y: torch.Tensor, + z: torch.Tensor, + b: Optional[Union[torch.Tensor, int]] = None, + depth: int = 16, +): + r"""Encodes :attr:`x`, :attr:`y`, :attr:`z` coordinates to the shuffled keys + based on pre-computed look up tables. The speed of this function is much + faster than the method based on for-loop. + + Args: + x (torch.Tensor): The x coordinate. + y (torch.Tensor): The y coordinate. + z (torch.Tensor): The z coordinate. + b (torch.Tensor or int): The batch index of the coordinates, and should be + smaller than 32768. If :attr:`b` is :obj:`torch.Tensor`, the size of + :attr:`b` must be the same as :attr:`x`, :attr:`y`, and :attr:`z`. + depth (int): The depth of the shuffled key, and must be smaller than 17 (< 17). + """ + + EX, EY, EZ = _key_lut.encode_lut(x.device) + x, y, z = x.long(), y.long(), z.long() + + mask = 255 if depth > 8 else (1 << depth) - 1 + key = EX[x & mask] | EY[y & mask] | EZ[z & mask] + if depth > 8: + mask = (1 << (depth - 8)) - 1 + key16 = EX[(x >> 8) & mask] | EY[(y >> 8) & mask] | EZ[(z >> 8) & mask] + key = key16 << 24 | key + + if b is not None: + b = b.long() + key = b << 48 | key + + return key + + +def key2xyz(key: torch.Tensor, depth: int = 16): + r"""Decodes the shuffled key to :attr:`x`, :attr:`y`, :attr:`z` coordinates + and the batch index based on pre-computed look up tables. + + Args: + key (torch.Tensor): The shuffled key. + depth (int): The depth of the shuffled key, and must be smaller than 17 (< 17). + """ + + DX, DY, DZ = _key_lut.decode_lut(key.device) + x, y, z = torch.zeros_like(key), torch.zeros_like(key), torch.zeros_like(key) + + b = key >> 48 + key = key & ((1 << 48) - 1) + + n = (depth + 2) // 3 + for i in range(n): + k = key >> (i * 9) & 511 + x = x | (DX[k] << (i * 3)) + y = y | (DY[k] << (i * 3)) + z = z | (DZ[k] << (i * 3)) + + return x, y, z, b diff --git a/submodules/lang_seg/.gitignore b/submodules/lang_seg/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..034ea5f0cef5d7514ee0b5ab4ad444702e60a6b6 --- /dev/null +++ b/submodules/lang_seg/.gitignore @@ -0,0 +1,131 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +checkpoints/ +logs/ diff --git a/submodules/lang_seg/LICENSE b/submodules/lang_seg/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..96cb80f815c71311e3bbb1e6e86a52a90912fa00 --- /dev/null +++ b/submodules/lang_seg/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Intelligent Systems Lab Org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/submodules/lang_seg/README.MD b/submodules/lang_seg/README.MD new file mode 100644 index 0000000000000000000000000000000000000000..71862236d310e1993805e179369fcfadcd33dc30 --- /dev/null +++ b/submodules/lang_seg/README.MD @@ -0,0 +1,243 @@ +# Language-driven Semantic Segmentation (LSeg) +The repo contains official PyTorch Implementation of paper [Language-driven Semantic Segmentation](https://arxiv.org/abs/2201.03546). + +ICLR 2022 + +#### Authors: +* [Boyi Li](https://sites.google.com/site/boyilics/home) +* [Kilian Q. Weinberger](http://kilian.cs.cornell.edu/index.html) +* [Serge Belongie](https://scholar.google.com/citations?user=ORr4XJYAAAAJ&hl=zh-CN) +* [Vladlen Koltun](http://vladlen.info/) +* [Rene Ranftl](https://scholar.google.at/citations?user=cwKg158AAAAJ&hl=de) + + +### Overview + + +We present LSeg, a novel model for language-driven semantic image segmentation. LSeg uses a text encoder to compute embeddings of descriptive input labels (e.g., ''grass'' or 'building'') together with a transformer-based image encoder that computes dense per-pixel embeddings of the input image. The image encoder is trained with a contrastive objective to align pixel embeddings to the text embedding of the corresponding semantic class. The text embeddings provide a flexible label representation in which semantically similar labels map to similar regions in the embedding space (e.g., ''cat'' and ''furry''). This allows LSeg to generalize to previously unseen categories at test time, without retraining or even requiring a single additional training sample. We demonstrate that our approach achieves highly competitive zero-shot performance compared to existing zero- and few-shot semantic segmentation methods, and even matches the accuracy of traditional segmentation algorithms when a fixed label set is provided. + +Please check our [Video Demo (4k)](https://www.youtube.com/watch?v=bmU75rsmv6s) to further showcase the capabilities of LSeg. + +## Usage +### Installation +Option 1: + +``` pip install -r requirements.txt ``` + +Option 2: +``` +conda install ipython +pip install torch==1.7.1 torchvision==0.8.2 torchaudio==0.7.2 +pip install git+https://github.com/zhanghang1989/PyTorch-Encoding/ +pip install pytorch-lightning==1.3.5 +pip install opencv-python +pip install imageio +pip install ftfy regex tqdm +pip install git+https://github.com/openai/CLIP.git +pip install altair +pip install streamlit +pip install --upgrade protobuf +pip install timm +pip install tensorboardX +pip install matplotlib +pip install test-tube +pip install wandb +``` + +### Data Preparation +By default, for training, testing and demo, we use [ADE20k](https://groups.csail.mit.edu/vision/datasets/ADE20K/). + +``` +python prepare_ade20k.py +unzip ../datasets/ADEChallengeData2016.zip +``` + +Note: for demo, if you want to use random inputs, you can ignore data loading and comment the code at [link](https://github.com/isl-org/lang-seg/blob/main/modules/lseg_module.py#L55). + + +### 🌻 Try demo now + +#### Download Demo Model + + + + + + + + + + + + + + + + + +
namebackbonetext encoderurl
Model for demoViT-L/16CLIP ViT-B/32download
+ +#### 👉 Option 1: Running interactive app +Download the model for demo and put it under folder `checkpoints` as `checkpoints/demo_e200.ckpt`. + +Then ``` streamlit run lseg_app.py ``` + +#### 👉 Option 2: Jupyter Notebook +Download the model for demo and put it under folder `checkpoints` as `checkpoints/demo_e200.ckpt`. + +Then follow [lseg_demo.ipynb](https://github.com/isl-org/lang-seg/blob/main/lseg_demo.ipynb) to play around with LSeg. Enjoy! + + + +### Training and Testing Example +Training: Backbone = ViT-L/16, Text Encoder from CLIP ViT-B/32 + +``` bash train.sh ``` + +Testing: Backbone = ViT-L/16, Text Encoder from CLIP ViT-B/32 + +``` bash test.sh ``` + +### Zero-shot Experiments +#### Data Preparation +Please follow [HSNet](https://github.com/juhongm999/hsnet) and put all dataset in `data/Dataset_HSN` + +#### Pascal-5i +``` +for fold in 0 1 2 3; do +python -u test_lseg_zs.py --backbone clip_resnet101 --module clipseg_DPT_test_v2 --dataset pascal \ +--widehead --no-scaleinv --arch_option 0 --ignore_index 255 --fold ${fold} --nshot 0 \ +--weights checkpoints/pascal_fold${fold}.ckpt +done +``` +#### COCO-20i +``` +for fold in 0 1 2 3; do +python -u test_lseg_zs.py --backbone clip_resnet101 --module clipseg_DPT_test_v2 --dataset coco \ +--widehead --no-scaleinv --arch_option 0 --ignore_index 255 --fold ${fold} --nshot 0 \ +--weights checkpoints/pascal_fold${fold}.ckpt +done +``` +#### FSS +``` +python -u test_lseg_zs.py --backbone clip_vitl16_384 --module clipseg_DPT_test_v2 --dataset fss \ +--widehead --no-scaleinv --arch_option 0 --ignore_index 255 --fold 0 --nshot 0 \ +--weights checkpoints/fss_l16.ckpt +``` + +``` +python -u test_lseg_zs.py --backbone clip_resnet101 --module clipseg_DPT_test_v2 --dataset fss \ +--widehead --no-scaleinv --arch_option 0 --ignore_index 255 --fold 0 --nshot 0 \ +--weights checkpoints/fss_rn101.ckpt +``` + +#### Model Zoo + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
datasetfoldbackbonetext encoderperformanceurl
pascal0ResNet101CLIP ViT-B/3252.8download
pascal1ResNet101CLIP ViT-B/3253.8download
pascal2ResNet101CLIP ViT-B/3244.4download
pascal3ResNet101CLIP ViT-B/3238.5download
coco0ResNet101CLIP ViT-B/3222.1download
coco1ResNet101CLIP ViT-B/3225.1download
coco2ResNet101CLIP ViT-B/3224.9download
coco3ResNet101CLIP ViT-B/3221.5download
fss-ResNet101CLIP ViT-B/3284.7download
fss-ViT-L/16CLIP ViT-B/3287.8download
+ +If you find this repo useful, please cite: +``` +@inproceedings{ +li2022languagedriven, +title={Language-driven Semantic Segmentation}, +author={Boyi Li and Kilian Q Weinberger and Serge Belongie and Vladlen Koltun and Rene Ranftl}, +booktitle={International Conference on Learning Representations}, +year={2022}, +url={https://openreview.net/forum?id=RriDjddCLN} +} +``` + +## Acknowledgement +Thanks to the code base from [DPT](https://github.com/isl-org/DPT), [Pytorch_lightning](https://github.com/PyTorchLightning/pytorch-lightning), [CLIP](https://github.com/openai/CLIP), [Pytorch Encoding](https://github.com/zhanghang1989/PyTorch-Encoding), [Streamlit](https://streamlit.io/), [Wandb](https://wandb.ai/site) diff --git a/submodules/lang_seg/additional_utils/encoding_models.py b/submodules/lang_seg/additional_utils/encoding_models.py new file mode 100644 index 0000000000000000000000000000000000000000..9d98809a6030e8941396bbda439a7f978dbb5266 --- /dev/null +++ b/submodules/lang_seg/additional_utils/encoding_models.py @@ -0,0 +1,164 @@ +########################################################################### +# Referred to: https://github.com/zhanghang1989/PyTorch-Encoding +########################################################################### +import math +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.parallel.data_parallel import DataParallel +from torch.nn.parallel.scatter_gather import scatter +import threading +import torch +from torch.cuda._utils import _get_device_index +from torch.cuda.amp import autocast +from torch._utils import ExceptionWrapper + +up_kwargs = {'mode': 'bilinear', 'align_corners': True} + +__all__ = ['MultiEvalModule'] + +class MultiEvalModule(DataParallel): + """Multi-size Segmentation Eavluator""" + def __init__(self, module, nclass, device_ids=None, flip=True, + scales=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75]): + super(MultiEvalModule, self).__init__(module, device_ids) + self.nclass = nclass + self.base_size = module.base_size + self.crop_size = module.crop_size + self.scales = scales + self.flip = flip + print('MultiEvalModule: base_size {}, crop_size {}'. \ + format(self.base_size, self.crop_size)) + + def parallel_forward(self, inputs, **kwargs): + """Multi-GPU Mult-size Evaluation + + Args: + inputs: list of Tensors + """ + inputs = [(input.unsqueeze(0).cuda(device),) + for input, device in zip(inputs, self.device_ids)] + replicas = self.replicate(self, self.device_ids[:len(inputs)]) + kwargs = scatter(kwargs, target_gpus, dim) if kwargs else [] + if len(inputs) < len(kwargs): + inputs.extend([() for _ in range(len(kwargs) - len(inputs))]) + elif len(kwargs) < len(inputs): + kwargs.extend([{} for _ in range(len(inputs) - len(kwargs))]) + outputs = self.parallel_apply(replicas, inputs, kwargs) + #for out in outputs: + # print('out.size()', out.size()) + return outputs + + def forward(self, image): + """Mult-size Evaluation""" + # only single image is supported for evaluation + batch, _, h, w = image.size() + assert(batch == 1) + stride_rate = 2.0/3.0 + crop_size = self.crop_size + stride = int(crop_size * stride_rate) + with torch.cuda.device_of(image): + scores = image.new().resize_(batch,self.nclass,h,w).zero_().cuda() + + for scale in self.scales: + long_size = int(math.ceil(self.base_size * scale)) + if h > w: + height = long_size + width = int(1.0 * w * long_size / h + 0.5) + short_size = width + else: + width = long_size + height = int(1.0 * h * long_size / w + 0.5) + short_size = height + """ + short_size = int(math.ceil(self.base_size * scale)) + if h > w: + width = short_size + height = int(1.0 * h * short_size / w) + long_size = height + else: + height = short_size + width = int(1.0 * w * short_size / h) + long_size = width + """ + # resize image to current size + cur_img = resize_image(image, height, width, **self.module._up_kwargs) + if long_size <= crop_size: + pad_img = pad_image(cur_img, self.module.mean, + self.module.std, crop_size) + outputs = module_inference(self.module, pad_img, self.flip) + outputs = crop_image(outputs, 0, height, 0, width) + else: + if short_size < crop_size: + # pad if needed + pad_img = pad_image(cur_img, self.module.mean, + self.module.std, crop_size) + else: + pad_img = cur_img + _,_,ph,pw = pad_img.size() + assert(ph >= height and pw >= width) + # grid forward and normalize + h_grids = int(math.ceil(1.0 * (ph-crop_size)/stride)) + 1 + w_grids = int(math.ceil(1.0 * (pw-crop_size)/stride)) + 1 + with torch.cuda.device_of(image): + outputs = image.new().resize_(batch,self.nclass,ph,pw).zero_().cuda() + count_norm = image.new().resize_(batch,1,ph,pw).zero_().cuda() + # grid evaluation + for idh in range(h_grids): + for idw in range(w_grids): + h0 = idh * stride + w0 = idw * stride + h1 = min(h0 + crop_size, ph) + w1 = min(w0 + crop_size, pw) + crop_img = crop_image(pad_img, h0, h1, w0, w1) + # pad if needed + pad_crop_img = pad_image(crop_img, self.module.mean, + self.module.std, crop_size) + output = module_inference(self.module, pad_crop_img, self.flip) + outputs[:,:,h0:h1,w0:w1] += crop_image(output, + 0, h1-h0, 0, w1-w0) + count_norm[:,:,h0:h1,w0:w1] += 1 + assert((count_norm==0).sum()==0) + outputs = outputs / count_norm + outputs = outputs[:,:,:height,:width] + + score = resize_image(outputs, h, w, **self.module._up_kwargs) + scores += score + + return scores + + +def module_inference(module, image, flip=True): + output = module.evaluate(image) + if flip: + fimg = flip_image(image) + foutput = module.evaluate(fimg) + output += flip_image(foutput) + return output + +def resize_image(img, h, w, **up_kwargs): + return F.interpolate(img, (h, w), **up_kwargs) + +def pad_image(img, mean, std, crop_size): + b,c,h,w = img.size() + assert(c==3) + padh = crop_size - h if h < crop_size else 0 + padw = crop_size - w if w < crop_size else 0 + pad_values = -np.array(mean) / np.array(std) + img_pad = img.new().resize_(b,c,h+padh,w+padw) + for i in range(c): + # note that pytorch pad params is in reversed orders + img_pad[:,i,:,:] = F.pad(img[:,i,:,:], (0, padw, 0, padh), value=pad_values[i]) + assert(img_pad.size(2)>=crop_size and img_pad.size(3)>=crop_size) + return img_pad + +def crop_image(img, h0, h1, w0, w1): + return img[:,:,h0:h1,w0:w1] + +def flip_image(img): + assert(img.dim()==4) + with torch.cuda.device_of(img): + idx = torch.arange(img.size(3)-1, -1, -1).type_as(img).long() + return img.index_select(3, idx) \ No newline at end of file diff --git a/submodules/lang_seg/additional_utils/models.py b/submodules/lang_seg/additional_utils/models.py new file mode 100644 index 0000000000000000000000000000000000000000..c7ce035564b5eb6ca7529376434c4fcbfa89e46f --- /dev/null +++ b/submodules/lang_seg/additional_utils/models.py @@ -0,0 +1,250 @@ +########################################################################### +# Referred to: https://github.com/zhanghang1989/PyTorch-Encoding +########################################################################### +import math +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.parallel.data_parallel import DataParallel +from torch.nn.parallel.scatter_gather import scatter +import threading +import torch +from torch.cuda._utils import _get_device_index +from torch.cuda.amp import autocast +from torch._utils import ExceptionWrapper + +up_kwargs = {'mode': 'bilinear', 'align_corners': True} + +__all__ = ['LSeg_MultiEvalModule'] + + +class LSeg_MultiEvalModule(DataParallel): + """Multi-size Segmentation Eavluator""" + def __init__(self, module, device_ids=None, flip=True, + scales=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75]): + super(LSeg_MultiEvalModule, self).__init__(module, device_ids) + self.base_size = module.base_size + self.crop_size = module.crop_size + self.scales = scales + self.flip = flip + print('MultiEvalModule: base_size {}, crop_size {}'. \ + format(self.base_size, self.crop_size)) + + def parallel_forward(self, inputs, label_set='', **kwargs): + """Multi-GPU Mult-size Evaluation + + Args: + inputs: list of Tensors + """ + if len(label_set) < 10: + print('** MultiEvalModule parallel_forward phase: {} **'.format(label_set)) + self.nclass = len(label_set) + inputs = [(input.unsqueeze(0).cuda(device),) + for input, device in zip(inputs, self.device_ids)] + replicas = self.replicate(self, self.device_ids[:len(inputs)]) + kwargs = scatter(kwargs, target_gpus, dim) if kwargs else [] + if len(inputs) < len(kwargs): + inputs.extend([() for _ in range(len(kwargs) - len(inputs))]) + elif len(kwargs) < len(inputs): + kwargs.extend([{} for _ in range(len(inputs) - len(kwargs))]) + outputs = parallel_apply(replicas, inputs, label_set, kwargs) + return outputs + + def forward(self, image, label_set=''): + """Mult-size Evaluation""" + # only single image is supported for evaluation + if len(label_set) < 10: + print('** MultiEvalModule forward phase: {} **'.format(label_set)) + batch, _, h, w = image.size() + assert(batch == 1) + self.nclass = len(label_set) + stride_rate = 2.0/3.0 + crop_size = self.crop_size + stride = int(crop_size * stride_rate) + with torch.cuda.device_of(image): + scores = image.new().resize_(batch,self.nclass,h,w).zero_().cuda() + + for scale in self.scales: + long_size = int(math.ceil(self.base_size * scale)) + if h > w: + height = long_size + width = int(1.0 * w * long_size / h + 0.5) + short_size = width + else: + width = long_size + height = int(1.0 * h * long_size / w + 0.5) + short_size = height + """ + short_size = int(math.ceil(self.base_size * scale)) + if h > w: + width = short_size + height = int(1.0 * h * short_size / w) + long_size = height + else: + height = short_size + width = int(1.0 * w * short_size / h) + long_size = width + """ + # resize image to current size + cur_img = resize_image(image, height, width, **self.module._up_kwargs) + if long_size <= crop_size: + pad_img = pad_image(cur_img, self.module.mean, + self.module.std, crop_size) + outputs = module_inference(self.module, pad_img, label_set, self.flip) + outputs = crop_image(outputs, 0, height, 0, width) + else: + if short_size < crop_size: + # pad if needed + pad_img = pad_image(cur_img, self.module.mean, + self.module.std, crop_size) + else: + pad_img = cur_img + _,_,ph,pw = pad_img.shape #.size() + assert(ph >= height and pw >= width) + # grid forward and normalize + h_grids = int(math.ceil(1.0 * (ph-crop_size)/stride)) + 1 + w_grids = int(math.ceil(1.0 * (pw-crop_size)/stride)) + 1 + with torch.cuda.device_of(image): + outputs = image.new().resize_(batch,self.nclass,ph,pw).zero_().cuda() + count_norm = image.new().resize_(batch,1,ph,pw).zero_().cuda() + # grid evaluation + for idh in range(h_grids): + for idw in range(w_grids): + h0 = idh * stride + w0 = idw * stride + h1 = min(h0 + crop_size, ph) + w1 = min(w0 + crop_size, pw) + crop_img = crop_image(pad_img, h0, h1, w0, w1) + # pad if needed + pad_crop_img = pad_image(crop_img, self.module.mean, + self.module.std, crop_size) + output = module_inference(self.module, pad_crop_img, label_set, self.flip) + outputs[:,:,h0:h1,w0:w1] += crop_image(output, + 0, h1-h0, 0, w1-w0) + count_norm[:,:,h0:h1,w0:w1] += 1 + assert((count_norm==0).sum()==0) + outputs = outputs / count_norm + outputs = outputs[:,:,:height,:width] + score = resize_image(outputs, h, w, **self.module._up_kwargs) + scores += score + return scores + +def module_inference(module, image, label_set, flip=True): + output = module.evaluate_random(image, label_set) + if flip: + fimg = flip_image(image) + foutput = module.evaluate_random(fimg, label_set) + output += flip_image(foutput) + return output + +def resize_image(img, h, w, **up_kwargs): + return F.interpolate(img, (h, w), **up_kwargs) + +def pad_image(img, mean, std, crop_size): + b,c,h,w = img.shape #.size() + assert(c==3) + padh = crop_size - h if h < crop_size else 0 + padw = crop_size - w if w < crop_size else 0 + pad_values = -np.array(mean) / np.array(std) + img_pad = img.new().resize_(b,c,h+padh,w+padw) + for i in range(c): + # note that pytorch pad params is in reversed orders + img_pad[:,i,:,:] = F.pad(img[:,i,:,:], (0, padw, 0, padh), value=pad_values[i]) + assert(img_pad.size(2)>=crop_size and img_pad.size(3)>=crop_size) + return img_pad + +def crop_image(img, h0, h1, w0, w1): + return img[:,:,h0:h1,w0:w1] + +def flip_image(img): + assert(img.dim()==4) + with torch.cuda.device_of(img): + idx = torch.arange(img.size(3)-1, -1, -1).type_as(img).long() + return img.index_select(3, idx) + + +def get_a_var(obj): + if isinstance(obj, torch.Tensor): + return obj + + if isinstance(obj, list) or isinstance(obj, tuple): + for result in map(get_a_var, obj): + if isinstance(result, torch.Tensor): + return result + if isinstance(obj, dict): + for result in map(get_a_var, obj.items()): + if isinstance(result, torch.Tensor): + return result + return None + + +def parallel_apply(modules, inputs, label_set, kwargs_tup=None, devices=None): + r"""Applies each `module` in :attr:`modules` in parallel on arguments + contained in :attr:`inputs` (positional) and :attr:`kwargs_tup` (keyword) + on each of :attr:`devices`. + + Args: + modules (Module): modules to be parallelized + inputs (tensor): inputs to the modules + devices (list of int or torch.device): CUDA devices + + :attr:`modules`, :attr:`inputs`, :attr:`kwargs_tup` (if given), and + :attr:`devices` (if given) should all have same length. Moreover, each + element of :attr:`inputs` can either be a single object as the only argument + to a module, or a collection of positional arguments. + """ + assert len(modules) == len(inputs) + if kwargs_tup is not None: + assert len(modules) == len(kwargs_tup) + else: + kwargs_tup = ({},) * len(modules) + if devices is not None: + assert len(modules) == len(devices) + else: + devices = [None] * len(modules) + devices = [_get_device_index(x, True) for x in devices] + lock = threading.Lock() + results = {} + grad_enabled, autocast_enabled = torch.is_grad_enabled(), torch.is_autocast_enabled() + + def _worker(i, module, input, label_set, kwargs, device=None): + torch.set_grad_enabled(grad_enabled) + if device is None: + device = get_a_var(input).get_device() + try: + with torch.cuda.device(device), autocast(enabled=autocast_enabled): + # this also avoids accidental slicing of `input` if it is a Tensor + if not isinstance(input, (list, tuple)): + input = (input,) + output = module(*input, label_set, **kwargs) + with lock: + results[i] = output + except Exception: + with lock: + results[i] = ExceptionWrapper( + where="in replica {} on device {}".format(i, device)) + + if len(modules) > 1: + threads = [threading.Thread(target=_worker, + args=(i, module, input, label_set, kwargs, device)) + for i, (module, input, kwargs, device) in + enumerate(zip(modules, inputs, kwargs_tup, devices))] + + for thread in threads: + thread.start() + for thread in threads: + thread.join() + else: + _worker(0, modules[0], inputs[0], label_set, kwargs_tup[0], devices[0]) + + outputs = [] + for i in range(len(inputs)): + output = results[i] + if isinstance(output, ExceptionWrapper): + output.reraise() + outputs.append(output) + return outputs + + diff --git a/submodules/lang_seg/data/__init__.py b/submodules/lang_seg/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f9b08ab83627d05f2c8158aa39117429196233ea --- /dev/null +++ b/submodules/lang_seg/data/__init__.py @@ -0,0 +1,24 @@ +import copy + +import itertools +import functools +import numpy as np +import torch +import torch.utils.data +import torchvision.transforms as torch_transforms +import encoding.datasets as enc_ds + +encoding_datasets = { + x: functools.partial(enc_ds.get_dataset, x) + for x in ["coco", "ade20k", "pascal_voc", "pascal_aug", "pcontext", "citys"] +} + + +def get_dataset(name, **kwargs): + if name in encoding_datasets: + return encoding_datasets[name.lower()](**kwargs) + assert False, f"dataset {name} not found" + + +def get_available_datasets(): + return list(encoding_datasets.keys()) diff --git a/submodules/lang_seg/fewshot_data/README.md b/submodules/lang_seg/fewshot_data/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b52f7d9ee23c0bce5bce9289ae51b33706db09f2 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/README.md @@ -0,0 +1,197 @@ +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/hypercorrelation-squeeze-for-few-shot/few-shot-semantic-segmentation-on-pascal-5i-1)](https://paperswithcode.com/sota/few-shot-semantic-segmentation-on-pascal-5i-1?p=hypercorrelation-squeeze-for-few-shot) +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/hypercorrelation-squeeze-for-few-shot/few-shot-semantic-segmentation-on-pascal-5i-5)](https://paperswithcode.com/sota/few-shot-semantic-segmentation-on-pascal-5i-5?p=hypercorrelation-squeeze-for-few-shot) +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/hypercorrelation-squeeze-for-few-shot/few-shot-semantic-segmentation-on-pascal-5i)](https://paperswithcode.com/sota/few-shot-semantic-segmentation-on-pascal-5i?p=hypercorrelation-squeeze-for-few-shot) +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/hypercorrelation-squeeze-for-few-shot/few-shot-semantic-segmentation-on-coco-20i-1)](https://paperswithcode.com/sota/few-shot-semantic-segmentation-on-coco-20i-1?p=hypercorrelation-squeeze-for-few-shot) +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/hypercorrelation-squeeze-for-few-shot/few-shot-semantic-segmentation-on-coco-20i-5)](https://paperswithcode.com/sota/few-shot-semantic-segmentation-on-coco-20i-5?p=hypercorrelation-squeeze-for-few-shot) +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/hypercorrelation-squeeze-for-few-shot/few-shot-semantic-segmentation-on-coco-20i-10)](https://paperswithcode.com/sota/few-shot-semantic-segmentation-on-coco-20i-10?p=hypercorrelation-squeeze-for-few-shot) +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/hypercorrelation-squeeze-for-few-shot/few-shot-semantic-segmentation-on-fss-1000-1)](https://paperswithcode.com/sota/few-shot-semantic-segmentation-on-fss-1000-1?p=hypercorrelation-squeeze-for-few-shot) +[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/hypercorrelation-squeeze-for-few-shot/few-shot-semantic-segmentation-on-fss-1000-5)](https://paperswithcode.com/sota/few-shot-semantic-segmentation-on-fss-1000-5?p=hypercorrelation-squeeze-for-few-shot) + + +## Hypercorrelation Squeeze for Few-Shot Segmentation +This is the implementation of the paper "Hypercorrelation Squeeze for Few-Shot Segmentation" by Juhong Min, Dahyun Kang, and Minsu Cho. Implemented on Python 3.7 and Pytorch 1.5.1. + +

+ +

+ +For more information, check out project [[website](http://cvlab.postech.ac.kr/research/HSNet/)] and the paper on [[arXiv](https://arxiv.org/abs/2104.01538)]. + +## Requirements + +- Python 3.7 +- PyTorch 1.5.1 +- cuda 10.1 +- tensorboard 1.14 + +Conda environment settings: +```bash +conda create -n hsnet python=3.7 +conda activate hsnet + +conda install pytorch=1.5.1 torchvision cudatoolkit=10.1 -c pytorch +conda install -c conda-forge tensorflow +pip install tensorboardX +``` +## Preparing Few-Shot Segmentation Datasets +Download following datasets: + +> #### 1. PASCAL-5i +> Download PASCAL VOC2012 devkit (train/val data): +> ```bash +> wget http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar +> ``` +> Download PASCAL VOC2012 SDS extended mask annotations from our [[Google Drive](https://drive.google.com/file/d/10zxG2VExoEZUeyQl_uXga2OWHjGeZaf2/view?usp=sharing)]. + +> #### 2. COCO-20i +> Download COCO2014 train/val images and annotations: +> ```bash +> wget http://images.cocodataset.org/zips/train2014.zip +> wget http://images.cocodataset.org/zips/val2014.zip +> wget http://images.cocodataset.org/annotations/annotations_trainval2014.zip +> ``` +> Download COCO2014 train/val annotations from our Google Drive: [[train2014.zip](https://drive.google.com/file/d/1cwup51kcr4m7v9jO14ArpxKMA4O3-Uge/view?usp=sharing)], [[val2014.zip](https://drive.google.com/file/d/1PNw4U3T2MhzAEBWGGgceXvYU3cZ7mJL1/view?usp=sharing)]. +> (and locate both train2014/ and val2014/ under annotations/ directory). + +> #### 3. FSS-1000 +> Download FSS-1000 images and annotations from our [[Google Drive](https://drive.google.com/file/d/1Fn-cUESMMF1pQy8Xff-vPQvXJdZoUlP3/view?usp=sharing)]. + +Create a directory '../Datasets_HSN' for the above three few-shot segmentation datasets and appropriately place each dataset to have following directory structure: + + ../ # parent directory + ├── ./ # current (project) directory + │ ├── common/ # (dir.) helper functions + │ ├── data/ # (dir.) dataloaders and splits for each FSSS dataset + │ ├── model/ # (dir.) implementation of Hypercorrelation Squeeze Network model + │ ├── README.md # intstruction for reproduction + │ ├── train.py # code for training HSNet + │ └── test.py # code for testing HSNet + └── Datasets_HSN/ + ├── VOC2012/ # PASCAL VOC2012 devkit + │ ├── Annotations/ + │ ├── ImageSets/ + │ ├── ... + │ └── SegmentationClassAug/ + ├── COCO2014/ + │ ├── annotations/ + │ │ ├── train2014/ # (dir.) training masks (from Google Drive) + │ │ ├── val2014/ # (dir.) validation masks (from Google Drive) + │ │ └── ..some json files.. + │ ├── train2014/ + │ └── val2014/ + └── FSS-1000/ # (dir.) contains 1000 object classes + ├── abacus/ + ├── ... + └── zucchini/ + +## Training +> ### 1. PASCAL-5i +> ```bash +> python train.py --backbone {vgg16, resnet50, resnet101} +> --fold {0, 1, 2, 3} +> --benchmark pascal +> --lr 1e-3 +> --bsz 20 +> --logpath "your_experiment_name" +> ``` +> * Training takes approx. 2 days until convergence (trained with four 2080 Ti GPUs). + + +> ### 2. COCO-20i +> ```bash +> python train.py --backbone {resnet50, resnet101} +> --fold {0, 1, 2, 3} +> --benchmark coco +> --lr 1e-3 +> --bsz 40 +> --logpath "your_experiment_name" +> ``` +> * Training takes approx. 1 week until convergence (trained four Titan RTX GPUs). + +> ### 3. FSS-1000 +> ```bash +> python train.py --backbone {vgg16, resnet50, resnet101} +> --benchmark fss +> --lr 1e-3 +> --bsz 20 +> --logpath "your_experiment_name" +> ``` +> * Training takes approx. 3 days until convergence (trained with four 2080 Ti GPUs). + +> ### Babysitting training: +> Use tensorboard to babysit training progress: +> - For each experiment, a directory that logs training progress will be automatically generated under logs/ directory. +> - From terminal, run 'tensorboard --logdir logs/' to monitor the training progress. +> - Choose the best model when the validation (mIoU) curve starts to saturate. + + + +## Testing + +> ### 1. PASCAL-5i +> Pretrained models with tensorboard logs are available on our [[Google Drive](https://drive.google.com/drive/folders/1z4KgjgOu--k6YuIj3qWrGg264GRcMis2?usp=sharing)]. +> ```bash +> python test.py --backbone {vgg16, resnet50, resnet101} +> --fold {0, 1, 2, 3} +> --benchmark pascal +> --nshot {1, 5} +> --load "path_to_trained_model/best_model.pt" +> ``` + + +> ### 2. COCO-20i +> Pretrained models with tensorboard logs are available on our [[Google Drive](https://drive.google.com/drive/folders/1WpwmCQzxTWhJD5aLQhsgJASaoxxqmFUk?usp=sharing)]. +> ```bash +> python test.py --backbone {resnet50, resnet101} +> --fold {0, 1, 2, 3} +> --benchmark coco +> --nshot {1, 5} +> --load "path_to_trained_model/best_model.pt" +> ``` + +> ### 3. FSS-1000 +> Pretrained models with tensorboard logs are available on our [[Google Drive](https://drive.google.com/drive/folders/1JOaaJknGwsrSEPoLF3x6_lDiy4XfAe99?usp=sharing)]. +> ```bash +> python test.py --backbone {vgg16, resnet50, resnet101} +> --benchmark fss +> --nshot {1, 5} +> --load "path_to_trained_model/best_model.pt" +> ``` + +> ### 4. Evaluation *without support feature masking* on PASCAL-5i +> * To reproduce the results in Tab.1 of our main paper, **COMMENT OUT line 51 in hsnet.py**: support_feats = self.mask_feature(support_feats, support_mask.clone()) +> +> Pretrained models with tensorboard logs are available on our [[Google Drive](https://drive.google.com/drive/folders/18YWMCePIrza194pZvVMqQBuYqhwBmJwd?usp=sharing)]. +> ```bash +> python test.py --backbone resnet101 +> --fold {0, 1, 2, 3} +> --benchmark pascal +> --nshot {1, 5} +> --load "path_to_trained_model/best_model.pt" +> ``` + + +## Visualization + +* To visualize mask predictions, add command line argument **--visualize**: + (prediction results will be saved under vis/ directory) +```bash + python test.py '...other arguments...' --visualize +``` + +#### Example qualitative results (1-shot): + +

+ +

+ +## BibTeX +If you use this code for your research, please consider citing: +````BibTeX +@InProceedings{min2021hypercorrelation, + title={Hypercorrelation Squeeze for Few-Shot Segmentation}, + author={Juhong Min and Dahyun Kang and Minsu Cho}, + booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)}, + year={2021} +} +```` diff --git a/submodules/lang_seg/fewshot_data/common/evaluation.py b/submodules/lang_seg/fewshot_data/common/evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..2d34ab0180aea4bab37088a2212cf580383cd766 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/common/evaluation.py @@ -0,0 +1,39 @@ +r""" Evaluate mask prediction """ +import torch + + +class Evaluator: + r""" Computes intersection and union between prediction and ground-truth """ + @classmethod + def initialize(cls): + cls.ignore_index = 255 + + @classmethod + def classify_prediction(cls, pred_mask, gt_mask, query_ignore_idx=None): + # gt_mask = batch.get('query_mask') + + # # Apply ignore_index in PASCAL-5i masks (following evaluation scheme in PFE-Net (TPAMI 2020)) + # query_ignore_idx = batch.get('query_ignore_idx') + if query_ignore_idx is not None: + assert torch.logical_and(query_ignore_idx, gt_mask).sum() == 0 + query_ignore_idx *= cls.ignore_index + gt_mask = gt_mask + query_ignore_idx + pred_mask[gt_mask == cls.ignore_index] = cls.ignore_index + + # compute intersection and union of each episode in a batch + area_inter, area_pred, area_gt = [], [], [] + for _pred_mask, _gt_mask in zip(pred_mask, gt_mask): + _inter = _pred_mask[_pred_mask == _gt_mask] + if _inter.size(0) == 0: # as torch.histc returns error if it gets empty tensor (pytorch 1.5.1) + _area_inter = torch.tensor([0, 0], device=_pred_mask.device) + else: + _area_inter = torch.histc(_inter, bins=2, min=0, max=1) + area_inter.append(_area_inter) + area_pred.append(torch.histc(_pred_mask, bins=2, min=0, max=1)) + area_gt.append(torch.histc(_gt_mask, bins=2, min=0, max=1)) + area_inter = torch.stack(area_inter).t() + area_pred = torch.stack(area_pred).t() + area_gt = torch.stack(area_gt).t() + area_union = area_pred + area_gt - area_inter + + return area_inter, area_union diff --git a/submodules/lang_seg/fewshot_data/common/logger.py b/submodules/lang_seg/fewshot_data/common/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..7c0dd422fc17c94b46df8946e657cbc121d61216 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/common/logger.py @@ -0,0 +1,135 @@ +r""" Logging during training/testing """ +import datetime +import logging +import os + +from tensorboardX import SummaryWriter +import torch + + +class AverageMeter: + r""" Stores loss, evaluation results """ + def __init__(self, dataset): + self.benchmark = dataset.benchmark + self.class_ids_interest = dataset.class_ids + self.class_ids_interest = torch.tensor(self.class_ids_interest).cuda() + + if self.benchmark == 'pascal': + self.nclass = 20 + elif self.benchmark == 'coco': + self.nclass = 80 + elif self.benchmark == 'fss': + self.nclass = 1000 + + self.intersection_buf = torch.zeros([2, self.nclass]).float().cuda() + self.union_buf = torch.zeros([2, self.nclass]).float().cuda() + self.ones = torch.ones_like(self.union_buf) + self.loss_buf = [] + + def update(self, inter_b, union_b, class_id, loss): + self.intersection_buf.index_add_(1, class_id, inter_b.float()) + self.union_buf.index_add_(1, class_id, union_b.float()) + if loss is None: + loss = torch.tensor(0.0) + self.loss_buf.append(loss) + + def compute_iou(self): + iou = self.intersection_buf.float() / \ + torch.max(torch.stack([self.union_buf, self.ones]), dim=0)[0] + iou = iou.index_select(1, self.class_ids_interest) + miou = iou[1].mean() * 100 + + fb_iou = (self.intersection_buf.index_select(1, self.class_ids_interest).sum(dim=1) / + self.union_buf.index_select(1, self.class_ids_interest).sum(dim=1)).mean() * 100 + + return miou, fb_iou + + def write_result(self, split, epoch): + iou, fb_iou = self.compute_iou() + + loss_buf = torch.stack(self.loss_buf) + msg = '\n*** %s ' % split + msg += '[@Epoch %02d] ' % epoch + msg += 'Avg L: %6.5f ' % loss_buf.mean() + msg += 'mIoU: %5.2f ' % iou + msg += 'FB-IoU: %5.2f ' % fb_iou + + msg += '***\n' + Logger.info(msg) + + def write_process(self, batch_idx, datalen, epoch, write_batch_idx=20): + if batch_idx % write_batch_idx == 0: + msg = '[Epoch: %02d] ' % epoch if epoch != -1 else '' + msg += '[Batch: %04d/%04d] ' % (batch_idx+1, datalen) + iou, fb_iou = self.compute_iou() + if epoch != -1: + loss_buf = torch.stack(self.loss_buf) + msg += 'L: %6.5f ' % loss_buf[-1] + msg += 'Avg L: %6.5f ' % loss_buf.mean() + msg += 'mIoU: %5.2f | ' % iou + msg += 'FB-IoU: %5.2f' % fb_iou + Logger.info(msg) + return iou, fb_iou + + +class Logger: + r""" Writes evaluation results of training/testing """ + @classmethod + def initialize(cls, args, training): + logtime = datetime.datetime.now().__format__('_%m%d_%H%M%S') + logpath = args.logpath if training else '_TEST_' + args.load.split('/')[-2].split('.')[0] + logtime + if logpath == '': logpath = logtime + + cls.logpath = os.path.join('logs', logpath + '.log') + cls.benchmark = args.benchmark + if not os.path.exists(cls.logpath): + os.makedirs(cls.logpath) + + logging.basicConfig(filemode='w', + filename=os.path.join(cls.logpath, 'log.txt'), + level=logging.INFO, + format='%(message)s', + datefmt='%m-%d %H:%M:%S') + + # Console log config + console = logging.StreamHandler() + console.setLevel(logging.INFO) + formatter = logging.Formatter('%(message)s') + console.setFormatter(formatter) + logging.getLogger('').addHandler(console) + + # Tensorboard writer + cls.tbd_writer = SummaryWriter(os.path.join(cls.logpath, 'tbd/runs')) + + # Log arguments + logging.info('\n:=========== Few-shot Seg. with HSNet ===========') + for arg_key in args.__dict__: + logging.info('| %20s: %-24s' % (arg_key, str(args.__dict__[arg_key]))) + logging.info(':================================================\n') + + @classmethod + def info(cls, msg): + r""" Writes log message to log.txt """ + logging.info(msg) + + @classmethod + def save_model_miou(cls, model, epoch, val_miou): + torch.save(model.state_dict(), os.path.join(cls.logpath, 'best_model.pt')) + cls.info('Model saved @%d w/ val. mIoU: %5.2f.\n' % (epoch, val_miou)) + + @classmethod + def log_params(cls, model): + backbone_param = 0 + learner_param = 0 + for k in model.state_dict().keys(): + n_param = model.state_dict()[k].view(-1).size(0) + if k.split('.')[0] in 'backbone': + if k.split('.')[1] in ['classifier', 'fc']: # as fc layers are not used in HSNet + continue + backbone_param += n_param + else: + learner_param += n_param + Logger.info('Backbone # param.: %d' % backbone_param) + Logger.info('Learnable # param.: %d' % learner_param) + Logger.info('Total # param.: %d' % (backbone_param + learner_param)) + diff --git a/submodules/lang_seg/fewshot_data/common/utils.py b/submodules/lang_seg/fewshot_data/common/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7e3e8138d838b02042daffd1291b61230e5a329b --- /dev/null +++ b/submodules/lang_seg/fewshot_data/common/utils.py @@ -0,0 +1,32 @@ +r""" Helper functions """ +import random + +import torch +import numpy as np + + +def fix_randseed(seed): + r""" Set random seeds for reproducibility """ + if seed is None: + seed = int(random.random() * 1e5) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + + +def mean(x): + return sum(x) / len(x) if len(x) > 0 else 0.0 + + +def to_cuda(batch): + for key, value in batch.items(): + if isinstance(value, torch.Tensor): + batch[key] = value.cuda() + return batch + + +def to_cpu(tensor): + return tensor.detach().clone().cpu() diff --git a/submodules/lang_seg/fewshot_data/common/vis.py b/submodules/lang_seg/fewshot_data/common/vis.py new file mode 100644 index 0000000000000000000000000000000000000000..9f006f88ca0abce3a568376f9acf581bd5d9e692 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/common/vis.py @@ -0,0 +1,108 @@ +r""" Visualize model predictions """ +import os + +from PIL import Image +import numpy as np +import torchvision.transforms as transforms + +from fewshot_data.common import utils + + +class Visualizer: + + @classmethod + def initialize(cls, visualize): + cls.visualize = visualize + if not visualize: + return + + cls.colors = {'red': (255, 50, 50), 'blue': (102, 140, 255)} + for key, value in cls.colors.items(): + cls.colors[key] = tuple([c / 255 for c in cls.colors[key]]) + + # cls.mean_img = [0.485, 0.456, 0.406] + # cls.std_img = [0.229, 0.224, 0.225] + cls.mean_img = [0.5] * 3 + cls.std_img = [0.5] * 3 + cls.to_pil = transforms.ToPILImage() + cls.vis_path = './vis/' + if not os.path.exists(cls.vis_path): os.makedirs(cls.vis_path) + + @classmethod + def visualize_prediction_batch(cls, spt_img_b, spt_mask_b, qry_img_b, qry_mask_b, pred_mask_b, cls_id_b, batch_idx, iou_b=None): + spt_img_b = utils.to_cpu(spt_img_b) + spt_mask_b = utils.to_cpu(spt_mask_b) + qry_img_b = utils.to_cpu(qry_img_b) + qry_mask_b = utils.to_cpu(qry_mask_b) + pred_mask_b = utils.to_cpu(pred_mask_b) + cls_id_b = utils.to_cpu(cls_id_b) + + for sample_idx, (spt_img, spt_mask, qry_img, qry_mask, pred_mask, cls_id) in \ + enumerate(zip(spt_img_b, spt_mask_b, qry_img_b, qry_mask_b, pred_mask_b, cls_id_b)): + iou = iou_b[sample_idx] if iou_b is not None else None + cls.visualize_prediction(spt_img, spt_mask, qry_img, qry_mask, pred_mask, cls_id, batch_idx, sample_idx, True, iou) + + @classmethod + def to_numpy(cls, tensor, type): + if type == 'img': + return np.array(cls.to_pil(cls.unnormalize(tensor))).astype(np.uint8) + elif type == 'mask': + return np.array(tensor).astype(np.uint8) + else: + raise Exception('Undefined tensor type: %s' % type) + + @classmethod + def visualize_prediction(cls, spt_imgs, spt_masks, qry_img, qry_mask, pred_mask, cls_id, batch_idx, sample_idx, label, iou=None): + + spt_color = cls.colors['blue'] + qry_color = cls.colors['red'] + pred_color = cls.colors['red'] + + spt_imgs = [cls.to_numpy(spt_img, 'img') for spt_img in spt_imgs] + spt_pils = [cls.to_pil(spt_img) for spt_img in spt_imgs] + spt_masks = [cls.to_numpy(spt_mask, 'mask') for spt_mask in spt_masks] + spt_masked_pils = [Image.fromarray(cls.apply_mask(spt_img, spt_mask, spt_color)) for spt_img, spt_mask in zip(spt_imgs, spt_masks)] + + qry_img = cls.to_numpy(qry_img, 'img') + qry_pil = cls.to_pil(qry_img) + qry_mask = cls.to_numpy(qry_mask, 'mask') + pred_mask = cls.to_numpy(pred_mask, 'mask') + pred_masked_pil = Image.fromarray(cls.apply_mask(qry_img.astype(np.uint8), pred_mask.astype(np.uint8), pred_color)) + qry_masked_pil = Image.fromarray(cls.apply_mask(qry_img.astype(np.uint8), qry_mask.astype(np.uint8), qry_color)) + + merged_pil = cls.merge_image_pair(spt_masked_pils + [pred_masked_pil, qry_masked_pil]) + + iou = iou.item() if iou else 0.0 + merged_pil.save(cls.vis_path + '%d_%d_class-%d_iou-%.2f' % (batch_idx, sample_idx, cls_id, iou) + '.jpg') + + @classmethod + def merge_image_pair(cls, pil_imgs): + r""" Horizontally aligns a pair of pytorch tensor images (3, H, W) and returns PIL object """ + + canvas_width = sum([pil.size[0] for pil in pil_imgs]) + canvas_height = max([pil.size[1] for pil in pil_imgs]) + canvas = Image.new('RGB', (canvas_width, canvas_height)) + + xpos = 0 + for pil in pil_imgs: + canvas.paste(pil, (xpos, 0)) + xpos += pil.size[0] + + return canvas + + @classmethod + def apply_mask(cls, image, mask, color, alpha=0.5): + r""" Apply mask to the given image. """ + for c in range(3): + image[:, :, c] = np.where(mask == 1, + image[:, :, c] * + (1 - alpha) + alpha * color[c] * 255, + image[:, :, c]) + return image + + @classmethod + def unnormalize(cls, img): + img = img.clone() + for im_channel, mean, std in zip(img, cls.mean_img, cls.std_img): + im_channel.mul_(std).add_(mean) + return img diff --git a/submodules/lang_seg/fewshot_data/data/coco.py b/submodules/lang_seg/fewshot_data/data/coco.py new file mode 100644 index 0000000000000000000000000000000000000000..d8df47e5a0a778d80fc73f29fa66ac7acc8ac349 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/coco.py @@ -0,0 +1,115 @@ +r""" COCO-20i few-shot semantic segmentation dataset """ +import os +import pickle + +from torch.utils.data import Dataset +import torch.nn.functional as F +import torch +import PIL.Image as Image +import numpy as np + + +class DatasetCOCO(Dataset): + def __init__(self, datapath, fold, transform, split, shot, use_original_imgsize): + self.split = 'val' if split in ['val', 'test'] else 'trn' + self.fold = fold + self.nfolds = 4 + self.nclass = 80 + self.benchmark = 'coco' + self.shot = shot + self.split_coco = split if split == 'val2014' else 'train2014' + self.base_path = os.path.join(datapath, 'COCO2014') + self.transform = transform + self.use_original_imgsize = use_original_imgsize + + self.class_ids = self.build_class_ids() + self.img_metadata_classwise = self.build_img_metadata_classwise() + self.img_metadata = self.build_img_metadata() + + def __len__(self): + return len(self.img_metadata) if self.split == 'trn' else 1000 + + def __getitem__(self, idx): + # ignores idx during training & testing and perform uniform sampling over object classes to form an episode + # (due to the large size of the COCO dataset) + query_img, query_mask, support_imgs, support_masks, query_name, support_names, class_sample, org_qry_imsize = self.load_frame() + + query_img = self.transform(query_img) + query_mask = query_mask.float() + if not self.use_original_imgsize: + query_mask = F.interpolate(query_mask.unsqueeze(0).unsqueeze(0).float(), query_img.size()[-2:], mode='nearest').squeeze() + + if self.shot: + support_imgs = torch.stack([self.transform(support_img) for support_img in support_imgs]) + for midx, smask in enumerate(support_masks): + support_masks[midx] = F.interpolate(smask.unsqueeze(0).unsqueeze(0).float(), support_imgs.size()[-2:], mode='nearest').squeeze() + support_masks = torch.stack(support_masks) + + + batch = {'query_img': query_img, + 'query_mask': query_mask, + 'query_name': query_name, + + 'org_query_imsize': org_qry_imsize, + + 'support_imgs': support_imgs, + 'support_masks': support_masks, + 'support_names': support_names, + 'class_id': torch.tensor(class_sample)} + + return batch + + def build_class_ids(self): + nclass_trn = self.nclass // self.nfolds + class_ids_val = [self.fold + self.nfolds * v for v in range(nclass_trn)] + class_ids_trn = [x for x in range(self.nclass) if x not in class_ids_val] + class_ids = class_ids_trn if self.split == 'trn' else class_ids_val + + return class_ids + + def build_img_metadata_classwise(self): + with open('fewshot_data/data/splits/coco/%s/fold%d.pkl' % (self.split, self.fold), 'rb') as f: + img_metadata_classwise = pickle.load(f) + return img_metadata_classwise + + def build_img_metadata(self): + img_metadata = [] + for k in self.img_metadata_classwise.keys(): + img_metadata += self.img_metadata_classwise[k] + return sorted(list(set(img_metadata))) + + def read_mask(self, name): + mask_path = os.path.join(self.base_path, 'annotations', name) + mask = torch.tensor(np.array(Image.open(mask_path[:mask_path.index('.jpg')] + '.png'))) + return mask + + def load_frame(self): + class_sample = np.random.choice(self.class_ids, 1, replace=False)[0] + query_name = np.random.choice(self.img_metadata_classwise[class_sample], 1, replace=False)[0] + query_img = Image.open(os.path.join(self.base_path, query_name)).convert('RGB') + query_mask = self.read_mask(query_name) + + org_qry_imsize = query_img.size + + query_mask[query_mask != class_sample + 1] = 0 + query_mask[query_mask == class_sample + 1] = 1 + + support_names = [] + if self.shot: + while True: # keep sampling support set if query == support + support_name = np.random.choice(self.img_metadata_classwise[class_sample], 1, replace=False)[0] + if query_name != support_name: support_names.append(support_name) + if len(support_names) == self.shot: break + + support_imgs = [] + support_masks = [] + if self.shot: + for support_name in support_names: + support_imgs.append(Image.open(os.path.join(self.base_path, support_name)).convert('RGB')) + support_mask = self.read_mask(support_name) + support_mask[support_mask != class_sample + 1] = 0 + support_mask[support_mask == class_sample + 1] = 1 + support_masks.append(support_mask) + + return query_img, query_mask, support_imgs, support_masks, query_name, support_names, class_sample, org_qry_imsize + diff --git a/submodules/lang_seg/fewshot_data/data/dataset.py b/submodules/lang_seg/fewshot_data/data/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..3560a5218eb957ce674e62440f6049f864ee1091 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/dataset.py @@ -0,0 +1,42 @@ +r""" Dataloader builder for few-shot semantic segmentation dataset """ +from torchvision import transforms +from torch.utils.data import DataLoader + +from fewshot_data.data.pascal import DatasetPASCAL +from fewshot_data.data.coco import DatasetCOCO +from fewshot_data.data.fss import DatasetFSS + + +class FSSDataset: + @classmethod + def initialize(cls, img_size, datapath, use_original_imgsize, imagenet_norm=False): + cls.datasets = { + 'pascal': DatasetPASCAL, + 'coco': DatasetCOCO, + 'fss': DatasetFSS, + } + + if imagenet_norm: + cls.img_mean = [0.485, 0.456, 0.406] + cls.img_std = [0.229, 0.224, 0.225] + print('use norm: {}, {}'.format(cls.img_mean, cls.img_std)) + else: + cls.img_mean = [0.5] * 3 + cls.img_std = [0.5] * 3 + print('use norm: {}, {}'.format(cls.img_mean, cls.img_std)) + + cls.datapath = datapath + cls.use_original_imgsize = use_original_imgsize + + cls.transform = transforms.Compose([transforms.Resize(size=(img_size, img_size)), + transforms.ToTensor(), + transforms.Normalize(cls.img_mean, cls.img_std)]) + + @classmethod + def build_dataloader(cls, benchmark, bsz, nworker, fold, split, shot=1): + shuffle = split == 'trn' + nworker = nworker if split == 'trn' else 0 + dataset = cls.datasets[benchmark](cls.datapath, fold=fold, transform=cls.transform, split=split, shot=shot, use_original_imgsize=cls.use_original_imgsize) + dataloader = DataLoader(dataset, batch_size=bsz, shuffle=shuffle, num_workers=nworker) + + return dataloader diff --git a/submodules/lang_seg/fewshot_data/data/fss.py b/submodules/lang_seg/fewshot_data/data/fss.py new file mode 100644 index 0000000000000000000000000000000000000000..4edbd72791553c8f0c7a68d88f43012ca4320170 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/fss.py @@ -0,0 +1,140 @@ +r""" FSS-1000 few-shot semantic segmentation dataset """ +import os +import glob + +from torch.utils.data import Dataset +import torch.nn.functional as F +import torch +import PIL.Image as Image +import numpy as np + + +class DatasetFSS(Dataset): + def __init__(self, datapath, fold, transform, split, shot, use_original_imgsize=None): + self.split = split + self.benchmark = 'fss' + self.shot = shot + + self.base_path = os.path.join(datapath, 'FSS-1000') + + # Given predefined test split, load randomly generated training/val splits: + # (reference regarding trn/val/test splits: https://github.com/HKUSTCV/FSS-1000/issues/7)) + with open('fewshot_data/data/splits/fss/%s.txt' % split, 'r') as f: + self.categories = f.read().split('\n')[:-1] + self.categories = sorted(self.categories) + + self.class_ids = self.build_class_ids() + self.img_metadata = self.build_img_metadata() + + self.transform = transform + + def __len__(self): + return len(self.img_metadata) + + def __getitem__(self, idx): + query_name, support_names, class_sample = self.sample_episode(idx) + query_img, query_mask, support_imgs, support_masks = self.load_frame(query_name, support_names) + + query_img = self.transform(query_img) + query_mask = F.interpolate(query_mask.unsqueeze(0).unsqueeze(0).float(), query_img.size()[-2:], mode='nearest').squeeze() + if self.shot: + support_imgs = torch.stack([self.transform(support_img) for support_img in support_imgs]) + + support_masks_tmp = [] + for smask in support_masks: + smask = F.interpolate(smask.unsqueeze(0).unsqueeze(0).float(), support_imgs.size()[-2:], mode='nearest').squeeze() + support_masks_tmp.append(smask) + support_masks = torch.stack(support_masks_tmp) + + batch = {'query_img': query_img, + 'query_mask': query_mask, + 'query_name': query_name, + + 'support_imgs': support_imgs, + 'support_masks': support_masks, + 'support_names': support_names, + + 'class_id': torch.tensor(class_sample)} + + return batch + + def load_frame(self, query_name, support_names): + query_img = Image.open(query_name).convert('RGB') + if self.shot: + support_imgs = [Image.open(name).convert('RGB') for name in support_names] + else: + support_imgs = [] + + query_id = query_name.split('/')[-1].split('.')[0] + query_name = os.path.join(os.path.dirname(query_name), query_id) + '.png' + + if self.shot: + support_ids = [name.split('/')[-1].split('.')[0] for name in support_names] + support_names = [os.path.join(os.path.dirname(name), sid) + '.png' for name, sid in zip(support_names, support_ids)] + + query_mask = self.read_mask(query_name) + if self.shot: + support_masks = [self.read_mask(name) for name in support_names] + else: + support_masks = [] + + return query_img, query_mask, support_imgs, support_masks + + def read_mask(self, img_name): + mask = torch.tensor(np.array(Image.open(img_name).convert('L'))) + mask[mask < 128] = 0 + mask[mask >= 128] = 1 + return mask + + def sample_episode(self, idx): + query_name = self.img_metadata[idx] + class_sample = self.categories.index(query_name.split('/')[-2]) + if self.split == 'val': + class_sample += 520 + elif self.split == 'test': + class_sample += 760 + + support_names = [] + # here we only test with shot=1 + if self.split == 'test' and self.shot == 1: + while True: + support_name = 1 + support_name = os.path.join(os.path.dirname(query_name), str(support_name)) + '.jpg' + if query_name != support_name: + support_names.append(support_name) + else: + print('Error in sample_episode!') + exit() + if len(support_names) == self.shot: break + elif self.shot: + while True: # keep sampling support set if query == support + support_name = np.random.choice(range(1, 11), 1, replace=False)[0] + support_name = os.path.join(os.path.dirname(query_name), str(support_name)) + '.jpg' + if query_name != support_name: support_names.append(support_name) + if len(support_names) == self.shot: break + + return query_name, support_names, class_sample + + def build_class_ids(self): + if self.split == 'trn': + class_ids = range(0, 520) + elif self.split == 'val': + class_ids = range(520, 760) + elif self.split == 'test': + class_ids = range(760, 1000) + return class_ids + + def build_img_metadata(self): + img_metadata = [] + for cat in self.categories: + img_paths = sorted([path for path in glob.glob('%s/*' % os.path.join(self.base_path, cat))]) + if self.split == 'test' and self.shot == 1: + for i in range(1, len(img_paths)): + img_path = img_paths[i] + if os.path.basename(img_path).split('.')[1] == 'jpg': + img_metadata.append(img_path) + else: + for img_path in img_paths: + if os.path.basename(img_path).split('.')[1] == 'jpg': + img_metadata.append(img_path) + return img_metadata diff --git a/submodules/lang_seg/fewshot_data/data/pascal.py b/submodules/lang_seg/fewshot_data/data/pascal.py new file mode 100644 index 0000000000000000000000000000000000000000..b92fe5caee0deb9853a1c46c7d1a9733ce8e8cea --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/pascal.py @@ -0,0 +1,152 @@ +r""" PASCAL-5i few-shot semantic segmentation dataset """ +import os + +from torch.utils.data import Dataset +import torch.nn.functional as F +import torch +import PIL.Image as Image +import numpy as np + + +class DatasetPASCAL(Dataset): + def __init__(self, datapath, fold, transform, split, shot, use_original_imgsize): + self.split = 'val' if split in ['val', 'test'] else 'trn' + self.fold = fold + self.nfolds = 4 + self.nclass = 20 + self.benchmark = 'pascal' + self.shot = shot + self.use_original_imgsize = use_original_imgsize + + self.img_path = os.path.join(datapath, 'VOC2012/JPEGImages/') + self.ann_path = os.path.join(datapath, 'VOC2012/SegmentationClassAug/') + self.transform = transform + + self.class_ids = self.build_class_ids() + self.img_metadata = self.build_img_metadata() + self.img_metadata_classwise = self.build_img_metadata_classwise() + + def __len__(self): + return len(self.img_metadata) if self.split == 'trn' else 1000 + + def __getitem__(self, idx): + idx %= len(self.img_metadata) # for testing, as n_images < 1000 + query_name, support_names, class_sample = self.sample_episode(idx) + query_img, query_cmask, support_imgs, support_cmasks, org_qry_imsize = self.load_frame(query_name, support_names) + + query_img = self.transform(query_img) + if not self.use_original_imgsize: + query_cmask = F.interpolate(query_cmask.unsqueeze(0).unsqueeze(0).float(), query_img.size()[-2:], mode='nearest').squeeze() + query_mask, query_ignore_idx = self.extract_ignore_idx(query_cmask.float(), class_sample) + + if self.shot: + support_imgs = torch.stack([self.transform(support_img) for support_img in support_imgs]) + + support_masks = [] + support_ignore_idxs = [] + for scmask in support_cmasks: + scmask = F.interpolate(scmask.unsqueeze(0).unsqueeze(0).float(), support_imgs.size()[-2:], mode='nearest').squeeze() + support_mask, support_ignore_idx = self.extract_ignore_idx(scmask, class_sample) + support_masks.append(support_mask) + support_ignore_idxs.append(support_ignore_idx) + support_masks = torch.stack(support_masks) + support_ignore_idxs = torch.stack(support_ignore_idxs) + else: + support_masks = [] + support_ignore_idxs = [] + batch = {'query_img': query_img, + 'query_mask': query_mask, + 'query_name': query_name, + 'query_ignore_idx': query_ignore_idx, + + 'org_query_imsize': org_qry_imsize, + + 'support_imgs': support_imgs, + 'support_masks': support_masks, + 'support_names': support_names, + 'support_ignore_idxs': support_ignore_idxs, + + 'class_id': torch.tensor(class_sample)} + + return batch + + def extract_ignore_idx(self, mask, class_id): + boundary = (mask / 255).floor() + mask[mask != class_id + 1] = 0 + mask[mask == class_id + 1] = 1 + + return mask, boundary + + def load_frame(self, query_name, support_names): + query_img = self.read_img(query_name) + query_mask = self.read_mask(query_name) + support_imgs = [self.read_img(name) for name in support_names] + support_masks = [self.read_mask(name) for name in support_names] + + org_qry_imsize = query_img.size + + return query_img, query_mask, support_imgs, support_masks, org_qry_imsize + + def read_mask(self, img_name): + r"""Return segmentation mask in PIL Image""" + mask = torch.tensor(np.array(Image.open(os.path.join(self.ann_path, img_name) + '.png'))) + return mask + + def read_img(self, img_name): + r"""Return RGB image in PIL Image""" + return Image.open(os.path.join(self.img_path, img_name) + '.jpg') + + def sample_episode(self, idx): + query_name, class_sample = self.img_metadata[idx] + + support_names = [] + if self.shot: + while True: # keep sampling support set if query == support + support_name = np.random.choice(self.img_metadata_classwise[class_sample], 1, replace=False)[0] + if query_name != support_name: support_names.append(support_name) + if len(support_names) == self.shot: break + + return query_name, support_names, class_sample + + def build_class_ids(self): + nclass_trn = self.nclass // self.nfolds + class_ids_val = [self.fold * nclass_trn + i for i in range(nclass_trn)] + class_ids_trn = [x for x in range(self.nclass) if x not in class_ids_val] + + if self.split == 'trn': + return class_ids_trn + else: + return class_ids_val + + def build_img_metadata(self): + + def read_metadata(split, fold_id): + fold_n_metadata = os.path.join('fewshot_data/data/splits/pascal/%s/fold%d.txt' % (split, fold_id)) + with open(fold_n_metadata, 'r') as f: + fold_n_metadata = f.read().split('\n')[:-1] + fold_n_metadata = [[data.split('__')[0], int(data.split('__')[1]) - 1] for data in fold_n_metadata] + return fold_n_metadata + + img_metadata = [] + if self.split == 'trn': # For training, read image-metadata of "the other" folds + for fold_id in range(self.nfolds): + if fold_id == self.fold: # Skip validation fold + continue + img_metadata += read_metadata(self.split, fold_id) + elif self.split == 'val': # For validation, read image-metadata of "current" fold + img_metadata = read_metadata(self.split, self.fold) + else: + raise Exception('Undefined split %s: ' % self.split) + + print('Total (%s) images are : %d' % (self.split, len(img_metadata))) + + return img_metadata + + def build_img_metadata_classwise(self): + img_metadata_classwise = {} + for class_id in range(self.nclass): + img_metadata_classwise[class_id] = [] + + for img_name, img_class in self.img_metadata: + img_metadata_classwise[img_class] += [img_name] + return img_metadata_classwise diff --git a/submodules/lang_seg/fewshot_data/data/splits/coco/trn/fold0.pkl b/submodules/lang_seg/fewshot_data/data/splits/coco/trn/fold0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..46dd52ede143a92af9e90887fd1820125f5511fc --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/coco/trn/fold0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a94e113d8c150a7e20723360dd9779eaa1305ccddc28dff4501e29ed3df5c45 +size 3330536 diff --git a/submodules/lang_seg/fewshot_data/data/splits/coco/trn/fold1.pkl b/submodules/lang_seg/fewshot_data/data/splits/coco/trn/fold1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..d8af5a688a48aebdfd60255a919cabcf3edbeb89 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/coco/trn/fold1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69629cdd84fd57b3bb4e281b5bf65440f7ecf9b62eb119f1d8d7fe78b0fb02a8 +size 4090562 diff --git a/submodules/lang_seg/fewshot_data/data/splits/coco/trn/fold2.pkl b/submodules/lang_seg/fewshot_data/data/splits/coco/trn/fold2.pkl new file mode 100644 index 0000000000000000000000000000000000000000..651346e26f173a917d164dcad2d5b246be40c89d --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/coco/trn/fold2.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:594adc4b8ebecf9b8b559284f334ff3dd3fa3c150317cbc6ce7917ce81a08c03 +size 4064175 diff --git a/submodules/lang_seg/fewshot_data/data/splits/coco/trn/fold3.pkl b/submodules/lang_seg/fewshot_data/data/splits/coco/trn/fold3.pkl new file mode 100644 index 0000000000000000000000000000000000000000..d8ed7dfe4fb8a433c484b1f620cd9c7f3727ca62 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/coco/trn/fold3.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07b38ccc9e15ebfd1ab6288eeaad3a5f289bbba3419807212169fd44b89cc5c5 +size 4099479 diff --git a/submodules/lang_seg/fewshot_data/data/splits/coco/val/fold0.pkl b/submodules/lang_seg/fewshot_data/data/splits/coco/val/fold0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..8eb78fdb37f43246aa8482a204357cbf96ac2a25 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/coco/val/fold0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96002389be83b4730d13b814398f54eb2baf5c3283598ec2288d60624d6ef9c3 +size 1272708 diff --git a/submodules/lang_seg/fewshot_data/data/splits/coco/val/fold1.pkl b/submodules/lang_seg/fewshot_data/data/splits/coco/val/fold1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..03e7fe4dfeeab90d12af8e4a297effd474a58351 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/coco/val/fold1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a34deecdf2858fc436f5ea4d5a2d46b6674ce09f7ebce41c34b694116381fa56 +size 732057 diff --git a/submodules/lang_seg/fewshot_data/data/splits/coco/val/fold2.pkl b/submodules/lang_seg/fewshot_data/data/splits/coco/val/fold2.pkl new file mode 100644 index 0000000000000000000000000000000000000000..99c3658ee0bb4dd988b0b3bd8b6fd3c8d1cebf3c --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/coco/val/fold2.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f275c5e9bf538f6ef8a95054f6daf2519b53785c4ab7fd07e633dda57dac1e5 +size 624406 diff --git a/submodules/lang_seg/fewshot_data/data/splits/coco/val/fold3.pkl b/submodules/lang_seg/fewshot_data/data/splits/coco/val/fold3.pkl new file mode 100644 index 0000000000000000000000000000000000000000..9c8b6b688868d229ad11fb2f59999d1413d2d54c --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/coco/val/fold3.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:927597bd78036a460abdab243330667af6ceffdb483bc3331ae55d148f00c449 +size 602257 diff --git a/submodules/lang_seg/fewshot_data/data/splits/fss/test.txt b/submodules/lang_seg/fewshot_data/data/splits/fss/test.txt new file mode 100644 index 0000000000000000000000000000000000000000..32d0bd083facea98ee28c04383c7511da5d21c9d --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/fss/test.txt @@ -0,0 +1,240 @@ +bus +hotel_slipper +burj_al +reflex_camera +abe's_flyingfish +oiltank_car +doormat +fish_eagle +barber_shaver +motorbike +feather_clothes +wandering_albatross +rice_cooker +delta_wing +fish +nintendo_switch +bustard +diver +minicooper +cathedrale_paris +big_ben +combination_lock +villa_savoye +american_alligator +gym_ball +andean_condor +leggings +pyramid_cube +jet_aircraft +meatloaf +reel +swan +osprey +crt_screen +microscope +rubber_eraser +arrow +monkey +mitten +spiderman +parthenon +bat +chess_king +sulphur_butterfly +quail_egg +oriole +iron_man +wooden_boat +anise +steering_wheel +groenendael +dwarf_beans +pteropus +chalk_brush +bloodhound +moon +english_foxhound +boxing_gloves +peregine_falcon +pyraminx +cicada +screw +shower_curtain +tredmill +bulb +bell_pepper +lemur_catta +doughnut +twin_tower +astronaut +nintendo_3ds +fennel_bulb +indri +captain_america_shield +kunai +broom +iphone +earphone1 +flying_squirrel +onion +vinyl +sydney_opera_house +oyster +harmonica +egg +breast_pump +guitar +potato_chips +tunnel +cuckoo +rubick_cube +plastic_bag +phonograph +net_surface_shoes +goldfinch +ipad +mite_predator +coffee_mug +golden_plover +f1_racing +lapwing +nintendo_gba +pizza +rally_car +drilling_platform +cd +fly +magpie_bird +leaf_fan +little_blue_heron +carriage +moist_proof_pad +flying_snakes +dart_target +warehouse_tray +nintendo_wiiu +chiffon_cake +bath_ball +manatee +cloud +marimba +eagle +ruler +soymilk_machine +sled +seagull +glider_flyingfish +doublebus +transport_helicopter +window_screen +truss_bridge +wasp +snowman +poached_egg +strawberry +spinach +earphone2 +downy_pitch +taj_mahal +rocking_chair +cablestayed_bridge +sealion +banana_boat +pheasant +stone_lion +electronic_stove +fox +iguana +rugby_ball +hang_glider +water_buffalo +lotus +paper_plane +missile +flamingo +american_chamelon +kart +chinese_knot +cabbage_butterfly +key +church +tiltrotor +helicopter +french_fries +water_heater +snow_leopard +goblet +fan +snowplow +leafhopper +pspgo +black_bear +quail +condor +chandelier +hair_razor +white_wolf +toaster +pidan +pyramid +chicken_leg +letter_opener +apple_icon +porcupine +chicken +stingray +warplane +windmill +bamboo_slip +wig +flying_geckos +stonechat +haddock +australian_terrier +hover_board +siamang +canton_tower +santa_sledge +arch_bridge +curlew +sushi +beet_root +accordion +leaf_egg +stealth_aircraft +stork +bucket +hawk +chess_queen +ocarina +knife +whippet +cantilever_bridge +may_bug +wagtail +leather_shoes +wheelchair +shumai +speedboat +vacuum_cup +chess_knight +pumpkin_pie +wooden_spoon +bamboo_dragonfly +ganeva_chair +soap +clearwing_flyingfish +pencil_sharpener1 +cricket +photocopier +nintendo_sp +samarra_mosque +clam +charge_battery +flying_frog +ferrari911 +polo_shirt +echidna +coin +tower_pisa diff --git a/submodules/lang_seg/fewshot_data/data/splits/fss/trn.txt b/submodules/lang_seg/fewshot_data/data/splits/fss/trn.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e131276d38efc8543accc6e3048bd313294643e --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/fss/trn.txt @@ -0,0 +1,520 @@ +fountain +taxi +assult_rifle +radio +comb +box_turtle +igloo +head_cabbage +cottontail +coho +ashtray +joystick +sleeping_bag +jackfruit +trailer_truck +shower_cap +ibex +kinguin +squirrel +ac_wall +sidewinder +remote_control +marshmallow +bolotie +polar_bear +rock_beauty +tokyo_tower +wafer +red_bayberry +electronic_toothbrush +hartebeest +cassette +oil_filter +bomb +walnut +toilet_tissue +memory_stick +wild_boar +cableways +chihuahua +envelope +bison +poker +pubg_lvl3helmet +indian_cobra +staffordshire +park_bench +wombat +black_grouse +submarine +washer +agama +coyote +feeder +sarong +buckingham_palace +frog +steam_locomotive +acorn +german_pointer +obelisk +polecat +black_swan +butterfly +mountain_tent +gorilla +sloth_bear +aubergine +stinkhorn +stole +owl +mooli +pool_table +collar +lhasa_apso +ambulance +spade +pufferfish +paint_brush +lark +golf_ball +hock +fork +drake +bee_house +mooncake +wok +cocacola +water_bike +ladder +psp +bassoon +bear +border_terrier +petri_dish +pill_bottle +aircraft_carrier +panther +canoe +baseball_player +turtle +espresso +throne +cornet +coucal +eletrical_switch +bra +snail +backpack +jacamar +scroll_brush +gliding_lizard +raft +pinwheel +grasshopper +green_mamba +eft_newt +computer_mouse +vine_snake +recreational_vehicle +llama +meerkat +chainsaw +ferret +garbage_can +kangaroo +litchi +carbonara +housefinch +modem +tebby_cat +thatch +face_powder +tomb +apple +ladybug +killer_whale +rocket +airship +surfboard +lesser_panda +jordan_logo +banana +nail_scissor +swab +perfume +punching_bag +victor_icon +waffle_iron +trimaran +garlic +flute +langur +starfish +parallel_bars +dandie_dinmont +cosmetic_brush +screwdriver +brick_card +balance_weight +hornet +carton +toothpaste +bracelet +egg_tart +pencil_sharpener2 +swimming_glasses +howler_monkey +camel +dragonfly +lionfish +convertible +mule +usb +conch +papaya +garbage_truck +dingo +radiator +solar_dish +streetcar +trilobite +bouzouki +ringlet_butterfly +space_shuttle +waffle +american_staffordshire +violin +flowerpot +forklift +manx +sundial +snowmobile +chickadee_bird +ruffed_grouse +brick_tea +paddle +stove +carousel +spatula +beaker +gas_pump +lawn_mower +speaker +tank +tresher +kappa_logo +hare +tennis_racket +shopping_cart +thimble +tractor +anemone_fish +trolleybus +steak +capuchin +red_breasted_merganser +golden_retriever +light_tube +flatworm +melon_seed +digital_watch +jacko_lantern +brown_bear +cairn +mushroom +chalk +skull +stapler +potato +telescope +proboscis +microphone +torii +baseball_bat +dhole +excavator +fig +snake +bradypod +pepitas +prairie_chicken +scorpion +shotgun +bottle_cap +file_cabinet +grey_whale +one-armed_bandit +banded_gecko +flying_disc +croissant +toothbrush +miniskirt +pokermon_ball +gazelle +grey_fox +esport_chair +necklace +ptarmigan +watermelon +besom +pomelo +radio_telescope +studio_couch +black_stork +vestment +koala +brambling +muscle_car +window_shade +space_heater +sunglasses +motor_scooter +ladyfinger +pencil_box +titi_monkey +chicken_wings +mount_fuji +giant_panda +dart +fire_engine +running_shoe +dumbbell +donkey +loafer +hard_disk +globe +lifeboat +medical_kit +brain_coral +paper_towel +dugong +seatbelt +skunk +military_vest +cocktail_shaker +zucchini +quad_drone +ocicat +shih-tzu +teapot +tile_roof +cheese_burger +handshower +red_wolf +stop_sign +mouse +battery +adidas_logo2 +earplug +hummingbird +brush_pen +pistachio +hamster +air_strip +indian_elephant +otter +cucumber +scabbard +hawthorn +bullet_train +leopard +whale +cream +chinese_date +jellyfish +lobster +skua +single_log +chicory +bagel +beacon +pingpong_racket +spoon +yurt +wallaby +egret +christmas_stocking +mcdonald_uncle +wrench +spark_plug +triceratops +wall_clock +jinrikisha +pickup +rhinoceros +swimming_trunk +band-aid +spotted_salamander +leeks +marmot +warthog +cello +stool +chest +toilet_plunger +wardrobe +cannon +adidas_logo1 +drumstick +lady_slipper +puma_logo +great_wall +white_shark +witch_hat +vending_machine +wreck +chopsticks +garfish +african_elephant +children_slide +hornbill +zebra +boa_constrictor +armour +pineapple +angora +brick +car_wheel +wallet +boston_bull +hyena +lynx +crash_helmet +terrapin_turtle +persian_cat +shift_gear +cactus_ball +fur_coat +plate +pen +okra +mario +airedale +cowboy_hat +celery +macaque +candle +goose +raccoon +brasscica +almond +maotai_bottle +soccer_ball +sports_car +tobacco_pipe +water_polo +eggnog +hook +ostrich +patas +table_lamp +teddy +mongoose +spoonbill +redheart +crane +dinosaur +kitchen_knife +seal +baboon +golfcart +roller_coaster +avocado +birdhouse +yorkshire_terrier +saluki +basketball +buckler +harvester +afghan_hound +beam_bridge +guinea_pig +lorikeet +shakuhachi +motarboard +statue_liberty +police_car +sulphur_crested +gourd +sombrero +mailbox +adhensive_tape +night_snake +bushtit +mouthpiece +beaver +bathtub +printer +cumquat +orange +cleaver +quill_pen +panpipe +diamond +gypsy_moth +cauliflower +lampshade +cougar +traffic_light +briefcase +ballpoint +african_grey +kremlin +barometer +peacock +paper_crane +sunscreen +tofu +bedlington_terrier +snowball +carrot +tiger +mink +cristo_redentor +ladle +keyboard +maraca +monitor +water_snake +can_opener +mud_turtle +bald_eagle +carp +cn_tower +egyptian_cat +hen_of_the_woods +measuring_cup +roller_skate +kite +sandwich_cookies +sandwich +persimmon +chess_bishop +coffin +ruddy_turnstone +prayer_rug +rain_barrel +neck_brace +nematode +rosehip +dutch_oven +goldfish +blossom_card +dough +trench_coat +sponge +stupa +wash_basin +electric_fan +spring_scroll +potted_plant +sparrow +car_mirror +gecko +diaper +leatherback_turtle +strainer +guacamole +microwave diff --git a/submodules/lang_seg/fewshot_data/data/splits/fss/val.txt b/submodules/lang_seg/fewshot_data/data/splits/fss/val.txt new file mode 100644 index 0000000000000000000000000000000000000000..30dfb203710764df702e74d2f7e5e8763e92c7fe --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/fss/val.txt @@ -0,0 +1,240 @@ +handcuff +mortar +matchstick +wine_bottle +dowitcher +triumphal_arch +gyromitra +hatchet +airliner +broccoli +olive +pubg_lvl3backpack +calculator +toucan +shovel +sewing_machine +icecream +woodpecker +pig +relay_stick +mcdonald_sign +cpu +peanut +pumpkin +sturgeon +hammer +hami_melon +squirrel_monkey +shuriken +power_drill +pingpong_ball +crocodile +carambola +monarch_butterfly +drum +water_tower +panda +toilet_brush +pay_phone +yonex_icon +cricketball +revolver +chimpanzee +crab +corn +baseball +rabbit +croquet_ball +artichoke +abacus +harp +bell +gas_tank +scissors +vase +upright_piano +typewriter +bittern +impala +tray +fire_hydrant +beer_bottle +sock +soup_bowl +spider +cherry +macaw +toilet_seat +fire_balloon +french_ball +fox_squirrel +volleyball +cornmeal +folding_chair +pubg_airdrop +beagle +skateboard +narcissus +whiptail +cup +arabian_camel +badger +stopwatch +ab_wheel +ox +lettuce +monocycle +redshank +vulture +whistle +smoothing_iron +mashed_potato +conveyor +yoga_pad +tow_truck +siamese_cat +cigar +white_stork +sniper_rifle +stretcher +tulip +handkerchief +basset +iceberg +gibbon +lacewing +thrush +cheetah +bighorn_sheep +espresso_maker +pretzel +english_setter +sandbar +cheese +daisy +arctic_fox +briard +colubus +balance_beam +coffeepot +soap_dispenser +yawl +consomme +parking_meter +cactus +turnstile +taro +fire_screen +digital_clock +rose +pomegranate +bee_eater +schooner +ski_mask +jay_bird +plaice +red_fox +syringe +camomile +pickelhaube +blenheim_spaniel +pear +parachute +common_newt +bowtie +cigarette +oscilloscope +laptop +african_crocodile +apron +coconut +sandal +kwanyin +lion +eel +balloon +crepe +armadillo +kazoo +lemon +spider_monkey +tape_player +ipod +bee +sea_cucumber +suitcase +television +pillow +banjo +rock_snake +partridge +platypus +lycaenid_butterfly +pinecone +conversion_plug +wolf +frying_pan +timber_wolf +bluetick +crayon +giant_schnauzer +orang +scarerow +kobe_logo +loguat +saxophone +ceiling_fan +cardoon +equestrian_helmet +louvre_pyramid +hotdog +ironing_board +razor +nagoya_castle +loggerhead_turtle +lipstick +cradle +strongbox +raven +kit_fox +albatross +flat-coated_retriever +beer_glass +ice_lolly +sungnyemun +totem_pole +vacuum +bolete +mango +ginger +weasel +cabbage +refrigerator +school_bus +hippo +tiger_cat +saltshaker +piano_keyboard +windsor_tie +sea_urchin +microsd +barbell +swim_ring +bulbul_bird +water_ouzel +ac_ground +sweatshirt +umbrella +hair_drier +hammerhead_shark +tomato +projector +cushion +dishwasher +three-toed_sloth +tiger_shark +har_gow +baby +thor's_hammer +nike_logo diff --git a/submodules/lang_seg/fewshot_data/data/splits/pascal/trn/fold0.txt b/submodules/lang_seg/fewshot_data/data/splits/pascal/trn/fold0.txt new file mode 100644 index 0000000000000000000000000000000000000000..667b230414aaf22c699239a19a0b8354206a45d5 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/pascal/trn/fold0.txt @@ -0,0 +1,2286 @@ +2007_000032__01 +2007_000256__01 +2007_000480__01 +2007_000738__01 +2007_002099__01 +2007_002198__01 +2007_004009__01 +2007_004459__01 +2007_004988__01 +2007_006212__01 +2007_006232__01 +2007_008714__01 +2007_009030__01 +2007_009348__01 +2008_000033__01 +2008_000064__01 +2008_000197__01 +2008_000251__01 +2008_000291__01 +2008_000367__01 +2008_000585__01 +2008_000716__01 +2008_000756__01 +2008_000804__01 +2008_000883__01 +2008_001054__01 +2008_001227__01 +2008_001380__01 +2008_001448__01 +2008_001468__01 +2008_001719__01 +2008_001801__01 +2008_001805__01 +2008_002000__01 +2008_002138__01 +2008_002151__01 +2008_002195__01 +2008_002221__01 +2008_002454__01 +2008_002551__01 +2008_002673__01 +2008_002698__01 +2008_002719__01 +2008_002977__01 +2008_003041__01 +2008_003059__01 +2008_003189__01 +2008_003196__01 +2008_003261__01 +2008_003423__01 +2008_003475__01 +2008_003478__01 +2008_003575__01 +2008_003655__01 +2008_003673__01 +2008_003688__01 +2008_003703__01 +2008_003729__01 +2008_003743__01 +2008_003744__01 +2008_003788__01 +2008_003905__01 +2008_004030__01 +2008_004100__01 +2008_004138__01 +2008_004165__01 +2008_004239__01 +2008_004348__01 +2008_004378__01 +2008_004406__01 +2008_004532__01 +2008_004605__01 +2008_004630__01 +2008_004631__01 +2008_004646__01 +2008_004648__01 +2008_004726__01 +2008_004739__01 +2008_004771__01 +2008_004802__01 +2008_004819__01 +2008_004847__01 +2008_004885__01 +2008_004917__01 +2008_004981__01 +2008_005036__01 +2008_005057__01 +2008_005078__01 +2008_005114__01 +2008_005253__01 +2008_005310__01 +2008_005362__01 +2008_005373__01 +2008_005443__01 +2008_005455__01 +2008_005538__01 +2008_005611__01 +2008_005631__01 +2008_005634__01 +2008_005685__01 +2008_005750__01 +2008_005796__01 +2008_005834__01 +2008_005873__01 +2008_005889__01 +2008_005907__01 +2008_005916__01 +2008_006085__01 +2008_006140__01 +2008_006169__01 +2008_006200__01 +2008_006213__01 +2008_006265__01 +2008_006269__01 +2008_006353__01 +2008_006364__01 +2008_006394__01 +2008_006401__01 +2008_006463__01 +2008_006548__01 +2008_006564__01 +2008_006604__01 +2008_006619__01 +2008_006621__01 +2008_006623__01 +2008_006631__01 +2008_006637__01 +2008_006671__01 +2008_006677__01 +2008_006761__01 +2008_006777__01 +2008_006778__01 +2008_006896__01 +2008_006933__01 +2008_006951__01 +2008_006980__01 +2008_007145__01 +2008_007161__01 +2008_007195__01 +2008_007211__01 +2008_007229__01 +2008_007374__01 +2008_007397__01 +2008_007480__01 +2008_007504__01 +2008_007533__01 +2008_007579__01 +2008_007604__01 +2008_007618__01 +2008_007629__01 +2008_007758__01 +2008_007764__01 +2008_007879__01 +2008_007883__01 +2008_007884__01 +2008_007970__01 +2008_008004__01 +2008_008025__01 +2008_008028__01 +2008_008043__01 +2008_008044__01 +2008_008048__01 +2008_008086__01 +2008_008096__01 +2008_008130__01 +2008_008150__01 +2008_008179__01 +2008_008180__01 +2008_008191__01 +2008_008215__01 +2008_008254__01 +2008_008294__01 +2008_008300__01 +2008_008344__01 +2008_008345__01 +2008_008359__01 +2008_008373__01 +2008_008411__01 +2008_008432__01 +2008_008464__01 +2008_008467__01 +2008_008479__01 +2008_008496__01 +2008_008501__01 +2008_008507__01 +2008_008508__01 +2008_008519__01 +2008_008570__01 +2008_008607__01 +2008_008765__01 +2009_000052__01 +2009_000084__01 +2009_000122__01 +2009_000133__01 +2009_000159__01 +2009_000225__01 +2009_000260__01 +2009_000320__01 +2009_000327__01 +2009_000367__01 +2009_000397__01 +2009_000417__01 +2009_000500__01 +2009_000513__01 +2009_000545__01 +2009_000560__01 +2009_000661__01 +2009_000662__01 +2009_000695__01 +2009_000734__01 +2009_000796__01 +2009_000801__01 +2009_000817__01 +2009_000887__01 +2009_000909__01 +2009_000967__01 +2009_001027__01 +2009_001056__01 +2009_001090__01 +2009_001129__01 +2009_001195__01 +2009_001199__01 +2009_001312__01 +2009_001316__01 +2009_001321__01 +2009_001344__01 +2009_001345__01 +2009_001372__01 +2009_001387__01 +2009_001390__01 +2009_001435__01 +2009_001501__01 +2009_001509__01 +2009_001517__01 +2009_001541__01 +2009_001549__01 +2009_001595__01 +2009_001696__01 +2009_001713__01 +2009_001781__01 +2009_001884__01 +2009_001940__01 +2009_002001__01 +2009_002011__01 +2009_002099__01 +2009_002191__01 +2009_002199__01 +2009_002211__01 +2009_002254__01 +2009_002258__01 +2009_002272__01 +2009_002314__01 +2009_002388__01 +2009_002432__01 +2009_002448__01 +2009_002531__01 +2009_002599__01 +2009_002614__01 +2009_002625__01 +2009_002674__01 +2009_002714__01 +2009_002752__01 +2009_002770__01 +2009_002798__01 +2009_002910__01 +2009_002914__01 +2009_002941__01 +2009_002999__01 +2009_003033__01 +2009_003044__01 +2009_003066__01 +2009_003097__01 +2009_003107__01 +2009_003108__01 +2009_003147__01 +2009_003153__01 +2009_003199__01 +2009_003219__01 +2009_003278__01 +2009_003309__01 +2009_003372__01 +2009_003396__01 +2009_003456__01 +2009_003488__01 +2009_003524__01 +2009_003546__01 +2009_003571__01 +2009_003608__01 +2009_003635__01 +2009_003667__01 +2009_003704__01 +2009_003705__01 +2009_003760__01 +2009_003799__01 +2009_003883__01 +2009_003947__01 +2009_003950__01 +2009_003994__01 +2009_004075__01 +2009_004108__01 +2009_004203__01 +2009_004308__01 +2009_004317__01 +2009_004409__01 +2009_004414__01 +2009_004446__01 +2009_004475__01 +2009_004492__01 +2009_004535__01 +2009_004548__01 +2009_004601__01 +2009_004620__01 +2009_004626__01 +2009_004812__01 +2009_004846__01 +2009_004917__01 +2009_004944__01 +2009_004980__01 +2009_005019__01 +2009_005031__01 +2009_005036__01 +2009_005075__01 +2009_005111__01 +2009_005120__01 +2009_005127__01 +2009_005142__01 +2009_005210__01 +2009_005215__01 +2009_005218__01 +2009_005232__01 +2010_000091__01 +2010_000111__01 +2010_000196__01 +2010_000229__01 +2010_000250__01 +2010_000260__01 +2010_000270__01 +2010_000285__01 +2010_000295__01 +2010_000308__01 +2010_000323__01 +2010_000392__01 +2010_000418__01 +2010_000437__01 +2010_000477__01 +2010_000541__01 +2010_000549__01 +2010_000568__01 +2010_000715__01 +2010_000802__01 +2010_000885__01 +2010_000981__01 +2010_000986__01 +2010_001039__01 +2010_001085__01 +2010_001139__01 +2010_001205__01 +2010_001294__01 +2010_001320__01 +2010_001413__01 +2010_001426__01 +2010_001505__01 +2010_001583__01 +2010_001652__01 +2010_001718__01 +2010_001931__01 +2010_001950__01 +2010_001979__01 +2010_002019__01 +2010_002065__01 +2010_002085__01 +2010_002207__01 +2010_002289__01 +2010_002295__01 +2010_002357__01 +2010_002429__01 +2010_002496__01 +2010_002532__01 +2010_002542__01 +2010_002577__01 +2010_002579__01 +2010_002638__01 +2010_002684__01 +2010_002695__01 +2010_002816__01 +2010_002827__01 +2010_002954__01 +2010_003025__01 +2010_003051__01 +2010_003053__01 +2010_003054__01 +2010_003279__01 +2010_003353__01 +2010_003361__01 +2010_003458__01 +2010_003474__01 +2010_003496__01 +2010_003559__01 +2010_003655__01 +2010_003680__01 +2010_003686__01 +2010_003723__01 +2010_003737__01 +2010_003798__01 +2010_003801__01 +2010_003816__01 +2010_003933__01 +2010_003945__01 +2010_003983__01 +2010_004084__01 +2010_004118__01 +2010_004184__01 +2010_004238__01 +2010_004242__01 +2010_004312__01 +2010_004357__01 +2010_004404__01 +2010_004455__01 +2010_004546__01 +2010_004557__01 +2010_004601__01 +2010_004721__01 +2010_004729__01 +2010_004748__01 +2010_004791__01 +2010_004817__01 +2010_004855__01 +2010_004888__01 +2010_004938__01 +2010_004991__01 +2010_004998__01 +2010_005022__01 +2010_005059__01 +2010_005101__01 +2010_005106__01 +2010_005167__01 +2010_005184__01 +2010_005213__01 +2010_005224__01 +2010_005268__01 +2010_005301__01 +2010_005365__01 +2010_005382__01 +2010_005557__01 +2010_005632__01 +2010_005683__01 +2010_005715__01 +2010_005776__01 +2010_005942__01 +2010_005953__01 +2010_005974__01 +2010_005978__01 +2010_005998__01 +2010_006023__01 +2010_006032__01 +2010_006082__01 +2011_000103__01 +2011_000129__01 +2011_000163__01 +2011_000243__01 +2011_000276__01 +2011_000329__01 +2011_000359__01 +2011_000531__01 +2011_000586__01 +2011_000592__01 +2011_000629__01 +2011_000656__01 +2011_000698__01 +2011_000790__01 +2011_000793__01 +2011_000845__01 +2011_000872__01 +2011_000882__01 +2011_000922__01 +2011_000975__01 +2011_001044__01 +2011_001073__01 +2011_001081__01 +2011_001133__01 +2011_001158__01 +2011_001188__01 +2011_001257__01 +2011_001315__01 +2011_001320__01 +2011_001455__01 +2011_001480__01 +2011_001514__01 +2011_001541__01 +2011_001647__01 +2011_001699__01 +2011_001753__01 +2011_001789__01 +2011_001824__01 +2011_001858__01 +2011_001871__01 +2011_001875__01 +2011_001962__01 +2011_002012__01 +2011_002091__01 +2011_002114__01 +2011_002132__01 +2011_002193__01 +2011_002222__01 +2011_002248__01 +2011_002269__01 +2011_002284__01 +2011_002346__01 +2011_002380__01 +2011_002470__01 +2011_002494__01 +2011_002504__01 +2011_002552__01 +2011_002605__01 +2011_002617__01 +2011_002638__01 +2011_002673__01 +2011_002751__01 +2011_002851__01 +2011_002930__01 +2011_002992__01 +2011_003012__01 +2011_003034__01 +2011_003111__01 +2011_003154__01 +2011_003176__01 +2011_003177__01 +2011_003275__01 +2007_000584__02 +2007_002227__02 +2007_004769__02 +2007_005273__02 +2007_005368__02 +2007_005702__02 +2007_006317__02 +2007_006581__02 +2007_007447__02 +2007_009295__02 +2007_009788__02 +2008_000036__02 +2008_000090__02 +2008_000133__02 +2008_000191__02 +2008_000194__02 +2008_000196__02 +2008_000531__02 +2008_000562__02 +2008_000615__02 +2008_000764__02 +2008_000803__02 +2008_001225__02 +2008_001226__02 +2008_001336__02 +2008_001375__02 +2008_001402__02 +2008_001626__02 +2008_001813__02 +2008_001860__02 +2008_001914__02 +2008_001986__02 +2008_002129__02 +2008_002204__02 +2008_002322__02 +2008_002370__02 +2008_002384__02 +2008_002418__02 +2008_002515__02 +2008_002631__02 +2008_002679__02 +2008_002701__02 +2008_002714__02 +2008_002715__02 +2008_002787__02 +2008_002880__02 +2008_002883__02 +2008_002894__02 +2008_002932__02 +2008_002948__02 +2008_003075__02 +2008_003140__02 +2008_003205__02 +2008_003329__02 +2008_003351__02 +2008_003395__02 +2008_003617__02 +2008_003682__02 +2008_003768__02 +2008_003812__02 +2008_003819__02 +2008_003940__02 +2008_004040__02 +2008_004142__02 +2008_004280__02 +2008_004325__02 +2008_004441__02 +2008_004482__02 +2008_004574__02 +2008_004592__02 +2008_004603__02 +2008_004784__02 +2008_004862__02 +2008_005147__02 +2008_005190__02 +2008_005201__02 +2008_005276__02 +2008_005356__02 +2008_005412__02 +2008_005484__02 +2008_005553__02 +2008_005599__02 +2008_005713__02 +2008_005737__02 +2008_005808__02 +2008_005839__02 +2008_005846__02 +2008_005921__02 +2008_005970__02 +2008_005978__02 +2008_005982__02 +2008_006064__02 +2008_006088__02 +2008_006154__02 +2008_006195__02 +2008_006234__02 +2008_006253__02 +2008_006257__02 +2008_006361__02 +2008_006410__02 +2008_006424__02 +2008_006467__02 +2008_006519__02 +2008_006579__02 +2008_006645__02 +2008_006807__02 +2008_006813__02 +2008_006855__02 +2008_007019__02 +2008_007103__02 +2008_007222__02 +2008_007265__02 +2008_007410__02 +2008_007421__02 +2008_007470__02 +2008_007510__02 +2008_007719__02 +2008_007761__02 +2008_007766__02 +2008_007833__02 +2008_007861__02 +2008_007882__02 +2008_007915__02 +2008_007935__02 +2008_007993__02 +2008_008064__02 +2008_008069__02 +2008_008080__02 +2008_008084__02 +2008_008095__02 +2008_008097__02 +2008_008112__02 +2008_008210__02 +2008_008307__02 +2008_008315__02 +2008_008320__02 +2008_008325__02 +2008_008337__02 +2008_008357__02 +2008_008368__02 +2008_008443__02 +2008_008528__02 +2008_008530__02 +2008_008564__02 +2008_008572__02 +2008_008579__02 +2008_008591__02 +2008_008611__02 +2008_008619__02 +2008_008652__02 +2008_008671__02 +2008_008691__02 +2008_008707__02 +2008_008708__02 +2008_008717__02 +2008_008718__02 +2008_008724__02 +2008_008745__02 +2008_008753__02 +2009_000055__02 +2009_000161__02 +2009_000195__02 +2009_000237__02 +2009_000286__02 +2009_000344__02 +2009_000430__02 +2009_000445__02 +2009_000461__02 +2009_000634__02 +2009_000696__02 +2009_000724__02 +2009_000756__02 +2009_000757__02 +2009_000805__02 +2009_000820__02 +2009_000882__02 +2009_000896__02 +2009_000975__02 +2009_000985__02 +2009_001096__02 +2009_001181__02 +2009_001364__02 +2009_001384__02 +2009_001456__02 +2009_001480__02 +2009_001481__02 +2009_001558__02 +2009_001642__02 +2009_001695__02 +2009_001704__02 +2009_001758__02 +2009_001780__02 +2009_001830__02 +2009_001840__02 +2009_001926__02 +2009_002058__02 +2009_002087__02 +2009_002119__02 +2009_002145__02 +2009_002194__02 +2009_002271__02 +2009_002338__02 +2009_002350__02 +2009_002371__02 +2009_002400__02 +2009_002401__02 +2009_002404__02 +2009_002406__02 +2009_002438__02 +2009_002488__02 +2009_002668__02 +2009_002671__02 +2009_002764__02 +2009_002777__02 +2009_002779__02 +2009_002780__02 +2009_002785__02 +2009_002833__02 +2009_002835__02 +2009_002983__02 +2009_002985__02 +2009_003007__02 +2009_003042__02 +2009_003075__02 +2009_003164__02 +2009_003175__02 +2009_003187__02 +2009_003353__02 +2009_003381__02 +2009_003469__02 +2009_003519__02 +2009_003644__02 +2009_003646__02 +2009_003832__02 +2009_003836__02 +2009_003860__02 +2009_003905__02 +2009_003908__02 +2009_004069__02 +2009_004121__02 +2009_004303__02 +2009_004340__02 +2009_004364__02 +2009_004444__02 +2009_004508__02 +2009_004551__02 +2009_004561__02 +2009_004571__02 +2009_004629__02 +2009_004642__02 +2009_004664__02 +2009_004731__02 +2009_004758__02 +2009_004781__02 +2009_004797__02 +2009_004822__02 +2009_004831__02 +2009_004876__02 +2009_004905__02 +2009_004914__02 +2009_004933__02 +2009_004934__02 +2009_004984__02 +2009_005064__02 +2009_005082__02 +2009_005098__02 +2009_005103__02 +2009_005144__02 +2009_005154__02 +2009_005172__02 +2009_005178__02 +2009_005203__02 +2009_005265__02 +2009_005300__02 +2010_000056__02 +2010_000113__02 +2010_000209__02 +2010_000254__02 +2010_000356__02 +2010_000401__02 +2010_000444__02 +2010_000462__02 +2010_000557__02 +2010_000571__02 +2010_000624__02 +2010_000685__02 +2010_000747__02 +2010_000829__02 +2010_000893__02 +2010_000898__02 +2010_000910__02 +2010_001049__02 +2010_001119__02 +2010_001183__02 +2010_001254__02 +2010_001338__02 +2010_001431__02 +2010_001514__02 +2010_001540__02 +2010_001592__02 +2010_001606__02 +2010_001753__02 +2010_001921__02 +2010_001922__02 +2010_001970__02 +2010_002055__02 +2010_002180__02 +2010_002211__02 +2010_002326__02 +2010_002475__02 +2010_002497__02 +2010_002656__02 +2010_002660__02 +2010_002708__02 +2010_002767__02 +2010_002801__02 +2010_002814__02 +2010_002820__02 +2010_002915__02 +2010_002927__02 +2010_003062__02 +2010_003115__02 +2010_003251__02 +2010_003264__02 +2010_003314__02 +2010_003350__02 +2010_003367__02 +2010_003379__02 +2010_003503__02 +2010_003585__02 +2010_003603__02 +2010_003671__02 +2010_003689__02 +2010_003701__02 +2010_003703__02 +2010_003729__02 +2010_003799__02 +2010_003811__02 +2010_003828__02 +2010_003906__02 +2010_003914__02 +2010_004072__02 +2010_004154__02 +2010_004182__02 +2010_004307__02 +2010_004439__02 +2010_004537__02 +2010_004609__02 +2010_004809__02 +2010_004878__02 +2010_004894__02 +2010_005000__02 +2010_005093__02 +2010_005111__02 +2010_005116__02 +2010_005211__02 +2010_005374__02 +2010_005542__02 +2010_005556__02 +2010_005658__02 +2010_005670__02 +2010_005782__02 +2010_005810__02 +2010_005840__02 +2010_005848__02 +2010_005909__02 +2010_006031__02 +2010_006063__02 +2011_000098__02 +2011_000297__02 +2011_000376__02 +2011_000453__02 +2011_000465__02 +2011_000474__02 +2011_000485__02 +2011_000505__02 +2011_000579__02 +2011_000690__02 +2011_000791__02 +2011_000933__02 +2011_000950__02 +2011_001028__02 +2011_001030__02 +2011_001138__02 +2011_001153__02 +2011_001213__02 +2011_001271__02 +2011_001422__02 +2011_001441__02 +2011_001501__02 +2011_001524__02 +2011_001535__02 +2011_001538__02 +2011_001599__02 +2011_001612__02 +2011_001632__02 +2011_001643__02 +2011_001732__02 +2011_001806__02 +2011_001896__02 +2011_001914__02 +2011_001937__02 +2011_001945__02 +2011_001966__02 +2011_001987__02 +2011_002003__02 +2011_002006__02 +2011_002049__02 +2011_002111__02 +2011_002113__02 +2011_002135__02 +2011_002192__02 +2011_002281__02 +2011_002325__02 +2011_002388__02 +2011_002406__02 +2011_002455__02 +2011_002543__02 +2011_002657__02 +2011_002742__02 +2011_002811__02 +2011_002913__02 +2011_002958__02 +2011_003063__02 +2011_003148__02 +2011_003167__02 +2011_003168__02 +2011_003169__02 +2007_000068__03 +2007_000363__03 +2007_000645__03 +2007_002212__03 +2007_002896__03 +2007_003118__03 +2007_003267__03 +2007_003330__03 +2007_005130__03 +2007_005264__03 +2007_006865__03 +2007_007481__03 +2007_008764__03 +2007_009422__03 +2007_009607__03 +2007_009759__03 +2008_000103__03 +2008_000131__03 +2008_000134__03 +2008_000192__03 +2008_000318__03 +2008_000339__03 +2008_000350__03 +2008_000361__03 +2008_000512__03 +2008_000515__03 +2008_000532__03 +2008_000678__03 +2008_000785__03 +2008_000798__03 +2008_000832__03 +2008_000861__03 +2008_000960__03 +2008_001020__03 +2008_001185__03 +2008_001194__03 +2008_001264__03 +2008_001351__03 +2008_001387__03 +2008_001415__03 +2008_001495__03 +2008_001522__03 +2008_001673__03 +2008_001679__03 +2008_001770__03 +2008_001810__03 +2008_001829__03 +2008_001856__03 +2008_002064__03 +2008_002073__03 +2008_002200__03 +2008_002208__03 +2008_002255__03 +2008_002349__03 +2008_002350__03 +2008_002369__03 +2008_002389__03 +2008_002399__03 +2008_002471__03 +2008_002647__03 +2008_002684__03 +2008_002784__03 +2008_002847__03 +2008_002961__03 +2008_002970__03 +2008_002983__03 +2008_003023__03 +2008_003087__03 +2008_003106__03 +2008_003160__03 +2008_003161__03 +2008_003170__03 +2008_003211__03 +2008_003222__03 +2008_003232__03 +2008_003291__03 +2008_003360__03 +2008_003382__03 +2008_003426__03 +2008_003439__03 +2008_003462__03 +2008_003484__03 +2008_003485__03 +2008_003524__03 +2008_003580__03 +2008_003683__03 +2008_003694__03 +2008_003763__03 +2008_003789__03 +2008_003831__03 +2008_003894__03 +2008_003942__03 +2008_003997__03 +2008_004024__03 +2008_004042__03 +2008_004075__03 +2008_004087__03 +2008_004103__03 +2008_004105__03 +2008_004147__03 +2008_004234__03 +2008_004276__03 +2008_004296__03 +2008_004317__03 +2008_004362__03 +2008_004385__03 +2008_004425__03 +2008_004452__03 +2008_004520__03 +2008_004551__03 +2008_004620__03 +2008_004690__03 +2008_004730__03 +2008_004805__03 +2008_004808__03 +2008_004825__03 +2008_004837__03 +2008_004893__03 +2008_004942__03 +2008_004973__03 +2008_005080__03 +2008_005092__03 +2008_005096__03 +2008_005115__03 +2008_005182__03 +2008_005186__03 +2008_005209__03 +2008_005233__03 +2008_005260__03 +2008_005279__03 +2008_005296__03 +2008_005303__03 +2008_005357__03 +2008_005465__03 +2008_005477__03 +2008_005507__03 +2008_005521__03 +2008_005522__03 +2008_005531__03 +2008_005534__03 +2008_005584__03 +2008_005618__03 +2008_005626__03 +2008_005649__03 +2008_005687__03 +2008_005724__03 +2008_005734__03 +2008_005757__03 +2008_005774__03 +2008_005803__03 +2008_005810__03 +2008_005821__03 +2008_005903__03 +2008_005935__03 +2008_005956__03 +2008_006000__03 +2008_006002__03 +2008_006058__03 +2008_006090__03 +2008_006128__03 +2008_006164__03 +2008_006179__03 +2008_006186__03 +2008_006227__03 +2008_006281__03 +2008_006369__03 +2008_006376__03 +2008_006432__03 +2008_006447__03 +2008_006503__03 +2008_006566__03 +2008_006598__03 +2008_006626__03 +2008_006667__03 +2008_006715__03 +2008_006764__03 +2008_006882__03 +2008_006923__03 +2008_006924__03 +2008_006954__03 +2008_007003__03 +2008_007098__03 +2008_007124__03 +2008_007205__03 +2008_007250__03 +2008_007266__03 +2008_007317__03 +2008_007423__03 +2008_007434__03 +2008_007465__03 +2008_007486__03 +2008_007501__03 +2008_007546__03 +2008_007587__03 +2008_007593__03 +2008_007594__03 +2008_007608__03 +2008_007623__03 +2008_007656__03 +2008_007669__03 +2008_007673__03 +2008_007688__03 +2008_007709__03 +2008_007714__03 +2008_007730__03 +2008_007752__03 +2008_007788__03 +2008_007812__03 +2008_007839__03 +2008_007854__03 +2008_007870__03 +2008_007891__03 +2008_007936__03 +2008_007940__03 +2008_007948__03 +2008_008002__03 +2008_008007__03 +2008_008022__03 +2008_008037__03 +2008_008073__03 +2008_008105__03 +2008_008125__03 +2008_008134__03 +2008_008145__03 +2008_008166__03 +2008_008175__03 +2008_008194__03 +2008_008197__03 +2008_008211__03 +2008_008233__03 +2008_008242__03 +2008_008257__03 +2008_008274__03 +2008_008284__03 +2008_008287__03 +2008_008292__03 +2008_008309__03 +2008_008314__03 +2008_008346__03 +2008_008347__03 +2008_008354__03 +2008_008376__03 +2008_008377__03 +2008_008380__03 +2008_008387__03 +2008_008404__03 +2008_008406__03 +2008_008435__03 +2008_008447__03 +2008_008455__03 +2008_008461__03 +2008_008490__03 +2008_008506__03 +2008_008512__03 +2008_008544__03 +2008_008595__03 +2009_000026__03 +2009_000035__03 +2009_000040__03 +2009_000103__03 +2009_000146__03 +2009_000151__03 +2009_000206__03 +2009_000218__03 +2009_000254__03 +2009_000317__03 +2009_000370__03 +2009_000389__03 +2009_000393__03 +2009_000435__03 +2009_000486__03 +2009_000503__03 +2009_000527__03 +2009_000536__03 +2009_000579__03 +2009_000677__03 +2009_000692__03 +2009_000748__03 +2009_000758__03 +2009_000790__03 +2009_000849__03 +2009_000898__03 +2009_000904__03 +2009_000930__03 +2009_000932__03 +2009_000938__03 +2009_000958__03 +2009_000961__03 +2009_001040__03 +2009_001128__03 +2009_001166__03 +2009_001208__03 +2009_001221__03 +2009_001242__03 +2009_001264__03 +2009_001283__03 +2009_001301__03 +2009_001311__03 +2009_001348__03 +2009_001360__03 +2009_001375__03 +2009_001385__03 +2009_001397__03 +2009_001422__03 +2009_001453__03 +2009_001466__03 +2009_001554__03 +2009_001570__03 +2009_001587__03 +2009_001660__03 +2009_001693__03 +2009_001751__03 +2009_001801__03 +2009_001807__03 +2009_001929__03 +2009_001972__03 +2009_001999__03 +2009_002056__03 +2009_002072__03 +2009_002086__03 +2009_002153__03 +2009_002205__03 +2009_002226__03 +2009_002256__03 +2009_002262__03 +2009_002264__03 +2009_002286__03 +2009_002298__03 +2009_002319__03 +2009_002414__03 +2009_002423__03 +2009_002431__03 +2009_002525__03 +2009_002608__03 +2009_002629__03 +2009_002645__03 +2009_002685__03 +2009_002754__03 +2009_002791__03 +2009_002799__03 +2009_002809__03 +2009_002820__03 +2009_002831__03 +2009_002877__03 +2009_002890__03 +2009_002920__03 +2009_002962__03 +2009_002977__03 +2009_003002__03 +2009_003052__03 +2009_003088__03 +2009_003109__03 +2009_003168__03 +2009_003259__03 +2009_003285__03 +2009_003300__03 +2009_003399__03 +2009_003417__03 +2009_003487__03 +2009_003531__03 +2009_003606__03 +2009_003685__03 +2009_003713__03 +2009_003837__03 +2009_003870__03 +2009_003911__03 +2009_003912__03 +2009_003922__03 +2009_003962__03 +2009_003993__03 +2009_004001__03 +2009_004023__03 +2009_004044__03 +2009_004103__03 +2009_004141__03 +2009_004174__03 +2009_004315__03 +2009_004334__03 +2009_004346__03 +2009_004375__03 +2009_004552__03 +2009_004565__03 +2009_004625__03 +2009_004652__03 +2009_004661__03 +2009_004686__03 +2009_004713__03 +2009_004723__03 +2009_004754__03 +2009_004765__03 +2009_004817__03 +2009_004858__03 +2009_004898__03 +2009_004946__03 +2009_004959__03 +2009_005040__03 +2009_005076__03 +2009_005114__03 +2009_005131__03 +2009_005191__03 +2009_005198__03 +2009_005234__03 +2009_005282__03 +2010_000023__03 +2010_000075__03 +2010_000103__03 +2010_000151__03 +2010_000169__03 +2010_000184__03 +2010_000197__03 +2010_000222__03 +2010_000393__03 +2010_000621__03 +2010_000707__03 +2010_000863__03 +2010_000994__03 +2010_001042__03 +2010_001098__03 +2010_001120__03 +2010_001159__03 +2010_001201__03 +2010_001372__03 +2010_001441__03 +2010_001481__03 +2010_001520__03 +2010_001569__03 +2010_001584__03 +2010_001599__03 +2010_001603__03 +2010_001697__03 +2010_001715__03 +2010_001784__03 +2010_001806__03 +2010_001868__03 +2010_001919__03 +2010_001927__03 +2010_001988__03 +2010_001992__03 +2010_002023__03 +2010_002181__03 +2010_002242__03 +2010_002374__03 +2010_002436__03 +2010_002439__03 +2010_002569__03 +2010_002614__03 +2010_002621__03 +2010_002632__03 +2010_002666__03 +2010_002688__03 +2010_002740__03 +2010_002854__03 +2010_002873__03 +2010_002935__03 +2010_002955__03 +2010_003015__03 +2010_003044__03 +2010_003081__03 +2010_003097__03 +2010_003119__03 +2010_003255__03 +2010_003344__03 +2010_003395__03 +2010_003419__03 +2010_003503__03 +2010_003526__03 +2010_003546__03 +2010_003648__03 +2010_003724__03 +2010_003743__03 +2010_003929__03 +2010_004026__03 +2010_004187__03 +2010_004201__03 +2010_004230__03 +2010_004252__03 +2010_004352__03 +2010_004385__03 +2010_004431__03 +2010_004451__03 +2010_004483__03 +2010_004521__03 +2010_004570__03 +2010_004657__03 +2010_004847__03 +2010_004906__03 +2010_004953__03 +2010_005016__03 +2010_005023__03 +2010_005053__03 +2010_005068__03 +2010_005096__03 +2010_005129__03 +2010_005152__03 +2010_005217__03 +2010_005320__03 +2010_005350__03 +2010_005426__03 +2010_005472__03 +2010_005498__03 +2010_005511__03 +2010_005516__03 +2010_005586__03 +2010_005603__03 +2010_005608__03 +2010_005651__03 +2010_005725__03 +2010_005837__03 +2010_005896__03 +2010_005960__03 +2010_005968__03 +2010_006084__03 +2011_000017__03 +2011_000068__03 +2011_000114__03 +2011_000149__03 +2011_000222__03 +2011_000246__03 +2011_000268__03 +2011_000269__03 +2011_000320__03 +2011_000324__03 +2011_000374__03 +2011_000392__03 +2011_000494__03 +2011_000511__03 +2011_000554__03 +2011_000673__03 +2011_000718__03 +2011_000734__03 +2011_000831__03 +2011_000850__03 +2011_001016__03 +2011_001084__03 +2011_001117__03 +2011_001134__03 +2011_001139__03 +2011_001211__03 +2011_001229__03 +2011_001254__03 +2011_001326__03 +2011_001432__03 +2011_001463__03 +2011_001471__03 +2011_001505__03 +2011_001558__03 +2011_001652__03 +2011_001694__03 +2011_001712__03 +2011_001764__03 +2011_001779__03 +2011_001866__03 +2011_001873__03 +2011_001902__03 +2011_001924__03 +2011_001952__03 +2011_001967__03 +2011_002016__03 +2011_002055__03 +2011_002100__03 +2011_002221__03 +2011_002228__03 +2011_002230__03 +2011_002245__03 +2011_002265__03 +2011_002347__03 +2011_002381__03 +2011_002407__03 +2011_002458__03 +2011_002463__03 +2011_002503__03 +2011_002568__03 +2011_002582__03 +2011_002652__03 +2011_002674__03 +2011_002715__03 +2011_002776__03 +2011_002868__03 +2011_002889__03 +2011_003091__03 +2011_003163__03 +2011_003171__03 +2011_003213__03 +2007_000241__04 +2007_000713__04 +2007_001698__04 +2007_002234__04 +2007_002403__04 +2007_003910__04 +2007_006281__04 +2007_006490__04 +2007_006660__04 +2007_006673__04 +2007_007415__04 +2007_008219__04 +2007_008571__04 +2007_009527__04 +2007_009533__04 +2007_009947__04 +2008_000007__04 +2008_000036__04 +2008_000140__04 +2008_000195__04 +2008_000235__04 +2008_000262__04 +2008_000405__04 +2008_000414__04 +2008_000423__04 +2008_000437__04 +2008_000471__04 +2008_000505__04 +2008_000567__04 +2008_000833__04 +2008_000834__04 +2008_000957__04 +2008_001041__04 +2008_001047__04 +2008_001056__04 +2008_001136__04 +2008_001159__04 +2008_001160__04 +2008_001167__04 +2008_001202__04 +2008_001389__04 +2008_001420__04 +2008_001516__04 +2008_001529__04 +2008_001575__04 +2008_001620__04 +2008_001649__04 +2008_001652__04 +2008_001843__04 +2008_001849__04 +2008_001858__04 +2008_001970__04 +2008_001977__04 +2008_001987__04 +2008_002046__04 +2008_002131__04 +2008_002194__04 +2008_002250__04 +2008_002335__04 +2008_002610__04 +2008_002625__04 +2008_002773__04 +2008_002850__04 +2008_002887__04 +2008_002966__04 +2008_003062__04 +2008_003081__04 +2008_003100__04 +2008_003112__04 +2008_003187__04 +2008_003313__04 +2008_003362__04 +2008_003402__04 +2008_003449__04 +2008_003458__04 +2008_003480__04 +2008_003652__04 +2008_003701__04 +2008_003840__04 +2008_003849__04 +2008_003870__04 +2008_003913__04 +2008_003988__04 +2008_004014__04 +2008_004026__04 +2008_004053__04 +2008_004124__04 +2008_004242__04 +2008_004291__04 +2008_004469__04 +2008_004570__04 +2008_004636__04 +2008_004718__04 +2008_004812__04 +2008_004904__04 +2008_004920__04 +2008_004921__04 +2008_004969__04 +2008_004983__04 +2008_005063__04 +2008_005071__04 +2008_005108__04 +2008_005244__04 +2008_005321__04 +2008_005359__04 +2008_005367__04 +2008_005502__04 +2008_005504__04 +2008_005512__04 +2008_005517__04 +2008_005526__04 +2008_005536__04 +2008_005541__04 +2008_005593__04 +2008_005610__04 +2008_005683__04 +2008_005695__04 +2008_005720__04 +2008_005764__04 +2008_005855__04 +2008_005863__04 +2008_005923__04 +2008_006046__04 +2008_006065__04 +2008_006094__04 +2008_006121__04 +2008_006185__04 +2008_006289__04 +2008_006349__04 +2008_006404__04 +2008_006407__04 +2008_006487__04 +2008_006489__04 +2008_006490__04 +2008_006586__04 +2008_006730__04 +2008_006767__04 +2008_006824__04 +2008_006825__04 +2008_006831__04 +2008_006833__04 +2008_006877__04 +2008_006903__04 +2008_006925__04 +2008_006949__04 +2008_006961__04 +2008_007056__04 +2008_007101__04 +2008_007156__04 +2008_007305__04 +2008_007382__04 +2008_007425__04 +2008_007441__04 +2008_007574__04 +2008_007597__04 +2008_007599__04 +2008_007643__04 +2008_007653__04 +2008_007750__04 +2008_007755__04 +2008_007841__04 +2008_007883__04 +2008_007884__04 +2008_007953__04 +2008_007966__04 +2008_008083__04 +2008_008170__04 +2008_008235__04 +2008_008616__04 +2009_000093__04 +2009_000120__04 +2009_000305__04 +2009_000378__04 +2009_000385__04 +2009_000420__04 +2009_000477__04 +2009_000491__04 +2009_000516__04 +2009_000686__04 +2009_000690__04 +2009_000752__04 +2009_000829__04 +2009_000830__04 +2009_000996__04 +2009_001019__04 +2009_001282__04 +2009_001289__04 +2009_001406__04 +2009_001407__04 +2009_001544__04 +2009_001567__04 +2009_001715__04 +2009_001770__04 +2009_001794__04 +2009_001949__04 +2009_002096__04 +2009_002127__04 +2009_002146__04 +2009_002173__04 +2009_002289__04 +2009_002306__04 +2009_002308__04 +2009_002343__04 +2009_002358__04 +2009_002397__04 +2009_002449__04 +2009_002506__04 +2009_002530__04 +2009_002543__04 +2009_002577__04 +2009_002662__04 +2009_002759__04 +2009_002778__04 +2009_002830__04 +2009_002897__04 +2009_002946__04 +2009_003010__04 +2009_003019__04 +2009_003032__04 +2009_003136__04 +2009_003150__04 +2009_003247__04 +2009_003284__04 +2009_003345__04 +2009_003351__04 +2009_003400__04 +2009_003482__04 +2009_003511__04 +2009_003600__04 +2009_003652__04 +2009_003936__04 +2009_004007__04 +2009_004078__04 +2009_004148__04 +2009_004166__04 +2009_004178__04 +2009_004224__04 +2009_004273__04 +2009_004278__04 +2009_004397__04 +2009_004524__04 +2009_004539__04 +2009_004780__04 +2009_004786__04 +2009_004904__04 +2009_005006__04 +2009_005015__04 +2009_005118__04 +2009_005239__04 +2010_000024__04 +2010_000145__04 +2010_000195__04 +2010_000382__04 +2010_000630__04 +2010_000710__04 +2010_000748__04 +2010_000803__04 +2010_000887__04 +2010_001126__04 +2010_001212__04 +2010_001289__04 +2010_001456__04 +2010_001501__04 +2010_001525__04 +2010_001551__04 +2010_001729__04 +2010_001749__04 +2010_001807__04 +2010_001899__04 +2010_001916__04 +2010_001967__04 +2010_002117__04 +2010_002216__04 +2010_002315__04 +2010_002321__04 +2010_002364__04 +2010_002393__04 +2010_002498__04 +2010_002565__04 +2010_002620__04 +2010_002653__04 +2010_002702__04 +2010_002780__04 +2010_002899__04 +2010_003027__04 +2010_003103__04 +2010_003107__04 +2010_003117__04 +2010_003203__04 +2010_003260__04 +2010_003508__04 +2010_003599__04 +2010_003604__04 +2010_003640__04 +2010_003644__04 +2010_003679__04 +2010_003757__04 +2010_003815__04 +2010_003860__04 +2010_003878__04 +2010_003899__04 +2010_004006__04 +2010_004081__04 +2010_004124__04 +2010_004253__04 +2010_004486__04 +2010_004561__04 +2010_004567__04 +2010_004576__04 +2010_004714__04 +2010_004773__04 +2010_004852__04 +2010_004950__04 +2010_005192__04 +2010_005198__04 +2010_005332__04 +2010_005364__04 +2010_005588__04 +2010_005597__04 +2010_005780__04 +2010_005927__04 +2011_000086__04 +2011_000130__04 +2011_000161__04 +2011_000278__04 +2011_000286__04 +2011_000342__04 +2011_000347__04 +2011_000556__04 +2011_000631__04 +2011_000683__04 +2011_000744__04 +2011_000749__04 +2011_000839__04 +2011_000986__04 +2011_001001__04 +2011_001029__04 +2011_001097__04 +2011_001160__04 +2011_001176__04 +2011_001221__04 +2011_001252__04 +2011_001310__04 +2011_001384__04 +2011_001557__04 +2011_001582__04 +2011_001592__04 +2011_001710__04 +2011_001771__04 +2011_001901__04 +2011_001961__04 +2011_002036__04 +2011_002050__04 +2011_002073__04 +2011_002421__04 +2011_002459__04 +2011_002490__04 +2011_002558__04 +2011_002588__04 +2011_002661__04 +2011_002677__04 +2011_002740__04 +2011_002821__04 +2011_002871__04 +2011_002912__04 +2011_002943__04 +2011_003020__04 +2011_003025__04 +2011_003041__04 +2011_003081__04 +2011_003232__04 +2007_000170__05 +2007_000250__05 +2007_001185__05 +2007_001602__05 +2007_002953__05 +2007_003207__05 +2007_003431__05 +2007_004291__05 +2007_004476__05 +2007_007021__05 +2007_007250__05 +2007_008778__05 +2007_009709__05 +2008_000015__05 +2008_000023__05 +2008_000034__05 +2008_000112__05 +2008_000145__05 +2008_000154__05 +2008_000290__05 +2008_000522__05 +2008_000787__05 +2008_000851__05 +2008_001077__05 +2008_001199__05 +2008_001230__05 +2008_001304__05 +2008_001307__05 +2008_001405__05 +2008_001431__05 +2008_001444__05 +2008_001593__05 +2008_001737__05 +2008_001744__05 +2008_001837__05 +2008_001910__05 +2008_002002__05 +2008_002005__05 +2008_002153__05 +2008_002292__05 +2008_002311__05 +2008_002362__05 +2008_002372__05 +2008_002606__05 +2008_002870__05 +2008_003079__05 +2008_003186__05 +2008_003200__05 +2008_003276__05 +2008_003289__05 +2008_003463__05 +2008_003467__05 +2008_003579__05 +2008_003591__05 +2008_003635__05 +2008_003722__05 +2008_003969__05 +2008_004016__05 +2008_004037__05 +2008_004090__05 +2008_004126__05 +2008_004145__05 +2008_004178__05 +2008_004308__05 +2008_004418__05 +2008_004501__05 +2008_004564__05 +2008_004607__05 +2008_004666__05 +2008_004898__05 +2008_004926__05 +2008_004934__05 +2008_004937__05 +2008_004948__05 +2008_004979__05 +2008_004982__05 +2008_005023__05 +2008_005140__05 +2008_005182__05 +2008_005204__05 +2008_005324__05 +2008_005342__05 +2008_005560__05 +2008_005758__05 +2008_005867__05 +2008_005869__05 +2008_005881__05 +2008_005939__05 +2008_005945__05 +2008_005991__05 +2008_006004__05 +2008_006024__05 +2008_006027__05 +2008_006052__05 +2008_006072__05 +2008_006104__05 +2008_006111__05 +2008_006120__05 +2008_006136__05 +2008_006144__05 +2008_006182__05 +2008_006192__05 +2008_006282__05 +2008_006295__05 +2008_006427__05 +2008_006558__05 +2008_006585__05 +2008_006605__05 +2008_006641__05 +2008_006657__05 +2008_006701__05 +2008_006731__05 +2008_006792__05 +2008_006815__05 +2008_006890__05 +2008_006941__05 +2008_006969__05 +2008_007058__05 +2008_007076__05 +2008_007134__05 +2008_007142__05 +2008_007168__05 +2008_007261__05 +2008_007361__05 +2008_007394__05 +2008_007528__05 +2008_007585__05 +2008_007591__05 +2008_007664__05 +2008_007697__05 +2008_007704__05 +2008_007724__05 +2008_007858__05 +2008_007923__05 +2008_007954__05 +2008_008330__05 +2008_008694__05 +2008_008700__05 +2009_000058__05 +2009_000130__05 +2009_000247__05 +2009_000257__05 +2009_000277__05 +2009_000340__05 +2009_000377__05 +2009_000414__05 +2009_000463__05 +2009_000494__05 +2009_000502__05 +2009_000546__05 +2009_000562__05 +2009_000742__05 +2009_000763__05 +2009_000783__05 +2009_000797__05 +2009_000867__05 +2009_000992__05 +2009_001011__05 +2009_001024__05 +2009_001054__05 +2009_001059__05 +2009_001103__05 +2009_001120__05 +2009_001154__05 +2009_001207__05 +2009_001224__05 +2009_001309__05 +2009_001328__05 +2009_001339__05 +2009_001398__05 +2009_001414__05 +2009_001480__05 +2009_001494__05 +2009_001593__05 +2009_001625__05 +2009_001640__05 +2009_001705__05 +2009_001720__05 +2009_001743__05 +2009_001799__05 +2009_001810__05 +2009_001848__05 +2009_001907__05 +2009_001908__05 +2009_001933__05 +2009_001937__05 +2009_002060__05 +2009_002089__05 +2009_002137__05 +2009_002204__05 +2009_002235__05 +2009_002328__05 +2009_002464__05 +2009_002475__05 +2009_002567__05 +2009_002579__05 +2009_002620__05 +2009_002675__05 +2009_002814__05 +2009_002850__05 +2009_002940__05 +2009_002980__05 +2009_002984__05 +2009_003034__05 +2009_003114__05 +2009_003142__05 +2009_003191__05 +2009_003347__05 +2009_003349__05 +2009_003468__05 +2009_003562__05 +2009_003588__05 +2009_003609__05 +2009_003629__05 +2009_003664__05 +2009_003720__05 +2009_003793__05 +2009_003795__05 +2009_003818__05 +2009_003865__05 +2009_003888__05 +2009_003899__05 +2009_003914__05 +2009_003958__05 +2009_004031__05 +2009_004085__05 +2009_004112__05 +2009_004122__05 +2009_004162__05 +2009_004199__05 +2009_004228__05 +2009_004332__05 +2009_004432__05 +2009_004542__05 +2009_004656__05 +2009_004745__05 +2009_004749__05 +2009_004764__05 +2009_004856__05 +2009_005044__05 +2009_005150__05 +2009_005308__05 +2010_000050__05 +2010_000131__05 +2010_000137__05 +2010_000213__05 +2010_000310__05 +2010_000325__05 +2010_000352__05 +2010_000386__05 +2010_000591__05 +2010_000697__05 +2010_000721__05 +2010_000805__05 +2010_000897__05 +2010_000915__05 +2010_001013__05 +2010_001131__05 +2010_001188__05 +2010_001219__05 +2010_001224__05 +2010_001463__05 +2010_001535__05 +2010_001680__05 +2010_001698__05 +2010_001785__05 +2010_001819__05 +2010_001877__05 +2010_001884__05 +2010_002244__05 +2010_002269__05 +2010_002373__05 +2010_002388__05 +2010_002440__05 +2010_002455__05 +2010_002520__05 +2010_002539__05 +2010_002668__05 +2010_002772__05 +2010_002843__05 +2010_002858__05 +2010_002931__05 +2010_002941__05 +2010_002956__05 +2010_003017__05 +2010_003037__05 +2010_003092__05 +2010_003199__05 +2010_003304__05 +2010_003329__05 +2010_003512__05 +2010_003538__05 +2010_003539__05 +2010_003625__05 +2010_003762__05 +2010_003779__05 +2010_003994__05 +2010_004054__05 +2010_004197__05 +2010_004288__05 +2010_004290__05 +2010_004344__05 +2010_004441__05 +2010_004447__05 +2010_004460__05 +2010_004488__05 +2010_004665__05 +2010_004680__05 +2010_004751__05 +2010_004786__05 +2010_004866__05 +2010_005133__05 +2010_005155__05 +2010_005379__05 +2010_005409__05 +2010_005419__05 +2010_005441__05 +2010_005512__05 +2010_005663__05 +2010_005853__05 +2010_005865__05 +2010_005883__05 +2010_005952__05 +2010_005954__05 +2010_006000__05 +2010_006015__05 +2010_006021__05 +2011_000027__05 +2011_000206__05 +2011_000257__05 +2011_000344__05 +2011_000361__05 +2011_000375__05 +2011_000622__05 +2011_000713__05 +2011_000847__05 +2011_000920__05 +2011_000951__05 +2011_000996__05 +2011_001008__05 +2011_001168__05 +2011_001272__05 +2011_001295__05 +2011_001305__05 +2011_001327__05 +2011_001382__05 +2011_001521__05 +2011_001571__05 +2011_001700__05 +2011_001720__05 +2011_001820__05 +2011_002044__05 +2011_002088__05 +2011_002113__05 +2011_002131__05 +2011_002158__05 +2011_002163__05 +2011_002185__05 +2011_002301__05 +2011_002318__05 +2011_002324__05 +2011_002387__05 +2011_002433__05 +2011_002559__05 +2011_002560__05 +2011_002590__05 +2011_002890__05 +2011_002911__05 +2011_002920__05 +2011_002940__05 +2011_002966__05 +2011_002967__05 +2011_002974__05 +2011_003010__05 +2011_003158__05 +2011_003262__05 diff --git a/submodules/lang_seg/fewshot_data/data/splits/pascal/trn/fold1.txt b/submodules/lang_seg/fewshot_data/data/splits/pascal/trn/fold1.txt new file mode 100644 index 0000000000000000000000000000000000000000..3405a5c1b99f4c1a7e3377e1131f05e7b70a15da --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/pascal/trn/fold1.txt @@ -0,0 +1,3425 @@ +2007_000648__06 +2007_000768__06 +2007_001595__06 +2007_002024__06 +2007_002216__06 +2007_003715__06 +2007_004065__06 +2007_005262__06 +2007_006317__06 +2007_007003__06 +2007_007398__06 +2007_007480__06 +2007_007591__06 +2007_007908__06 +2007_009832__06 +2008_000032__06 +2008_000132__06 +2008_000238__06 +2008_000252__06 +2008_000982__06 +2008_001022__06 +2008_001820__06 +2008_001842__06 +2008_002179__06 +2008_002412__06 +2008_002451__06 +2008_003060__06 +2008_003067__06 +2008_003266__06 +2008_003302__06 +2008_003321__06 +2008_003373__06 +2008_003381__06 +2008_003489__06 +2008_003691__06 +2008_003811__06 +2008_003924__06 +2008_004055__06 +2008_004088__06 +2008_004613__06 +2008_004614__06 +2008_004679__06 +2008_004725__06 +2008_004794__06 +2008_004844__06 +2008_004866__06 +2008_004976__06 +2008_005074__06 +2008_005139__06 +2008_005196__06 +2008_005266__06 +2008_005277__06 +2008_005350__06 +2008_005360__06 +2008_005720__06 +2008_005761__06 +2008_005878__06 +2008_005891__06 +2008_005933__06 +2008_005984__06 +2008_006359__06 +2008_006483__06 +2008_006530__06 +2008_006635__06 +2008_006748__06 +2008_007056__06 +2008_007254__06 +2008_007352__06 +2008_007356__06 +2008_007375__06 +2008_007488__06 +2008_007515__06 +2008_007635__06 +2008_007641__06 +2008_007872__06 +2008_007931__06 +2008_007969__06 +2008_007997__06 +2008_008066__06 +2008_008080__06 +2008_008097__06 +2008_008113__06 +2008_008120__06 +2008_008234__06 +2008_008281__06 +2008_008343__06 +2008_008549__06 +2008_008598__06 +2008_008674__06 +2008_008683__06 +2008_008684__06 +2008_008701__06 +2008_008705__06 +2008_008706__06 +2008_008719__06 +2008_008748__06 +2009_000006__06 +2009_000027__06 +2009_000054__06 +2009_000060__06 +2009_000089__06 +2009_000097__06 +2009_000104__06 +2009_000135__06 +2009_000140__06 +2009_000176__06 +2009_000199__06 +2009_000203__06 +2009_000239__06 +2009_000405__06 +2009_000408__06 +2009_000519__06 +2009_000532__06 +2009_000557__06 +2009_000558__06 +2009_000568__06 +2009_000617__06 +2009_000648__06 +2009_000708__06 +2009_000886__06 +2009_000945__06 +2009_000960__06 +2009_000990__06 +2009_001044__06 +2009_001083__06 +2009_001091__06 +2009_001098__06 +2009_001113__06 +2009_001138__06 +2009_001148__06 +2009_001159__06 +2009_001197__06 +2009_001237__06 +2009_001326__06 +2009_001354__06 +2009_001366__06 +2009_001472__06 +2009_001590__06 +2009_001614__06 +2009_001633__06 +2009_001674__06 +2009_001675__06 +2009_001707__06 +2009_001734__06 +2009_001752__06 +2009_001811__06 +2009_001822__06 +2009_001847__06 +2009_001897__06 +2009_001910__06 +2009_001967__06 +2009_002052__06 +2009_002057__06 +2009_002066__06 +2009_002105__06 +2009_002126__06 +2009_002128__06 +2009_002152__06 +2009_002203__06 +2009_002271__06 +2009_002348__06 +2009_002398__06 +2009_002436__06 +2009_002457__06 +2009_002505__06 +2009_002512__06 +2009_002517__06 +2009_002532__06 +2009_002566__06 +2009_002570__06 +2009_002621__06 +2009_002626__06 +2009_002843__06 +2009_002853__06 +2009_002876__06 +2009_002933__06 +2009_003018__06 +2009_003053__06 +2009_003067__06 +2009_003200__06 +2009_003276__06 +2009_003383__06 +2009_003385__06 +2009_003422__06 +2009_003447__06 +2009_003458__06 +2009_003461__06 +2009_003491__06 +2009_003555__06 +2009_003566__06 +2009_003592__06 +2009_003614__06 +2009_003636__06 +2009_003669__06 +2009_003710__06 +2009_003757__06 +2009_003781__06 +2009_003916__06 +2009_003969__06 +2009_004018__06 +2009_004037__06 +2009_004040__06 +2009_004052__06 +2009_004055__06 +2009_004187__06 +2009_004197__06 +2009_004231__06 +2009_004258__06 +2009_004276__06 +2009_004290__06 +2009_004295__06 +2009_004328__06 +2009_004350__06 +2009_004369__06 +2009_004374__06 +2009_004383__06 +2009_004417__06 +2009_004440__06 +2009_004567__06 +2009_004667__06 +2009_004670__06 +2009_004681__06 +2009_004829__06 +2009_004836__06 +2009_004871__06 +2009_004947__06 +2009_004962__06 +2009_005060__06 +2009_005168__06 +2009_005201__06 +2009_005279__06 +2009_005293__06 +2010_000069__06 +2010_000148__06 +2010_000165__06 +2010_000187__06 +2010_000376__06 +2010_000480__06 +2010_000524__06 +2010_000603__06 +2010_000689__06 +2010_000726__06 +2010_000770__06 +2010_000797__06 +2010_000855__06 +2010_000947__06 +2010_001087__06 +2010_001204__06 +2010_001288__06 +2010_001364__06 +2010_001410__06 +2010_001453__06 +2010_001458__06 +2010_001630__06 +2010_001640__06 +2010_001650__06 +2010_001732__06 +2010_001771__06 +2010_001808__06 +2010_001892__06 +2010_001982__06 +2010_002102__06 +2010_002203__06 +2010_002218__06 +2010_002263__06 +2010_002356__06 +2010_002413__06 +2010_002438__06 +2010_002758__06 +2010_002774__06 +2010_002978__06 +2010_003040__06 +2010_003057__06 +2010_003173__06 +2010_003179__06 +2010_003534__06 +2010_003562__06 +2010_003635__06 +2010_003855__06 +2010_003877__06 +2010_003893__06 +2010_003936__06 +2010_004180__06 +2010_004263__06 +2010_004367__06 +2010_004554__06 +2010_004558__06 +2010_004588__06 +2010_004594__06 +2010_004616__06 +2010_004625__06 +2010_004627__06 +2010_004637__06 +2010_004696__06 +2010_004735__06 +2010_004806__06 +2010_004844__06 +2010_004903__06 +2010_004971__06 +2010_004997__06 +2010_005041__06 +2010_005080__06 +2010_005123__06 +2010_005277__06 +2010_005389__06 +2010_005415__06 +2010_005457__06 +2010_005502__06 +2010_005527__06 +2010_005596__06 +2010_005636__06 +2010_005756__06 +2010_005959__06 +2011_000007__06 +2011_000025__06 +2011_000048__06 +2011_000090__06 +2011_000095__06 +2011_000138__06 +2011_000181__06 +2011_000208__06 +2011_000258__06 +2011_000267__06 +2011_000369__06 +2011_000541__06 +2011_000551__06 +2011_000559__06 +2011_000703__06 +2011_000767__06 +2011_000804__06 +2011_000901__06 +2011_001015__06 +2011_001105__06 +2011_001146__06 +2011_001227__06 +2011_001255__06 +2011_001270__06 +2011_001355__06 +2011_001411__06 +2011_001479__06 +2011_001519__06 +2011_001536__06 +2011_001747__06 +2011_001826__06 +2011_001870__06 +2011_001949__06 +2011_001964__06 +2011_002046__06 +2011_002079__06 +2011_002215__06 +2011_002227__06 +2011_002330__06 +2011_002384__06 +2011_002457__06 +2011_002460__06 +2011_002629__06 +2011_002687__06 +2011_002780__06 +2011_002826__06 +2011_002925__06 +2011_002932__06 +2011_002944__06 +2011_002956__06 +2011_003086__06 +2011_003089__06 +2011_003109__06 +2011_003247__06 +2007_001857__07 +2007_002281__07 +2007_003815__07 +2007_004481__07 +2007_004705__07 +2007_004810__07 +2007_004830__07 +2007_005262__07 +2007_005273__07 +2007_005988__07 +2007_006151__07 +2007_006660__07 +2007_006900__07 +2007_007003__07 +2007_007591__07 +2007_007948__07 +2007_008801__07 +2007_009052__07 +2007_009322__07 +2007_009779__07 +2008_000027__07 +2008_000028__07 +2008_000052__07 +2008_000074__07 +2008_000085__07 +2008_000105__07 +2008_000109__07 +2008_000133__07 +2008_000143__07 +2008_000163__07 +2008_000174__07 +2008_000176__07 +2008_000185__07 +2008_000187__07 +2008_000189__07 +2008_000199__07 +2008_000203__07 +2008_000226__07 +2008_000237__07 +2008_000251__07 +2008_000260__07 +2008_000281__07 +2008_000304__07 +2008_000346__07 +2008_000399__07 +2008_000488__07 +2008_000531__07 +2008_000562__07 +2008_000563__07 +2008_000595__07 +2008_000599__07 +2008_000613__07 +2008_000619__07 +2008_000719__07 +2008_000828__07 +2008_000833__07 +2008_000880__07 +2008_000884__07 +2008_000939__07 +2008_000944__07 +2008_000952__07 +2008_000953__07 +2008_000959__07 +2008_000979__07 +2008_000982__07 +2008_001007__07 +2008_001042__07 +2008_001104__07 +2008_001142__07 +2008_001196__07 +2008_001208__07 +2008_001262__07 +2008_001274__07 +2008_001329__07 +2008_001359__07 +2008_001375__07 +2008_001440__07 +2008_001445__07 +2008_001446__07 +2008_001533__07 +2008_001632__07 +2008_001681__07 +2008_001716__07 +2008_001717__07 +2008_001746__07 +2008_001830__07 +2008_001867__07 +2008_001941__07 +2008_001961__07 +2008_002062__07 +2008_002118__07 +2008_002197__07 +2008_002198__07 +2008_002199__07 +2008_002229__07 +2008_002258__07 +2008_002322__07 +2008_002327__07 +2008_002366__07 +2008_002368__07 +2008_002401__07 +2008_002442__07 +2008_002444__07 +2008_002461__07 +2008_002466__07 +2008_002473__07 +2008_002483__07 +2008_002555__07 +2008_002643__07 +2008_002704__07 +2008_002705__07 +2008_002710__07 +2008_002741__07 +2008_002746__07 +2008_002834__07 +2008_002857__07 +2008_002875__07 +2008_002890__07 +2008_002891__07 +2008_002943__07 +2008_002977__07 +2008_003005__07 +2008_003021__07 +2008_003022__07 +2008_003030__07 +2008_003057__07 +2008_003060__07 +2008_003061__07 +2008_003082__07 +2008_003132__07 +2008_003140__07 +2008_003202__07 +2008_003255__07 +2008_003300__07 +2008_003321__07 +2008_003329__07 +2008_003378__07 +2008_003405__07 +2008_003415__07 +2008_003452__07 +2008_003479__07 +2008_003582__07 +2008_003647__07 +2008_003673__07 +2008_003680__07 +2008_003685__07 +2008_003719__07 +2008_003755__07 +2008_003764__07 +2008_003791__07 +2008_003794__07 +2008_003811__07 +2008_003844__07 +2008_003854__07 +2008_003860__07 +2008_003868__07 +2008_003888__07 +2008_003943__07 +2008_003970__07 +2008_003986__07 +2008_004020__07 +2008_004046__07 +2008_004080__07 +2008_004102__07 +2008_004112__07 +2008_004127__07 +2008_004142__07 +2008_004224__07 +2008_004231__07 +2008_004288__07 +2008_004307__07 +2008_004312__07 +2008_004326__07 +2008_004387__07 +2008_004411__07 +2008_004414__07 +2008_004441__07 +2008_004458__07 +2008_004464__07 +2008_004482__07 +2008_004488__07 +2008_004515__07 +2008_004549__07 +2008_004559__07 +2008_004574__07 +2008_004614__07 +2008_004668__07 +2008_004678__07 +2008_004684__07 +2008_004711__07 +2008_004713__07 +2008_004729__07 +2008_004832__07 +2008_004850__07 +2008_004858__07 +2008_004862__07 +2008_004872__07 +2008_004876__07 +2008_004899__07 +2008_004945__07 +2008_004955__07 +2008_004974__07 +2008_005032__07 +2008_005045__07 +2008_005054__07 +2008_005074__07 +2008_005101__07 +2008_005110__07 +2008_005127__07 +2008_005139__07 +2008_005159__07 +2008_005171__07 +2008_005216__07 +2008_005218__07 +2008_005220__07 +2008_005234__07 +2008_005283__07 +2008_005294__07 +2008_005316__07 +2008_005319__07 +2008_005325__07 +2008_005336__07 +2008_005349__07 +2008_005356__07 +2008_005378__07 +2008_005404__07 +2008_005405__07 +2008_005472__07 +2008_005548__07 +2008_005572__07 +2008_005593__07 +2008_005612__07 +2008_005638__07 +2008_005641__07 +2008_005663__07 +2008_005686__07 +2008_005707__07 +2008_005747__07 +2008_005761__07 +2008_005768__07 +2008_005822__07 +2008_005832__07 +2008_005865__07 +2008_005875__07 +2008_005891__07 +2008_005893__07 +2008_005902__07 +2008_005907__07 +2008_005936__07 +2008_005960__07 +2008_005972__07 +2008_005989__07 +2008_006017__07 +2008_006082__07 +2008_006092__07 +2008_006181__07 +2008_006188__07 +2008_006215__07 +2008_006220__07 +2008_006222__07 +2008_006224__07 +2008_006225__07 +2008_006240__07 +2008_006257__07 +2008_006272__07 +2008_006316__07 +2008_006336__07 +2008_006382__07 +2008_006392__07 +2008_006417__07 +2008_006438__07 +2008_006475__07 +2008_006483__07 +2008_006538__07 +2008_006562__07 +2008_006634__07 +2008_006638__07 +2008_006642__07 +2008_006649__07 +2008_006691__07 +2008_006694__07 +2008_006714__07 +2008_006718__07 +2008_006762__07 +2008_006774__07 +2008_006800__07 +2008_006819__07 +2008_006827__07 +2008_006833__07 +2008_006844__07 +2008_006872__07 +2008_006885__07 +2008_006907__07 +2008_006921__07 +2008_006936__07 +2008_006959__07 +2008_006961__07 +2008_006989__07 +2008_006992__07 +2008_007034__07 +2008_007060__07 +2008_007064__07 +2008_007090__07 +2008_007119__07 +2008_007138__07 +2008_007167__07 +2008_007171__07 +2008_007214__07 +2008_007221__07 +2008_007241__07 +2008_007264__07 +2008_007295__07 +2008_007311__07 +2008_007314__07 +2008_007321__07 +2008_007352__07 +2008_007364__07 +2008_007389__07 +2008_007409__07 +2008_007431__07 +2008_007456__07 +2008_007466__07 +2008_007480__07 +2008_007500__07 +2008_007529__07 +2008_007558__07 +2008_007573__07 +2008_007595__07 +2008_007611__07 +2008_007660__07 +2008_007698__07 +2008_007716__07 +2008_007739__07 +2008_007757__07 +2008_007786__07 +2008_007791__07 +2008_007793__07 +2008_007825__07 +2008_007869__07 +2008_007872__07 +2008_007877__07 +2008_007895__07 +2008_007947__07 +2008_007981__07 +2008_007997__07 +2008_008024__07 +2008_008097__07 +2008_008177__07 +2008_008210__07 +2008_008246__07 +2008_008262__07 +2008_008315__07 +2008_008338__07 +2008_008343__07 +2008_008357__07 +2008_008416__07 +2008_008432__07 +2008_008501__07 +2008_008679__07 +2008_008681__07 +2008_008701__07 +2009_000021__07 +2009_000027__07 +2009_000042__07 +2009_000078__07 +2009_000089__07 +2009_000097__07 +2009_000140__07 +2009_000160__07 +2009_000165__07 +2009_000169__07 +2009_000225__07 +2009_000288__07 +2009_000390__07 +2009_000397__07 +2009_000399__07 +2009_000402__07 +2009_000461__07 +2009_000522__07 +2009_000542__07 +2009_000547__07 +2009_000591__07 +2009_000602__07 +2009_000631__07 +2009_000683__07 +2009_000720__07 +2009_000726__07 +2009_000737__07 +2009_000741__07 +2009_000755__07 +2009_000777__07 +2009_000821__07 +2009_000886__07 +2009_000889__07 +2009_000906__07 +2009_000948__07 +2009_001078__07 +2009_001097__07 +2009_001197__07 +2009_001216__07 +2009_001237__07 +2009_001306__07 +2009_001374__07 +2009_001388__07 +2009_001463__07 +2009_001519__07 +2009_001546__07 +2009_001585__07 +2009_001591__07 +2009_001612__07 +2009_001675__07 +2009_001677__07 +2009_001678__07 +2009_001744__07 +2009_001767__07 +2009_001782__07 +2009_001811__07 +2009_001861__07 +2009_001898__07 +2009_001917__07 +2009_001945__07 +2009_001984__07 +2009_001988__07 +2009_002000__07 +2009_002010__07 +2009_002057__07 +2009_002058__07 +2009_002073__07 +2009_002103__07 +2009_002117__07 +2009_002120__07 +2009_002126__07 +2009_002128__07 +2009_002149__07 +2009_002267__07 +2009_002271__07 +2009_002299__07 +2009_002312__07 +2009_002343__07 +2009_002380__07 +2009_002387__07 +2009_002424__07 +2009_002517__07 +2009_002524__07 +2009_002621__07 +2009_002632__07 +2009_002695__07 +2009_002711__07 +2009_002744__07 +2009_002777__07 +2009_002784__07 +2009_002806__07 +2009_002824__07 +2009_002862__07 +2009_002894__07 +2009_002960__07 +2009_002971__07 +2009_003020__07 +2009_003031__07 +2009_003058__07 +2009_003091__07 +2009_003132__07 +2009_003138__07 +2009_003155__07 +2009_003183__07 +2009_003212__07 +2009_003238__07 +2009_003255__07 +2009_003267__07 +2009_003310__07 +2009_003360__07 +2009_003376__07 +2009_003416__07 +2009_003447__07 +2009_003500__07 +2009_003633__07 +2009_003709__07 +2009_003722__07 +2009_003725__07 +2009_003743__07 +2009_003781__07 +2009_003801__07 +2009_003802__07 +2009_003814__07 +2009_003838__07 +2009_003879__07 +2009_003905__07 +2009_003974__07 +2009_003977__07 +2009_004018__07 +2009_004096__07 +2009_004102__07 +2009_004133__07 +2009_004171__07 +2009_004175__07 +2009_004179__07 +2009_004202__07 +2009_004224__07 +2009_004232__07 +2009_004249__07 +2009_004272__07 +2009_004303__07 +2009_004327__07 +2009_004340__07 +2009_004364__07 +2009_004370__07 +2009_004436__07 +2009_004471__07 +2009_004532__07 +2009_004556__07 +2009_004572__07 +2009_004630__07 +2009_004694__07 +2009_004731__07 +2009_004798__07 +2009_004822__07 +2009_004824__07 +2009_004903__07 +2009_004913__07 +2009_004933__07 +2009_004986__07 +2009_005025__07 +2009_005085__07 +2009_005086__07 +2009_005104__07 +2009_005133__07 +2009_005154__07 +2009_005242__07 +2009_005293__07 +2009_005294__07 +2010_000031__07 +2010_000036__07 +2010_000043__07 +2010_000073__07 +2010_000198__07 +2010_000211__07 +2010_000246__07 +2010_000248__07 +2010_000264__07 +2010_000279__07 +2010_000302__07 +2010_000336__07 +2010_000381__07 +2010_000394__07 +2010_000466__07 +2010_000480__07 +2010_000493__07 +2010_000503__07 +2010_000508__07 +2010_000522__07 +2010_000526__07 +2010_000556__07 +2010_000571__07 +2010_000582__07 +2010_000590__07 +2010_000604__07 +2010_000645__07 +2010_000664__07 +2010_000689__07 +2010_000711__07 +2010_000722__07 +2010_000723__07 +2010_000908__07 +2010_000922__07 +2010_000928__07 +2010_001043__07 +2010_001044__07 +2010_001080__07 +2010_001110__07 +2010_001117__07 +2010_001118__07 +2010_001121__07 +2010_001152__07 +2010_001163__07 +2010_001185__07 +2010_001220__07 +2010_001275__07 +2010_001315__07 +2010_001321__07 +2010_001337__07 +2010_001360__07 +2010_001394__07 +2010_001431__07 +2010_001449__07 +2010_001478__07 +2010_001536__07 +2010_001550__07 +2010_001561__07 +2010_001572__07 +2010_001638__07 +2010_001668__07 +2010_001685__07 +2010_001757__07 +2010_001821__07 +2010_001828__07 +2010_001850__07 +2010_001893__07 +2010_001948__07 +2010_001954__07 +2010_001981__07 +2010_001993__07 +2010_001998__07 +2010_002044__07 +2010_002046__07 +2010_002048__07 +2010_002107__07 +2010_002132__07 +2010_002175__07 +2010_002185__07 +2010_002194__07 +2010_002236__07 +2010_002279__07 +2010_002363__07 +2010_002368__07 +2010_002387__07 +2010_002409__07 +2010_002424__07 +2010_002457__07 +2010_002507__07 +2010_002516__07 +2010_002578__07 +2010_002592__07 +2010_002598__07 +2010_002647__07 +2010_002702__07 +2010_002783__07 +2010_002790__07 +2010_002807__07 +2010_002841__07 +2010_002844__07 +2010_002865__07 +2010_002903__07 +2010_002948__07 +2010_002985__07 +2010_003024__07 +2010_003047__07 +2010_003057__07 +2010_003120__07 +2010_003129__07 +2010_003139__07 +2010_003191__07 +2010_003206__07 +2010_003212__07 +2010_003219__07 +2010_003256__07 +2010_003264__07 +2010_003290__07 +2010_003367__07 +2010_003385__07 +2010_003400__07 +2010_003415__07 +2010_003432__07 +2010_003450__07 +2010_003478__07 +2010_003481__07 +2010_003537__07 +2010_003561__07 +2010_003574__07 +2010_003585__07 +2010_003612__07 +2010_003635__07 +2010_003645__07 +2010_003656__07 +2010_003664__07 +2010_003729__07 +2010_003730__07 +2010_003734__07 +2010_003789__07 +2010_003807__07 +2010_003893__07 +2010_003931__07 +2010_003981__07 +2010_004005__07 +2010_004014__07 +2010_004021__07 +2010_004027__07 +2010_004045__07 +2010_004059__07 +2010_004096__07 +2010_004116__07 +2010_004139__07 +2010_004160__07 +2010_004172__07 +2010_004186__07 +2010_004239__07 +2010_004258__07 +2010_004291__07 +2010_004333__07 +2010_004349__07 +2010_004373__07 +2010_004390__07 +2010_004422__07 +2010_004459__07 +2010_004484__07 +2010_004533__07 +2010_004557__07 +2010_004569__07 +2010_004577__07 +2010_004581__07 +2010_004594__07 +2010_004601__07 +2010_004618__07 +2010_004646__07 +2010_004654__07 +2010_004661__07 +2010_004686__07 +2010_004690__07 +2010_004692__07 +2010_004747__07 +2010_004782__07 +2010_004785__07 +2010_004824__07 +2010_004844__07 +2010_004854__07 +2010_004865__07 +2010_004877__07 +2010_004890__07 +2010_004930__07 +2010_004995__07 +2010_005041__07 +2010_005075__07 +2010_005087__07 +2010_005119__07 +2010_005147__07 +2010_005167__07 +2010_005236__07 +2010_005261__07 +2010_005318__07 +2010_005323__07 +2010_005377__07 +2010_005409__07 +2010_005414__07 +2010_005437__07 +2010_005474__07 +2010_005483__07 +2010_005518__07 +2010_005546__07 +2010_005571__07 +2010_005587__07 +2010_005663__07 +2010_005672__07 +2010_005700__07 +2010_005733__07 +2010_005750__07 +2010_005752__07 +2010_005785__07 +2010_005804__07 +2010_005817__07 +2010_005836__07 +2010_005845__07 +2010_005868__07 +2010_005875__07 +2010_005891__07 +2010_005907__07 +2010_005949__07 +2010_005951__07 +2010_005959__07 +2010_005987__07 +2010_006009__07 +2010_006012__07 +2010_006033__07 +2011_000025__07 +2011_000076__07 +2011_000086__07 +2011_000094__07 +2011_000095__07 +2011_000128__07 +2011_000176__07 +2011_000252__07 +2011_000258__07 +2011_000293__07 +2011_000416__07 +2011_000432__07 +2011_000496__07 +2011_000505__07 +2011_000509__07 +2011_000560__07 +2011_000569__07 +2011_000701__07 +2011_000724__07 +2011_000853__07 +2011_000887__07 +2011_000897__07 +2011_000954__07 +2011_001004__07 +2011_001062__07 +2011_001124__07 +2011_001136__07 +2011_001193__07 +2011_001227__07 +2011_001260__07 +2011_001261__07 +2011_001271__07 +2011_001286__07 +2011_001311__07 +2011_001455__07 +2011_001524__07 +2011_001542__07 +2011_001606__07 +2011_001632__07 +2011_001641__07 +2011_001673__07 +2011_001747__07 +2011_001799__07 +2011_001801__07 +2011_001826__07 +2011_001891__07 +2011_001901__07 +2011_001929__07 +2011_001949__07 +2011_001961__07 +2011_001982__07 +2011_002021__07 +2011_002033__07 +2011_002096__07 +2011_002106__07 +2011_002135__07 +2011_002143__07 +2011_002174__07 +2011_002215__07 +2011_002224__07 +2011_002234__07 +2011_002253__07 +2011_002303__07 +2011_002366__07 +2011_002384__07 +2011_002410__07 +2011_002420__07 +2011_002484__07 +2011_002526__07 +2011_002598__07 +2011_002676__07 +2011_002725__07 +2011_002782__07 +2011_002790__07 +2011_002796__07 +2011_002925__07 +2011_002956__07 +2011_002962__07 +2011_003016__07 +2011_003029__07 +2011_003065__07 +2011_003073__07 +2011_003109__07 +2011_003121__07 +2011_003185__07 +2011_003192__07 +2011_003269__07 +2007_000528__08 +2007_000549__08 +2007_000876__08 +2007_001185__08 +2007_001825__08 +2007_002760__08 +2007_003525__08 +2007_003778__08 +2007_003788__08 +2007_004998__08 +2007_005688__08 +2007_006303__08 +2007_006641__08 +2007_007098__08 +2007_007530__08 +2007_007878__08 +2007_008403__08 +2007_008575__08 +2007_009216__08 +2007_009464__08 +2007_009630__08 +2007_009724__08 +2008_000056__08 +2008_000060__08 +2008_000062__08 +2008_000096__08 +2008_000112__08 +2008_000115__08 +2008_000116__08 +2008_000181__08 +2008_000196__08 +2008_000222__08 +2008_000227__08 +2008_000306__08 +2008_000358__08 +2008_000502__08 +2008_000536__08 +2008_000581__08 +2008_000641__08 +2008_000660__08 +2008_000670__08 +2008_000724__08 +2008_000793__08 +2008_000824__08 +2008_000839__08 +2008_000847__08 +2008_000860__08 +2008_000950__08 +2008_000999__08 +2008_001004__08 +2008_001071__08 +2008_001111__08 +2008_001210__08 +2008_001267__08 +2008_001290__08 +2008_001335__08 +2008_001350__08 +2008_001357__08 +2008_001374__08 +2008_001414__08 +2008_001592__08 +2008_001610__08 +2008_001708__08 +2008_001741__08 +2008_001792__08 +2008_001836__08 +2008_001876__08 +2008_001980__08 +2008_002004__08 +2008_002045__08 +2008_002067__08 +2008_002177__08 +2008_002182__08 +2008_002201__08 +2008_002215__08 +2008_002225__08 +2008_002234__08 +2008_002293__08 +2008_002294__08 +2008_002298__08 +2008_002299__08 +2008_002329__08 +2008_002410__08 +2008_002583__08 +2008_002597__08 +2008_002668__08 +2008_002682__08 +2008_002749__08 +2008_002766__08 +2008_002793__08 +2008_002845__08 +2008_002872__08 +2008_002899__08 +2008_002957__08 +2008_003045__08 +2008_003048__08 +2008_003052__08 +2008_003063__08 +2008_003088__08 +2008_003104__08 +2008_003114__08 +2008_003167__08 +2008_003213__08 +2008_003228__08 +2008_003231__08 +2008_003239__08 +2008_003244__08 +2008_003303__08 +2008_003343__08 +2008_003386__08 +2008_003420__08 +2008_003430__08 +2008_003501__08 +2008_003519__08 +2008_003557__08 +2008_003559__08 +2008_003562__08 +2008_003607__08 +2008_003622__08 +2008_003624__08 +2008_003659__08 +2008_003704__08 +2008_003726__08 +2008_003772__08 +2008_003776__08 +2008_003841__08 +2008_003932__08 +2008_003974__08 +2008_004006__08 +2008_004022__08 +2008_004048__08 +2008_004074__08 +2008_004119__08 +2008_004120__08 +2008_004155__08 +2008_004189__08 +2008_004232__08 +2008_004257__08 +2008_004284__08 +2008_004290__08 +2008_004303__08 +2008_004328__08 +2008_004347__08 +2008_004374__08 +2008_004422__08 +2008_004462__08 +2008_004490__08 +2008_004515__08 +2008_004538__08 +2008_004540__08 +2008_004579__08 +2008_004583__08 +2008_004635__08 +2008_004640__08 +2008_004672__08 +2008_004688__08 +2008_004732__08 +2008_004736__08 +2008_004767__08 +2008_004797__08 +2008_004856__08 +2008_004873__08 +2008_004933__08 +2008_004966__08 +2008_004967__08 +2008_004990__08 +2008_005003__08 +2008_005013__08 +2008_005061__08 +2008_005064__08 +2008_005072__08 +2008_005088__08 +2008_005109__08 +2008_005134__08 +2008_005181__08 +2008_005193__08 +2008_005235__08 +2008_005247__08 +2008_005252__08 +2008_005270__08 +2008_005281__08 +2008_005288__08 +2008_005300__08 +2008_005309__08 +2008_005331__08 +2008_005337__08 +2008_005376__08 +2008_005380__08 +2008_005386__08 +2008_005421__08 +2008_005449__08 +2008_005460__08 +2008_005463__08 +2008_005469__08 +2008_005480__08 +2008_005496__08 +2008_005510__08 +2008_005523__08 +2008_005550__08 +2008_005560__08 +2008_005566__08 +2008_005569__08 +2008_005574__08 +2008_005600__08 +2008_005614__08 +2008_005636__08 +2008_005657__08 +2008_005673__08 +2008_005699__08 +2008_005716__08 +2008_005735__08 +2008_005770__08 +2008_005777__08 +2008_005780__08 +2008_005805__08 +2008_005816__08 +2008_005853__08 +2008_005857__08 +2008_005882__08 +2008_005890__08 +2008_005943__08 +2008_005977__08 +2008_005979__08 +2008_006050__08 +2008_006068__08 +2008_006081__08 +2008_006099__08 +2008_006113__08 +2008_006135__08 +2008_006175__08 +2008_006190__08 +2008_006194__08 +2008_006218__08 +2008_006280__08 +2008_006285__08 +2008_006294__08 +2008_006307__08 +2008_006329__08 +2008_006347__08 +2008_006351__08 +2008_006362__08 +2008_006377__08 +2008_006384__08 +2008_006400__08 +2008_006403__08 +2008_006430__08 +2008_006449__08 +2008_006462__08 +2008_006512__08 +2008_006520__08 +2008_006561__08 +2008_006576__08 +2008_006599__08 +2008_006609__08 +2008_006629__08 +2008_006656__08 +2008_006657__08 +2008_006728__08 +2008_006746__08 +2008_006765__08 +2008_006781__08 +2008_006785__08 +2008_006793__08 +2008_006810__08 +2008_006817__08 +2008_006843__08 +2008_006863__08 +2008_006868__08 +2008_006910__08 +2008_006926__08 +2008_006956__08 +2008_006962__08 +2008_006967__08 +2008_006973__08 +2008_006999__08 +2008_007039__08 +2008_007059__08 +2008_007082__08 +2008_007085__08 +2008_007086__08 +2008_007106__08 +2008_007130__08 +2008_007151__08 +2008_007164__08 +2008_007165__08 +2008_007176__08 +2008_007187__08 +2008_007216__08 +2008_007239__08 +2008_007256__08 +2008_007260__08 +2008_007289__08 +2008_007324__08 +2008_007327__08 +2008_007353__08 +2008_007363__08 +2008_007403__08 +2008_007404__08 +2008_007469__08 +2008_007472__08 +2008_007494__08 +2008_007496__08 +2008_007610__08 +2008_007613__08 +2008_007630__08 +2008_007632__08 +2008_007640__08 +2008_007662__08 +2008_007683__08 +2008_007702__08 +2008_007726__08 +2008_007736__08 +2008_007777__08 +2008_007794__08 +2008_007842__08 +2008_007855__08 +2008_007873__08 +2008_007888__08 +2008_007913__08 +2008_007914__08 +2008_007941__08 +2008_007964__08 +2008_007975__08 +2008_007977__08 +2008_007989__08 +2008_008585__08 +2008_008590__08 +2009_000002__08 +2009_000030__08 +2009_000132__08 +2009_000150__08 +2009_000227__08 +2009_000249__08 +2009_000284__08 +2009_000303__08 +2009_000304__08 +2009_000330__08 +2009_000342__08 +2009_000350__08 +2009_000366__08 +2009_000438__08 +2009_000466__08 +2009_000493__08 +2009_000504__08 +2009_000511__08 +2009_000535__08 +2009_000553__08 +2009_000566__08 +2009_000599__08 +2009_000614__08 +2009_000638__08 +2009_000681__08 +2009_000684__08 +2009_000745__08 +2009_000782__08 +2009_000854__08 +2009_000862__08 +2009_000902__08 +2009_000915__08 +2009_000955__08 +2009_000966__08 +2009_000970__08 +2009_001016__08 +2009_001037__08 +2009_001057__08 +2009_001081__08 +2009_001094__08 +2009_001117__08 +2009_001126__08 +2009_001153__08 +2009_001198__08 +2009_001205__08 +2009_001227__08 +2009_001260__08 +2009_001270__08 +2009_001279__08 +2009_001305__08 +2009_001361__08 +2009_001369__08 +2009_001419__08 +2009_001444__08 +2009_001450__08 +2009_001484__08 +2009_001502__08 +2009_001522__08 +2009_001550__08 +2009_001562__08 +2009_001623__08 +2009_001689__08 +2009_001719__08 +2009_001835__08 +2009_001894__08 +2009_001906__08 +2009_001931__08 +2009_001933__08 +2009_001948__08 +2009_002008__08 +2009_002040__08 +2009_002053__08 +2009_002061__08 +2009_002104__08 +2009_002141__08 +2009_002176__08 +2009_002228__08 +2009_002311__08 +2009_002352__08 +2009_002363__08 +2009_002429__08 +2009_002439__08 +2009_002470__08 +2009_002499__08 +2009_002504__08 +2009_002561__08 +2009_002592__08 +2009_002615__08 +2009_002628__08 +2009_002667__08 +2009_002704__08 +2009_002739__08 +2009_002813__08 +2009_002836__08 +2009_002837__08 +2009_002855__08 +2009_002898__08 +2009_002917__08 +2009_002955__08 +2009_002961__08 +2009_002972__08 +2009_003013__08 +2009_003056__08 +2009_003070__08 +2009_003077__08 +2009_003118__08 +2009_003129__08 +2009_003143__08 +2009_003165__08 +2009_003201__08 +2009_003222__08 +2009_003312__08 +2009_003350__08 +2009_003380__08 +2009_003415__08 +2009_003419__08 +2009_003457__08 +2009_003489__08 +2009_003528__08 +2009_003541__08 +2009_003601__08 +2009_003605__08 +2009_003626__08 +2009_003647__08 +2009_003654__08 +2009_003655__08 +2009_003660__08 +2009_003663__08 +2009_003697__08 +2009_003698__08 +2009_003718__08 +2009_003783__08 +2009_003847__08 +2009_003867__08 +2009_003921__08 +2009_003955__08 +2009_003985__08 +2009_004022__08 +2009_004038__08 +2009_004051__08 +2009_004073__08 +2009_004083__08 +2009_004128__08 +2009_004150__08 +2009_004157__08 +2009_004176__08 +2009_004183__08 +2009_004210__08 +2009_004225__08 +2009_004234__08 +2009_004261__08 +2009_004291__08 +2009_004307__08 +2009_004358__08 +2009_004382__08 +2009_004394__08 +2009_004424__08 +2009_004434__08 +2009_004442__08 +2009_004448__08 +2009_004456__08 +2009_004477__08 +2009_004529__08 +2009_004598__08 +2009_004607__08 +2009_004639__08 +2009_004655__08 +2009_004683__08 +2009_004705__08 +2009_004779__08 +2009_004796__08 +2009_004813__08 +2009_004841__08 +2009_004877__08 +2009_004887__08 +2009_004940__08 +2009_004953__08 +2009_004971__08 +2009_004983__08 +2009_004999__08 +2009_005033__08 +2009_005037__08 +2009_005051__08 +2009_005095__08 +2009_005119__08 +2009_005160__08 +2009_005177__08 +2009_005211__08 +2009_005236__08 +2009_005251__08 +2009_005299__08 +2010_000001__08 +2010_000009__08 +2010_000043__08 +2010_000048__08 +2010_000054__08 +2010_000067__08 +2010_000099__08 +2010_000109__08 +2010_000114__08 +2010_000118__08 +2010_000157__08 +2010_000172__08 +2010_000175__08 +2010_000182__08 +2010_000218__08 +2010_000224__08 +2010_000244__08 +2010_000261__08 +2010_000269__08 +2010_000291__08 +2010_000303__08 +2010_000317__08 +2010_000347__08 +2010_000374__08 +2010_000389__08 +2010_000404__08 +2010_000409__08 +2010_000439__08 +2010_000442__08 +2010_000458__08 +2010_000468__08 +2010_000469__08 +2010_000488__08 +2010_000497__08 +2010_000500__08 +2010_000506__08 +2010_000519__08 +2010_000536__08 +2010_000538__08 +2010_000553__08 +2010_000576__08 +2010_000586__08 +2010_000616__08 +2010_000647__08 +2010_000661__08 +2010_000665__08 +2010_000675__08 +2010_000687__08 +2010_000692__08 +2010_000694__08 +2010_000702__08 +2010_000712__08 +2010_000737__08 +2010_000769__08 +2010_000791__08 +2010_000799__08 +2010_000815__08 +2010_000828__08 +2010_000842__08 +2010_000872__08 +2010_000875__08 +2010_000914__08 +2010_000948__08 +2010_000971__08 +2010_000974__08 +2010_000996__08 +2010_001012__08 +2010_001021__08 +2010_001025__08 +2010_001030__08 +2010_001054__08 +2010_001057__08 +2010_001089__08 +2010_001100__08 +2010_001103__08 +2010_001125__08 +2010_001140__08 +2010_001177__08 +2010_001193__08 +2010_001195__08 +2010_001215__08 +2010_001240__08 +2010_001247__08 +2010_001291__08 +2010_001328__08 +2010_001333__08 +2010_001344__08 +2010_001347__08 +2010_001361__08 +2010_001382__08 +2010_001385__08 +2010_001386__08 +2010_001401__08 +2010_001402__08 +2010_001417__08 +2010_001418__08 +2010_001435__08 +2010_001452__08 +2010_001457__08 +2010_001465__08 +2010_001468__08 +2010_001499__08 +2010_001516__08 +2010_001544__08 +2010_001555__08 +2010_001587__08 +2010_001590__08 +2010_001602__08 +2010_001637__08 +2010_001647__08 +2010_001660__08 +2010_001700__08 +2010_001712__08 +2010_001737__08 +2010_001744__08 +2010_001747__08 +2010_001754__08 +2010_001783__08 +2010_001787__08 +2010_001794__08 +2010_001827__08 +2010_001842__08 +2010_001863__08 +2010_001884__08 +2010_001885__08 +2010_001934__08 +2010_001938__08 +2010_001939__08 +2010_001957__08 +2010_001968__08 +2010_001994__08 +2010_002000__08 +2010_002026__08 +2010_002040__08 +2010_002057__08 +2010_002068__08 +2010_002086__08 +2010_002098__08 +2010_002118__08 +2010_002136__08 +2010_002138__08 +2010_002143__08 +2010_002167__08 +2010_002177__08 +2010_002192__08 +2010_002204__08 +2010_002224__08 +2010_002243__08 +2010_002255__08 +2010_002278__08 +2010_002286__08 +2010_002303__08 +2010_002307__08 +2010_002316__08 +2010_002333__08 +2010_002354__08 +2010_002371__08 +2010_002392__08 +2010_002406__08 +2010_002425__08 +2010_002448__08 +2010_002472__08 +2010_002479__08 +2010_002485__08 +2010_002504__08 +2010_002529__08 +2010_002533__08 +2010_002543__08 +2010_002553__08 +2010_002573__08 +2010_002583__08 +2010_002587__08 +2010_002594__08 +2010_002602__08 +2010_002625__08 +2010_002626__08 +2010_002645__08 +2010_002662__08 +2010_002676__08 +2010_002678__08 +2010_002692__08 +2010_002713__08 +2010_002723__08 +2010_002746__08 +2010_002771__08 +2010_002778__08 +2010_002789__08 +2010_002811__08 +2010_002831__08 +2010_002845__08 +2010_002855__08 +2010_002876__08 +2010_002891__08 +2010_002909__08 +2010_002917__08 +2010_002955__08 +2010_002965__08 +2010_002973__08 +2010_002976__08 +2010_002990__08 +2010_002993__08 +2010_003032__08 +2010_003050__08 +2010_003067__08 +2010_003106__08 +2010_003133__08 +2010_003151__08 +2010_003174__08 +2010_003190__08 +2010_003197__08 +2010_003223__08 +2010_003227__08 +2010_003230__08 +2010_003232__08 +2010_003238__08 +2010_003240__08 +2010_003249__08 +2010_003291__08 +2010_003299__08 +2010_003309__08 +2010_003358__08 +2010_003371__08 +2010_003421__08 +2010_003427__08 +2010_003435__08 +2010_003451__08 +2010_003467__08 +2010_003481__08 +2010_003483__08 +2010_003488__08 +2010_003497__08 +2010_003509__08 +2010_003527__08 +2010_003539__08 +2010_003551__08 +2010_003569__08 +2010_003579__08 +2010_003598__08 +2010_003641__08 +2010_003665__08 +2010_003672__08 +2010_003687__08 +2010_003696__08 +2010_003747__08 +2010_003752__08 +2010_003773__08 +2010_003800__08 +2010_003805__08 +2010_003818__08 +2010_003823__08 +2010_003845__08 +2010_003863__08 +2010_003875__08 +2010_003887__08 +2010_003898__08 +2010_003925__08 +2010_003928__08 +2010_003937__08 +2010_003943__08 +2010_003950__08 +2010_003966__08 +2010_003974__08 +2010_003976__08 +2010_003982__08 +2010_003988__08 +2010_004002__08 +2010_004007__08 +2010_004008__08 +2010_004023__08 +2010_004037__08 +2010_004048__08 +2010_004060__08 +2010_004133__08 +2010_004141__08 +2010_004144__08 +2010_004157__08 +2010_004171__08 +2010_004175__08 +2010_004197__08 +2010_004209__08 +2010_004244__08 +2010_004257__08 +2010_004271__08 +2010_004286__08 +2010_004301__08 +2010_004306__08 +2010_004327__08 +2010_004335__08 +2010_004346__08 +2010_004365__08 +2010_004402__08 +2010_004415__08 +2010_004425__08 +2010_004448__08 +2010_004457__08 +2010_004467__08 +2010_004501__08 +2010_004515__08 +2010_004518__08 +2010_004540__08 +2010_004545__08 +2010_004553__08 +2010_004573__08 +2010_004575__08 +2010_004584__08 +2010_004585__08 +2010_004621__08 +2010_004634__08 +2010_004638__08 +2010_004667__08 +2010_004676__08 +2010_004681__08 +2010_004691__08 +2010_004712__08 +2010_004717__08 +2010_004753__08 +2010_004760__08 +2010_004768__08 +2010_004808__08 +2010_004816__08 +2010_004822__08 +2010_004829__08 +2010_004841__08 +2010_004874__08 +2010_004891__08 +2010_004918__08 +2010_004933__08 +2010_004954__08 +2010_004963__08 +2010_004968__08 +2010_004973__08 +2010_005006__08 +2010_005017__08 +2010_005035__08 +2010_005048__08 +2010_005072__08 +2010_005082__08 +2010_005099__08 +2010_005115__08 +2010_005128__08 +2010_005143__08 +2010_005161__08 +2010_005185__08 +2010_005193__08 +2010_005202__08 +2010_005208__08 +2010_005216__08 +2010_005222__08 +2010_005230__08 +2010_005232__08 +2010_005238__08 +2010_005239__08 +2010_005250__08 +2010_005257__08 +2010_005264__08 +2010_005275__08 +2010_005287__08 +2010_005312__08 +2010_005317__08 +2010_005340__08 +2010_005386__08 +2010_005394__08 +2010_005408__08 +2010_005425__08 +2010_005442__08 +2010_005456__08 +2010_005462__08 +2010_005482__08 +2010_005492__08 +2010_005505__08 +2010_005513__08 +2010_005535__08 +2010_005551__08 +2010_005573__08 +2010_005592__08 +2010_005614__08 +2010_005627__08 +2010_005647__08 +2010_005652__08 +2010_005696__08 +2010_005697__08 +2010_005712__08 +2010_005732__08 +2010_005735__08 +2010_005763__08 +2010_005791__08 +2010_005796__08 +2010_005805__08 +2010_005806__08 +2010_005821__08 +2010_005833__08 +2010_005853__08 +2010_005855__08 +2010_005886__08 +2010_005892__08 +2010_005903__08 +2010_005906__08 +2010_005935__08 +2010_005938__08 +2010_005948__08 +2010_005973__08 +2010_005975__08 +2010_005984__08 +2010_006011__08 +2010_006025__08 +2010_006040__08 +2010_006058__08 +2010_006078__08 +2011_000044__08 +2011_000108__08 +2011_000162__08 +2011_000165__08 +2011_000229__08 +2011_000305__08 +2011_000345__08 +2011_000379__08 +2011_000426__08 +2011_000469__08 +2011_000596__08 +2011_000634__08 +2011_000704__08 +2011_000758__08 +2011_000769__08 +2011_000824__08 +2011_000855__08 +2011_000973__08 +2011_000999__08 +2011_001116__08 +2011_001128__08 +2011_001135__08 +2011_001173__08 +2011_001238__08 +2011_001264__08 +2011_001323__08 +2011_001369__08 +2011_001390__08 +2011_001414__08 +2011_001441__08 +2011_001568__08 +2011_001572__08 +2011_001620__08 +2011_001716__08 +2011_001733__08 +2011_001754__08 +2011_001841__08 +2011_001900__08 +2011_001911__08 +2011_001971__08 +2011_002004__08 +2011_002031__08 +2011_002034__08 +2011_002147__08 +2011_002186__08 +2011_002211__08 +2011_002276__08 +2011_002347__08 +2011_002389__08 +2011_002409__08 +2011_002461__08 +2011_002631__08 +2011_002664__08 +2011_002699__08 +2011_002724__08 +2011_002883__08 +2011_002916__08 +2011_003159__08 +2011_003166__08 +2011_003211__08 +2011_003216__08 +2007_000063__09 +2007_001027__09 +2007_001340__09 +2007_001439__09 +2007_002368__09 +2007_003205__09 +2007_003251__09 +2007_003541__09 +2007_003788__09 +2007_004166__09 +2007_005086__09 +2007_005266__09 +2007_005647__09 +2007_006004__09 +2007_006066__09 +2007_006477__09 +2007_006530__09 +2007_006605__09 +2007_007021__09 +2007_007432__09 +2007_007530__09 +2007_008072__09 +2007_008407__09 +2007_008468__09 +2007_008948__09 +2007_009435__09 +2007_009550__09 +2007_009649__09 +2008_000041__09 +2008_000043__09 +2008_000051__09 +2008_000067__09 +2008_000070__09 +2008_000089__09 +2008_000093__09 +2008_000115__09 +2008_000202__09 +2008_000227__09 +2008_000246__09 +2008_000273__09 +2008_000274__09 +2008_000297__09 +2008_000305__09 +2008_000381__09 +2008_000397__09 +2008_000418__09 +2008_000432__09 +2008_000473__09 +2008_000489__09 +2008_000495__09 +2008_000498__09 +2008_000572__09 +2008_000578__09 +2008_000588__09 +2008_000626__09 +2008_000690__09 +2008_000704__09 +2008_000760__09 +2008_000790__09 +2008_000817__09 +2008_000824__09 +2008_000835__09 +2008_000847__09 +2008_000870__09 +2008_000904__09 +2008_000914__09 +2008_000936__09 +2008_000941__09 +2008_001046__09 +2008_001083__09 +2008_001133__09 +2008_001143__09 +2008_001147__09 +2008_001320__09 +2008_001333__09 +2008_001434__09 +2008_001451__09 +2008_001467__09 +2008_001493__09 +2008_001503__09 +2008_001527__09 +2008_001563__09 +2008_001596__09 +2008_001613__09 +2008_001617__09 +2008_001624__09 +2008_001636__09 +2008_001666__09 +2008_001673__09 +2008_001694__09 +2008_001702__09 +2008_001723__09 +2008_001737__09 +2008_001751__09 +2008_001764__09 +2008_001772__09 +2008_001773__09 +2008_001783__09 +2008_001809__09 +2008_001841__09 +2008_001862__09 +2008_001910__09 +2008_001928__09 +2008_001997__09 +2008_002026__09 +2008_002071__09 +2008_002112__09 +2008_002114__09 +2008_002119__09 +2008_002148__09 +2008_002175__09 +2008_002218__09 +2008_002288__09 +2008_002311__09 +2008_002384__09 +2008_002403__09 +2008_002438__09 +2008_002442__09 +2008_002481__09 +2008_002485__09 +2008_002502__09 +2008_002508__09 +2008_002601__09 +2008_002613__09 +2008_002649__09 +2008_002674__09 +2008_002676__09 +2008_002776__09 +2008_002795__09 +2008_002869__09 +2008_002885__09 +2008_002903__09 +2008_002920__09 +2008_003073__09 +2008_003093__09 +2008_003136__09 +2008_003168__09 +2008_003178__09 +2008_003245__09 +2008_003252__09 +2008_003264__09 +2008_003265__09 +2008_003280__09 +2008_003290__09 +2008_003351__09 +2008_003407__09 +2008_003432__09 +2008_003448__09 +2008_003472__09 +2008_003501__09 +2008_003504__09 +2008_003547__09 +2008_003559__09 +2008_003667__09 +2008_003681__09 +2008_003689__09 +2008_003726__09 +2008_003753__09 +2008_003762__09 +2008_003774__09 +2008_003796__09 +2008_003838__09 +2008_003881__09 +2008_003904__09 +2008_003914__09 +2008_003967__09 +2008_003998__09 +2008_004003__09 +2008_004008__09 +2008_004071__09 +2008_004077__09 +2008_004097__09 +2008_004122__09 +2008_004124__09 +2008_004208__09 +2008_004280__09 +2008_004289__09 +2008_004297__09 +2008_004313__09 +2008_004344__09 +2008_004384__09 +2008_004408__09 +2008_004412__09 +2008_004435__09 +2008_004445__09 +2008_004457__09 +2008_004460__09 +2008_004487__09 +2008_004490__09 +2008_004492__09 +2008_004506__09 +2008_004553__09 +2008_004564__09 +2008_004579__09 +2008_004585__09 +2008_004592__09 +2008_004619__09 +2008_004632__09 +2008_004661__09 +2008_004665__09 +2008_004670__09 +2008_004697__09 +2008_004720__09 +2008_004777__09 +2008_004778__09 +2008_004834__09 +2008_004874__09 +2008_004896__09 +2008_004964__09 +2008_004984__09 +2008_004985__09 +2008_005037__09 +2008_005066__09 +2008_005068__09 +2008_005070__09 +2008_005081__09 +2008_005107__09 +2008_005111__09 +2008_005136__09 +2008_005150__09 +2008_005167__09 +2008_005191__09 +2008_005205__09 +2008_005248__09 +2008_005257__09 +2008_005323__09 +2008_005348__09 +2008_005414__09 +2008_005417__09 +2008_005456__09 +2008_005549__09 +2008_005552__09 +2008_005561__09 +2008_005574__09 +2008_005608__09 +2008_005616__09 +2008_005698__09 +2008_005767__09 +2008_005848__09 +2008_005857__09 +2008_005881__09 +2008_005937__09 +2008_005954__09 +2008_006045__09 +2008_006076__09 +2008_006111__09 +2008_006133__09 +2008_006151__09 +2008_006258__09 +2008_006271__09 +2008_006300__09 +2008_006303__09 +2008_006307__09 +2008_006311__09 +2008_006330__09 +2008_006350__09 +2008_006387__09 +2008_006470__09 +2008_006482__09 +2008_006606__09 +2008_006611__09 +2008_006624__09 +2008_006660__09 +2008_006665__09 +2008_006684__09 +2008_006696__09 +2008_006710__09 +2008_006712__09 +2008_006719__09 +2008_006733__09 +2008_006808__09 +2008_006834__09 +2008_006837__09 +2008_006841__09 +2008_006847__09 +2008_006863__09 +2008_006979__09 +2008_007006__09 +2008_007030__09 +2008_007061__09 +2008_007085__09 +2008_007097__09 +2008_007169__09 +2008_007246__09 +2008_007281__09 +2008_007287__09 +2008_007291__09 +2008_007293__09 +2008_007298__09 +2008_007307__09 +2008_007335__09 +2008_007339__09 +2008_007355__09 +2008_007361__09 +2008_007363__09 +2008_007388__09 +2008_007415__09 +2008_007438__09 +2008_007452__09 +2008_007561__09 +2008_007685__09 +2008_007691__09 +2008_007692__09 +2008_007704__09 +2008_007706__09 +2008_007741__09 +2008_007742__09 +2008_007768__09 +2008_007780__09 +2008_007843__09 +2008_007893__09 +2008_007922__09 +2008_007941__09 +2008_007949__09 +2008_007988__09 +2008_008070__09 +2008_008098__09 +2008_008155__09 +2008_008176__09 +2008_008330__09 +2008_008363__09 +2008_008365__09 +2008_008402__09 +2008_008439__09 +2008_008536__09 +2008_008538__09 +2008_008554__09 +2008_008589__09 +2008_008593__09 +2008_008621__09 +2008_008649__09 +2008_008658__09 +2008_008694__09 +2008_008773__09 +2009_000010__09 +2009_000014__09 +2009_000028__09 +2009_000059__09 +2009_000082__09 +2009_000109__09 +2009_000164__09 +2009_000182__09 +2009_000188__09 +2009_000192__09 +2009_000198__09 +2009_000209__09 +2009_000212__09 +2009_000214__09 +2009_000232__09 +2009_000251__09 +2009_000298__09 +2009_000304__09 +2009_000422__09 +2009_000439__09 +2009_000453__09 +2009_000483__09 +2009_000552__09 +2009_000567__09 +2009_000577__09 +2009_000590__09 +2009_000604__09 +2009_000624__09 +2009_000629__09 +2009_000655__09 +2009_000672__09 +2009_000760__09 +2009_000762__09 +2009_000794__09 +2009_000811__09 +2009_000816__09 +2009_000834__09 +2009_000851__09 +2009_000928__09 +2009_000973__09 +2009_000980__09 +2009_001000__09 +2009_001009__09 +2009_001012__09 +2009_001021__09 +2009_001028__09 +2009_001038__09 +2009_001057__09 +2009_001070__09 +2009_001085__09 +2009_001095__09 +2009_001105__09 +2009_001154__09 +2009_001217__09 +2009_001243__09 +2009_001249__09 +2009_001285__09 +2009_001303__09 +2009_001308__09 +2009_001313__09 +2009_001327__09 +2009_001357__09 +2009_001371__09 +2009_001398__09 +2009_001426__09 +2009_001431__09 +2009_001434__09 +2009_001437__09 +2009_001440__09 +2009_001447__09 +2009_001462__09 +2009_001490__09 +2009_001493__09 +2009_001507__09 +2009_001575__09 +2009_001598__09 +2009_001608__09 +2009_001611__09 +2009_001621__09 +2009_001623__09 +2009_001670__09 +2009_001676__09 +2009_001754__09 +2009_001784__09 +2009_001800__09 +2009_001806__09 +2009_001817__09 +2009_001825__09 +2009_001846__09 +2009_001848__09 +2009_001856__09 +2009_001871__09 +2009_001874__09 +2009_001875__09 +2009_001922__09 +2009_001960__09 +2009_001976__09 +2009_002003__09 +2009_002044__09 +2009_002055__09 +2009_002064__09 +2009_002078__09 +2009_002088__09 +2009_002098__09 +2009_002107__09 +2009_002111__09 +2009_002139__09 +2009_002169__09 +2009_002208__09 +2009_002236__09 +2009_002252__09 +2009_002257__09 +2009_002297__09 +2009_002301__09 +2009_002325__09 +2009_002328__09 +2009_002331__09 +2009_002362__09 +2009_002374__09 +2009_002409__09 +2009_002433__09 +2009_002443__09 +2009_002444__09 +2009_002452__09 +2009_002471__09 +2009_002523__09 +2009_002537__09 +2009_002546__09 +2009_002556__09 +2009_002558__09 +2009_002585__09 +2009_002612__09 +2009_002613__09 +2009_002634__09 +2009_002652__09 +2009_002663__09 +2009_002683__09 +2009_002688__09 +2009_002698__09 +2009_002710__09 +2009_002743__09 +2009_002744__09 +2009_002755__09 +2009_002762__09 +2009_002785__09 +2009_002816__09 +2009_002827__09 +2009_002842__09 +2009_002844__09 +2009_002952__09 +2009_002978__09 +2009_002995__09 +2009_003078__09 +2009_003098__09 +2009_003110__09 +2009_003116__09 +2009_003140__09 +2009_003150__09 +2009_003157__09 +2009_003191__09 +2009_003198__09 +2009_003251__09 +2009_003253__09 +2009_003265__09 +2009_003272__09 +2009_003317__09 +2009_003320__09 +2009_003352__09 +2009_003415__09 +2009_003440__09 +2009_003508__09 +2009_003520__09 +2009_003544__09 +2009_003545__09 +2009_003554__09 +2009_003654__09 +2009_003668__09 +2009_003671__09 +2009_003677__09 +2009_003694__09 +2009_003695__09 +2009_003720__09 +2009_003752__09 +2009_003759__09 +2009_003813__09 +2009_003818__09 +2009_003835__09 +2009_003840__09 +2009_003884__09 +2009_003888__09 +2009_003920__09 +2009_003961__09 +2009_003966__09 +2009_003973__09 +2009_004002__09 +2009_004005__09 +2009_004025__09 +2009_004032__09 +2009_004042__09 +2009_004088__09 +2009_004091__09 +2009_004092__09 +2009_004094__09 +2009_004129__09 +2009_004131__09 +2009_004148__09 +2009_004154__09 +2009_004165__09 +2009_004179__09 +2009_004180__09 +2009_004228__09 +2009_004229__09 +2009_004243__09 +2009_004244__09 +2009_004263__09 +2009_004312__09 +2009_004359__09 +2009_004377__09 +2009_004404__09 +2009_004419__09 +2009_004426__09 +2009_004429__09 +2009_004445__09 +2009_004452__09 +2009_004479__09 +2009_004499__09 +2009_004501__09 +2009_004514__09 +2009_004532__09 +2009_004536__09 +2009_004537__09 +2009_004561__09 +2009_004562__09 +2009_004631__09 +2009_004674__09 +2009_004718__09 +2009_004720__09 +2009_004734__09 +2009_004737__09 +2009_004746__09 +2009_004760__09 +2009_004769__09 +2009_004784__09 +2009_004823__09 +2009_004849__09 +2009_004869__09 +2009_004874__09 +2009_004890__09 +2009_004905__09 +2009_004921__09 +2009_004922__09 +2009_004939__09 +2009_004974__09 +2009_004975__09 +2009_004979__09 +2009_005016__09 +2009_005030__09 +2009_005042__09 +2009_005062__09 +2009_005069__09 +2009_005080__09 +2009_005145__09 +2009_005163__09 +2009_005165__09 +2009_005171__09 +2009_005183__09 +2009_005202__09 +2009_005204__09 +2009_005205__09 +2009_005216__09 +2009_005257__09 +2009_005263__09 +2009_005278__09 +2009_005286__09 +2009_005288__09 +2009_005300__09 +2009_005311__09 +2010_000015__09 +2010_000018__09 +2010_000090__09 +2010_000177__09 +2010_000178__09 +2010_000337__09 +2010_000361__09 +2010_000384__09 +2010_000386__09 +2010_000431__09 +2010_000435__09 +2010_000439__09 +2010_000447__09 +2010_000463__09 +2010_000470__09 +2010_000485__09 +2010_000492__09 +2010_000497__09 +2010_000511__09 +2010_000588__09 +2010_000591__09 +2010_000601__09 +2010_000646__09 +2010_000658__09 +2010_000669__09 +2010_000674__09 +2010_000694__09 +2010_000705__09 +2010_000754__09 +2010_000761__09 +2010_000765__09 +2010_000782__09 +2010_000787__09 +2010_000800__09 +2010_000821__09 +2010_000968__09 +2010_001094__09 +2010_001123__09 +2010_001188__09 +2010_001189__09 +2010_001199__09 +2010_001216__09 +2010_001224__09 +2010_001261__09 +2010_001271__09 +2010_001277__09 +2010_001343__09 +2010_001366__09 +2010_001434__09 +2010_001455__09 +2010_001511__09 +2010_001514__09 +2010_001528__09 +2010_001533__09 +2010_001535__09 +2010_001547__09 +2010_001574__09 +2010_001608__09 +2010_001674__09 +2010_001680__09 +2010_001682__09 +2010_001712__09 +2010_001726__09 +2010_001731__09 +2010_001743__09 +2010_001823__09 +2010_001846__09 +2010_001849__09 +2010_001853__09 +2010_001857__09 +2010_001911__09 +2010_001973__09 +2010_002032__09 +2010_002050__09 +2010_002086__09 +2010_002121__09 +2010_002166__09 +2010_002191__09 +2010_002199__09 +2010_002211__09 +2010_002227__09 +2010_002319__09 +2010_002337__09 +2010_002366__09 +2010_002370__09 +2010_002383__09 +2010_002400__09 +2010_002418__09 +2010_002575__09 +2010_002603__09 +2010_002605__09 +2010_002639__09 +2010_002642__09 +2010_002686__09 +2010_002721__09 +2010_002728__09 +2010_002742__09 +2010_002760__09 +2010_002811__09 +2010_002815__09 +2010_002830__09 +2010_002870__09 +2010_002877__09 +2010_002880__09 +2010_002881__09 +2010_002958__09 +2010_003002__09 +2010_003027__09 +2010_003078__09 +2010_003103__09 +2010_003115__09 +2010_003143__09 +2010_003149__09 +2010_003154__09 +2010_003157__09 +2010_003170__09 +2010_003379__09 +2010_003398__09 +2010_003401__09 +2010_003520__09 +2010_003535__09 +2010_003538__09 +2010_003563__09 +2010_003579__09 +2010_003632__09 +2010_003634__09 +2010_003665__09 +2010_003677__09 +2010_003719__09 +2010_003728__09 +2010_003861__09 +2010_003894__09 +2010_003925__09 +2010_003957__09 +2010_003976__09 +2010_004023__09 +2010_004053__09 +2010_004062__09 +2010_004095__09 +2010_004130__09 +2010_004173__09 +2010_004207__09 +2010_004222__09 +2010_004229__09 +2010_004231__09 +2010_004249__09 +2010_004275__09 +2010_004278__09 +2010_004306__09 +2010_004345__09 +2010_004358__09 +2010_004361__09 +2010_004387__09 +2010_004429__09 +2010_004467__09 +2010_004509__09 +2010_004573__09 +2010_004592__09 +2010_004738__09 +2010_004753__09 +2010_004775__09 +2010_004778__09 +2010_004786__09 +2010_004793__09 +2010_004805__09 +2010_004813__09 +2010_004889__09 +2010_004909__09 +2010_004910__09 +2010_004942__09 +2010_004952__09 +2010_004962__09 +2010_004982__09 +2010_004983__09 +2010_004992__09 +2010_005008__09 +2010_005044__09 +2010_005052__09 +2010_005094__09 +2010_005170__09 +2010_005188__09 +2010_005222__09 +2010_005238__09 +2010_005243__09 +2010_005266__09 +2010_005308__09 +2010_005349__09 +2010_005371__09 +2010_005372__09 +2010_005388__09 +2010_005398__09 +2010_005493__09 +2010_005512__09 +2010_005540__09 +2010_005620__09 +2010_005637__09 +2010_005654__09 +2010_005746__09 +2010_005764__09 +2010_005775__09 +2010_005800__09 +2010_005830__09 +2010_005835__09 +2010_005843__09 +2010_005882__09 +2010_005884__09 +2010_005894__09 +2010_005901__09 +2010_005906__09 +2010_005921__09 +2010_005930__09 +2010_005967__09 +2010_005972__09 +2010_005975__09 +2010_005982__09 +2010_005985__09 +2010_006010__09 +2010_006015__09 +2010_006041__09 +2010_006056__09 +2011_000006__09 +2011_000009__09 +2011_000016__09 +2011_000036__09 +2011_000037__09 +2011_000065__09 +2011_000071__09 +2011_000072__09 +2011_000082__09 +2011_000098__09 +2011_000108__09 +2011_000124__09 +2011_000152__09 +2011_000192__09 +2011_000196__09 +2011_000202__09 +2011_000213__09 +2011_000253__09 +2011_000282__09 +2011_000288__09 +2011_000290__09 +2011_000317__09 +2011_000382__09 +2011_000383__09 +2011_000492__09 +2011_000499__09 +2011_000518__09 +2011_000567__09 +2011_000572__09 +2011_000573__09 +2011_000575__09 +2011_000608__09 +2011_000682__09 +2011_000684__09 +2011_000689__09 +2011_000731__09 +2011_000745__09 +2011_000755__09 +2011_000815__09 +2011_000823__09 +2011_000837__09 +2011_000875__09 +2011_000885__09 +2011_000917__09 +2011_000919__09 +2011_000932__09 +2011_000957__09 +2011_000965__09 +2011_000983__09 +2011_001009__09 +2011_001023__09 +2011_001027__09 +2011_001031__09 +2011_001032__09 +2011_001106__09 +2011_001111__09 +2011_001135__09 +2011_001137__09 +2011_001139__09 +2011_001150__09 +2011_001167__09 +2011_001175__09 +2011_001208__09 +2011_001221__09 +2011_001223__09 +2011_001226__09 +2011_001304__09 +2011_001305__09 +2011_001319__09 +2011_001330__09 +2011_001366__09 +2011_001370__09 +2011_001404__09 +2011_001412__09 +2011_001451__09 +2011_001456__09 +2011_001466__09 +2011_001524__09 +2011_001526__09 +2011_001566__09 +2011_001573__09 +2011_001586__09 +2011_001600__09 +2011_001650__09 +2011_001666__09 +2011_001689__09 +2011_001715__09 +2011_001719__09 +2011_001766__09 +2011_001785__09 +2011_001791__09 +2011_001811__09 +2011_001833__09 +2011_001834__09 +2011_001842__09 +2011_001847__09 +2011_001884__09 +2011_001900__09 +2011_001930__09 +2011_001938__09 +2011_001941__09 +2011_001946__09 +2011_001972__09 +2011_002005__09 +2011_002038__09 +2011_002045__09 +2011_002163__09 +2011_002177__09 +2011_002179__09 +2011_002241__09 +2011_002260__09 +2011_002270__09 +2011_002335__09 +2011_002386__09 +2011_002422__09 +2011_002462__09 +2011_002531__09 +2011_002543__09 +2011_002551__09 +2011_002594__09 +2011_002614__09 +2011_002620__09 +2011_002640__09 +2011_002650__09 +2011_002709__09 +2011_002714__09 +2011_002750__09 +2011_002756__09 +2011_002798__09 +2011_002803__09 +2011_002805__09 +2011_002808__09 +2011_002814__09 +2011_002870__09 +2011_002887__09 +2011_002924__09 +2011_002927__09 +2011_002933__09 +2011_002947__09 +2011_002953__09 +2011_002971__09 +2011_002994__09 +2011_003005__09 +2011_003044__09 +2011_003047__09 +2011_003049__09 +2011_003074__09 +2011_003076__09 +2011_003079__09 +2011_003097__09 +2011_003150__09 +2011_003166__09 +2011_003183__09 +2011_003194__09 +2011_003201__09 +2011_003236__09 +2011_003259__09 +2007_000504__10 +2007_000904__10 +2007_001073__10 +2007_001764__10 +2007_001917__10 +2007_002088__10 +2007_002669__10 +2007_002789__10 +2007_004081__10 +2007_004500__10 +2007_004537__10 +2007_005124__10 +2007_005797__10 +2007_007772__10 +2007_009082__10 +2008_000335__10 +2008_000711__10 +2008_000876__10 +2008_000905__10 +2008_000964__10 +2008_001062__10 +2008_001359__10 +2008_001653__10 +2008_002278__10 +2008_002732__10 +2008_003065__10 +2008_003094__10 +2008_003256__10 +2008_003297__10 +2008_004394__10 +2008_004841__10 +2008_005010__10 +2008_005375__10 +2008_005714__10 +2008_006210__10 +2008_006290__10 +2008_006355__10 +2008_006496__10 +2008_006547__10 +2008_006827__10 +2008_006880__10 +2008_006904__10 +2008_007009__10 +2008_007014__10 +2008_007026__10 +2008_007398__10 +2008_007544__10 +2008_007586__10 +2008_007666__10 +2008_007729__10 +2008_007932__10 +2008_008115__10 +2008_008121__10 +2008_008132__10 +2008_008169__10 +2008_008199__10 +2008_008241__10 +2008_008428__10 +2008_008437__10 +2008_008521__10 +2008_008541__10 +2008_008617__10 +2008_008654__10 +2008_008675__10 +2008_008685__10 +2009_000066__10 +2009_000091__10 +2009_000250__10 +2009_000285__10 +2009_000471__10 +2009_000472__10 +2009_000559__10 +2009_000574__10 +2009_000626__10 +2009_000632__10 +2009_000694__10 +2009_000709__10 +2009_000768__10 +2009_000899__10 +2009_000971__10 +2009_001079__10 +2009_001104__10 +2009_001134__10 +2009_001145__10 +2009_001163__10 +2009_001257__10 +2009_001413__10 +2009_001475__10 +2009_001518__10 +2009_001673__10 +2009_001798__10 +2009_001873__10 +2009_002002__10 +2009_002019__10 +2009_002118__10 +2009_002136__10 +2009_002229__10 +2009_002360__10 +2009_002422__10 +2009_002472__10 +2009_002510__10 +2009_002518__10 +2009_002542__10 +2009_002588__10 +2009_002597__10 +2009_002648__10 +2009_002708__10 +2009_002772__10 +2009_002869__10 +2009_002901__10 +2009_002912__10 +2009_003089__10 +2009_003146__10 +2009_003173__10 +2009_003189__10 +2009_003218__10 +2009_003297__10 +2009_003455__10 +2009_003510__10 +2009_003521__10 +2009_003634__10 +2009_003825__10 +2009_003901__10 +2009_003933__10 +2009_003956__10 +2009_003975__10 +2009_004169__10 +2009_004186__10 +2009_004233__10 +2009_004284__10 +2009_004289__10 +2009_004316__10 +2009_004351__10 +2009_004366__10 +2009_004451__10 +2009_004679__10 +2009_004787__10 +2009_004929__10 +2009_005068__10 +2009_005081__10 +2009_005229__10 +2009_005307__10 +2010_000063__10 +2010_000498__10 +2010_000545__10 +2010_000548__10 +2010_000808__10 +2010_000910__10 +2010_001317__10 +2010_001406__10 +2010_001548__10 +2010_001671__10 +2010_001694__10 +2010_001803__10 +2010_001937__10 +2010_002018__10 +2010_002047__10 +2010_002133__10 +2010_002139__10 +2010_002254__10 +2010_002320__10 +2010_002716__10 +2010_002797__10 +2010_002834__10 +2010_002905__10 +2010_003011__10 +2010_003016__10 +2010_003201__10 +2010_003345__10 +2010_003383__10 +2010_003507__10 +2010_003821__10 +2010_003970__10 +2010_004140__10 +2010_004163__10 +2010_004210__10 +2010_004264__10 +2010_004283__10 +2010_004436__10 +2010_004456__10 +2010_004598__10 +2010_004655__10 +2010_005031__10 +2010_005061__10 +2010_005062__10 +2010_005223__10 +2010_005258__10 +2010_005276__10 +2010_005297__10 +2010_005424__10 +2010_005458__10 +2010_005468__10 +2010_005562__10 +2010_005668__10 +2010_005824__10 +2010_005993__10 +2011_000057__10 +2011_000069__10 +2011_000109__10 +2011_000166__10 +2011_000219__10 +2011_000299__10 +2011_000309__10 +2011_000763__10 +2011_000778__10 +2011_000784__10 +2011_001079__10 +2011_001163__10 +2011_001282__10 +2011_001318__10 +2011_001333__10 +2011_001354__10 +2011_001399__10 +2011_001424__10 +2011_001653__10 +2011_001679__10 +2011_001805__10 +2011_001840__10 +2011_001854__10 +2011_001942__10 +2011_001950__10 +2011_002018__10 +2011_002019__10 +2011_002134__10 +2011_002189__10 +2011_002272__10 +2011_002584__10 +2011_002609__10 +2011_002649__10 +2011_002719__10 +2011_002864__10 +2011_002873__10 +2011_002937__10 +2011_003016__10 +2011_003043__10 diff --git a/submodules/lang_seg/fewshot_data/data/splits/pascal/trn/fold2.txt b/submodules/lang_seg/fewshot_data/data/splits/pascal/trn/fold2.txt new file mode 100644 index 0000000000000000000000000000000000000000..5815a3c5355e8e5805ac2a85d51b2f6c9217ade2 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/pascal/trn/fold2.txt @@ -0,0 +1,5883 @@ +2007_000250__11 +2007_001185__11 +2007_001439__11 +2007_001609__11 +2007_001834__11 +2007_001901__11 +2007_002368__11 +2007_002668__11 +2007_002914__11 +2007_003251__11 +2007_003529__11 +2007_003668__11 +2007_004166__11 +2007_005086__11 +2007_005647__11 +2007_005790__11 +2007_006409__11 +2007_006699__11 +2007_007021__11 +2007_008072__11 +2007_008932__11 +2007_008945__11 +2007_009901__11 +2008_000041__11 +2008_000043__11 +2008_000227__11 +2008_000290__11 +2008_000338__11 +2008_000397__11 +2008_000418__11 +2008_000835__11 +2008_000885__11 +2008_001077__11 +2008_001083__11 +2008_001155__11 +2008_001230__11 +2008_001413__11 +2008_001434__11 +2008_001451__11 +2008_001723__11 +2008_001758__11 +2008_001773__11 +2008_001809__11 +2008_002032__11 +2008_002079__11 +2008_002114__11 +2008_002210__11 +2008_002362__11 +2008_002384__11 +2008_002508__11 +2008_002567__11 +2008_002892__11 +2008_002903__11 +2008_002960__11 +2008_003224__11 +2008_003245__11 +2008_003264__11 +2008_003534__11 +2008_003753__11 +2008_003774__11 +2008_003796__11 +2008_003881__11 +2008_003904__11 +2008_003998__11 +2008_004216__11 +2008_004289__11 +2008_004293__11 +2008_004321__11 +2008_004457__11 +2008_004564__11 +2008_004588__11 +2008_004776__11 +2008_004851__11 +2008_004948__11 +2008_004979__11 +2008_005070__11 +2008_005081__11 +2008_005150__11 +2008_005257__11 +2008_005342__11 +2008_005348__11 +2008_005570__11 +2008_005608__11 +2008_005945__11 +2008_005953__11 +2008_005954__11 +2008_005975__11 +2008_006120__11 +2008_006133__11 +2008_006151__11 +2008_006192__11 +2008_006311__11 +2008_006482__11 +2008_006588__11 +2008_006611__11 +2008_006705__11 +2008_006750__11 +2008_006808__11 +2008_006952__11 +2008_006969__11 +2008_007097__11 +2008_007261__11 +2008_007291__11 +2008_007355__11 +2008_007438__11 +2008_007692__11 +2008_007717__11 +2008_007853__11 +2008_007922__11 +2008_007949__11 +2008_008070__11 +2008_008093__11 +2008_008098__11 +2008_008155__11 +2008_008224__11 +2008_008363__11 +2008_008365__11 +2008_008388__11 +2008_008440__11 +2008_008474__11 +2008_008773__11 +2009_000058__11 +2009_000109__11 +2009_000141__11 +2009_000164__11 +2009_000188__11 +2009_000212__11 +2009_000247__11 +2009_000257__11 +2009_000300__11 +2009_000502__11 +2009_000546__11 +2009_000575__11 +2009_000655__11 +2009_000759__11 +2009_000760__11 +2009_000811__11 +2009_000831__11 +2009_000834__11 +2009_000846__11 +2009_000867__11 +2009_000969__11 +2009_000980__11 +2009_001011__11 +2009_001054__11 +2009_001057__11 +2009_001095__11 +2009_001105__11 +2009_001217__11 +2009_001319__11 +2009_001327__11 +2009_001339__11 +2009_001398__11 +2009_001426__11 +2009_001434__11 +2009_001447__11 +2009_001575__11 +2009_001608__11 +2009_001670__11 +2009_001754__11 +2009_001800__11 +2009_001825__11 +2009_001867__11 +2009_001871__11 +2009_001922__11 +2009_002003__11 +2009_002024__11 +2009_002044__11 +2009_002055__11 +2009_002078__11 +2009_002088__11 +2009_002098__11 +2009_002107__11 +2009_002111__11 +2009_002139__11 +2009_002169__11 +2009_002328__11 +2009_002433__11 +2009_002443__11 +2009_002444__11 +2009_002452__11 +2009_002471__11 +2009_002524__11 +2009_002546__11 +2009_002579__11 +2009_002612__11 +2009_002613__11 +2009_002634__11 +2009_002663__11 +2009_002688__11 +2009_002743__11 +2009_002790__11 +2009_002816__11 +2009_002844__11 +2009_002885__11 +2009_002952__11 +2009_002978__11 +2009_003006__11 +2009_003078__11 +2009_003115__11 +2009_003150__11 +2009_003154__11 +2009_003191__11 +2009_003198__11 +2009_003251__11 +2009_003253__11 +2009_003265__11 +2009_003282__11 +2009_003317__11 +2009_003349__11 +2009_003361__11 +2009_003440__11 +2009_003468__11 +2009_003520__11 +2009_003545__11 +2009_003577__11 +2009_003654__11 +2009_003690__11 +2009_003752__11 +2009_003759__11 +2009_003795__11 +2009_003818__11 +2009_003884__11 +2009_003888__11 +2009_003914__11 +2009_003973__11 +2009_004032__11 +2009_004042__11 +2009_004085__11 +2009_004094__11 +2009_004124__11 +2009_004154__11 +2009_004162__11 +2009_004177__11 +2009_004228__11 +2009_004263__11 +2009_004309__11 +2009_004312__11 +2009_004394__11 +2009_004399__11 +2009_004419__11 +2009_004426__11 +2009_004429__11 +2009_004499__11 +2009_004514__11 +2009_004532__11 +2009_004537__11 +2009_004562__11 +2009_004616__11 +2009_004720__11 +2009_004734__11 +2009_004769__11 +2009_004794__11 +2009_004847__11 +2009_004890__11 +2009_004974__11 +2009_004979__11 +2009_005001__11 +2009_005016__11 +2009_005069__11 +2009_005080__11 +2009_005145__11 +2009_005165__11 +2009_005171__11 +2009_005202__11 +2009_005205__11 +2009_005278__11 +2009_005311__11 +2010_000001__11 +2010_000088__11 +2010_000177__11 +2010_000178__11 +2010_000234__11 +2010_000269__11 +2010_000325__11 +2010_000337__11 +2010_000352__11 +2010_000419__11 +2010_000463__11 +2010_000591__11 +2010_000601__11 +2010_000669__11 +2010_000697__11 +2010_000754__11 +2010_000782__11 +2010_000821__11 +2010_000915__11 +2010_000938__11 +2010_001131__11 +2010_001181__11 +2010_001199__11 +2010_001261__11 +2010_001271__11 +2010_001287__11 +2010_001343__11 +2010_001366__11 +2010_001511__11 +2010_001535__11 +2010_001576__11 +2010_001608__11 +2010_001645__11 +2010_001680__11 +2010_001682__11 +2010_001777__11 +2010_001785__11 +2010_001823__11 +2010_001846__11 +2010_001853__11 +2010_001907__11 +2010_001939__11 +2010_002006__11 +2010_002032__11 +2010_002070__11 +2010_002199__11 +2010_002269__11 +2010_002274__11 +2010_002366__11 +2010_002373__11 +2010_002400__11 +2010_002418__11 +2010_002462__11 +2010_002520__11 +2010_002537__11 +2010_002539__11 +2010_002605__11 +2010_002639__11 +2010_002714__11 +2010_002721__11 +2010_002725__11 +2010_002728__11 +2010_002729__11 +2010_002742__11 +2010_002811__11 +2010_002830__11 +2010_002877__11 +2010_002880__11 +2010_003002__11 +2010_003078__11 +2010_003084__11 +2010_003091__11 +2010_003108__11 +2010_003154__11 +2010_003157__11 +2010_003263__11 +2010_003374__11 +2010_003520__11 +2010_003594__11 +2010_003625__11 +2010_003762__11 +2010_003822__11 +2010_003897__11 +2010_003976__11 +2010_004125__11 +2010_004229__11 +2010_004231__11 +2010_004275__11 +2010_004278__11 +2010_004290__11 +2010_004361__11 +2010_004380__11 +2010_004391__11 +2010_004467__11 +2010_004592__11 +2010_004597__11 +2010_004660__11 +2010_004726__11 +2010_004741__11 +2010_004751__11 +2010_004778__11 +2010_004793__11 +2010_004909__11 +2010_004910__11 +2010_004937__11 +2010_004942__11 +2010_004983__11 +2010_004987__11 +2010_004992__11 +2010_005094__11 +2010_005170__11 +2010_005273__11 +2010_005349__11 +2010_005369__11 +2010_005379__11 +2010_005398__11 +2010_005493__11 +2010_005576__11 +2010_005654__11 +2010_005666__11 +2010_005755__11 +2010_005764__11 +2010_005825__11 +2010_005865__11 +2010_005884__11 +2010_005901__11 +2010_005936__11 +2010_005938__11 +2010_005985__11 +2010_006010__11 +2010_006056__11 +2011_000016__11 +2011_000065__11 +2011_000098__11 +2011_000192__11 +2011_000196__11 +2011_000206__11 +2011_000257__11 +2011_000282__11 +2011_000361__11 +2011_000408__11 +2011_000427__11 +2011_000492__11 +2011_000530__11 +2011_000573__11 +2011_000692__11 +2011_000731__11 +2011_000745__11 +2011_000815__11 +2011_000823__11 +2011_000827__11 +2011_000837__11 +2011_000847__11 +2011_000875__11 +2011_000919__11 +2011_000932__11 +2011_000947__11 +2011_000996__11 +2011_001008__11 +2011_001009__11 +2011_001058__11 +2011_001106__11 +2011_001139__11 +2011_001167__11 +2011_001208__11 +2011_001215__11 +2011_001220__11 +2011_001221__11 +2011_001295__11 +2011_001305__11 +2011_001330__11 +2011_001336__11 +2011_001366__11 +2011_001382__11 +2011_001412__11 +2011_001526__11 +2011_001571__11 +2011_001586__11 +2011_001616__11 +2011_001678__11 +2011_001785__11 +2011_001819__11 +2011_001820__11 +2011_001842__11 +2011_001847__11 +2011_001884__11 +2011_001900__11 +2011_001920__11 +2011_001941__11 +2011_001975__11 +2011_002074__11 +2011_002142__11 +2011_002147__11 +2011_002163__11 +2011_002185__11 +2011_002241__11 +2011_002348__11 +2011_002394__11 +2011_002422__11 +2011_002462__11 +2011_002488__11 +2011_002531__11 +2011_002542__11 +2011_002556__11 +2011_002579__11 +2011_002606__11 +2011_002614__11 +2011_002640__11 +2011_002650__11 +2011_002714__11 +2011_002748__11 +2011_002798__11 +2011_002803__11 +2011_002805__11 +2011_002838__11 +2011_002880__11 +2011_002881__11 +2011_002911__11 +2011_002920__11 +2011_002924__11 +2011_002927__11 +2011_002933__11 +2011_002940__11 +2011_002966__11 +2011_002974__11 +2011_002985__11 +2011_003049__11 +2011_003050__11 +2011_003066__11 +2011_003076__11 +2011_003152__11 +2011_003158__11 +2011_003166__11 +2011_003183__11 +2011_003242__11 +2011_003262__11 +2007_000063__12 +2007_000720__12 +2007_001225__12 +2007_001340__12 +2007_001397__12 +2007_001825__12 +2007_002055__12 +2007_002611__12 +2007_003604__12 +2007_005988__12 +2007_007585__12 +2007_007930__12 +2007_009605__12 +2007_009889__12 +2007_009899__12 +2008_000019__12 +2008_000026__12 +2008_000053__12 +2008_000059__12 +2008_000066__12 +2008_000078__12 +2008_000099__12 +2008_000138__12 +2008_000162__12 +2008_000183__12 +2008_000268__12 +2008_000273__12 +2008_000330__12 +2008_000336__12 +2008_000419__12 +2008_000559__12 +2008_000620__12 +2008_000641__12 +2008_000676__12 +2008_000690__12 +2008_000706__12 +2008_000721__12 +2008_000761__12 +2008_000788__12 +2008_000808__12 +2008_000829__12 +2008_000837__12 +2008_000901__12 +2008_000931__12 +2008_000934__12 +2008_000956__12 +2008_000973__12 +2008_001137__12 +2008_001182__12 +2008_001220__12 +2008_001238__12 +2008_001248__12 +2008_001285__12 +2008_001410__12 +2008_001420__12 +2008_001430__12 +2008_001436__12 +2008_001456__12 +2008_001461__12 +2008_001479__12 +2008_001498__12 +2008_001550__12 +2008_001660__12 +2008_001676__12 +2008_001708__12 +2008_001797__12 +2008_001816__12 +2008_001854__12 +2008_001862__12 +2008_001871__12 +2008_001872__12 +2008_001921__12 +2008_001965__12 +2008_002003__12 +2008_002033__12 +2008_002084__12 +2008_002096__12 +2008_002116__12 +2008_002160__12 +2008_002227__12 +2008_002267__12 +2008_002304__12 +2008_002307__12 +2008_002317__12 +2008_002359__12 +2008_002395__12 +2008_002408__12 +2008_002430__12 +2008_002436__12 +2008_002441__12 +2008_002442__12 +2008_002494__12 +2008_002574__12 +2008_002584__12 +2008_002598__12 +2008_002610__12 +2008_002612__12 +2008_002621__12 +2008_002638__12 +2008_002687__12 +2008_002767__12 +2008_002814__12 +2008_002913__12 +2008_002992__12 +2008_003101__12 +2008_003133__12 +2008_003180__12 +2008_003272__12 +2008_003276__12 +2008_003286__12 +2008_003331__12 +2008_003347__12 +2008_003417__12 +2008_003443__12 +2008_003497__12 +2008_003520__12 +2008_003521__12 +2008_003572__12 +2008_003587__12 +2008_003607__12 +2008_003645__12 +2008_003665__12 +2008_003718__12 +2008_003746__12 +2008_003755__12 +2008_003776__12 +2008_003799__12 +2008_003801__12 +2008_003838__12 +2008_003852__12 +2008_003922__12 +2008_003944__12 +2008_003975__12 +2008_003978__12 +2008_004015__12 +2008_004021__12 +2008_004044__12 +2008_004058__12 +2008_004106__12 +2008_004121__12 +2008_004122__12 +2008_004174__12 +2008_004188__12 +2008_004218__12 +2008_004265__12 +2008_004273__12 +2008_004284__12 +2008_004293__12 +2008_004297__12 +2008_004306__12 +2008_004319__12 +2008_004324__12 +2008_004402__12 +2008_004410__12 +2008_004427__12 +2008_004480__12 +2008_004505__12 +2008_004515__12 +2008_004525__12 +2008_004528__12 +2008_004590__12 +2008_004647__12 +2008_004653__12 +2008_004662__12 +2008_004695__12 +2008_004702__12 +2008_004722__12 +2008_004745__12 +2008_004752__12 +2008_004760__12 +2008_004768__12 +2008_004774__12 +2008_004804__12 +2008_004821__12 +2008_004833__12 +2008_004845__12 +2008_004900__12 +2008_004931__12 +2008_004950__12 +2008_004975__12 +2008_004998__12 +2008_005015__12 +2008_005016__12 +2008_005046__12 +2008_005051__12 +2008_005065__12 +2008_005082__12 +2008_005084__12 +2008_005101__12 +2008_005111__12 +2008_005123__12 +2008_005137__12 +2008_005160__12 +2008_005167__12 +2008_005174__12 +2008_005178__12 +2008_005181__12 +2008_005194__12 +2008_005221__12 +2008_005240__12 +2008_005272__12 +2008_005333__12 +2008_005335__12 +2008_005336__12 +2008_005346__12 +2008_005361__12 +2008_005363__12 +2008_005379__12 +2008_005382__12 +2008_005406__12 +2008_005415__12 +2008_005436__12 +2008_005451__12 +2008_005467__12 +2008_005514__12 +2008_005527__12 +2008_005563__12 +2008_005564__12 +2008_005591__12 +2008_005601__12 +2008_005643__12 +2008_005652__12 +2008_005656__12 +2008_005679__12 +2008_005703__12 +2008_005728__12 +2008_005748__12 +2008_005798__12 +2008_005800__12 +2008_005823__12 +2008_005838__12 +2008_005845__12 +2008_005850__12 +2008_005860__12 +2008_005865__12 +2008_005871__12 +2008_005882__12 +2008_005883__12 +2008_005890__12 +2008_005897__12 +2008_005898__12 +2008_005964__12 +2008_005972__12 +2008_005979__12 +2008_005980__12 +2008_006007__12 +2008_006020__12 +2008_006032__12 +2008_006038__12 +2008_006041__12 +2008_006049__12 +2008_006067__12 +2008_006071__12 +2008_006074__12 +2008_006119__12 +2008_006129__12 +2008_006166__12 +2008_006170__12 +2008_006178__12 +2008_006203__12 +2008_006211__12 +2008_006239__12 +2008_006315__12 +2008_006320__12 +2008_006335__12 +2008_006356__12 +2008_006386__12 +2008_006416__12 +2008_006421__12 +2008_006425__12 +2008_006436__12 +2008_006452__12 +2008_006474__12 +2008_006502__12 +2008_006511__12 +2008_006546__12 +2008_006568__12 +2008_006578__12 +2008_006587__12 +2008_006602__12 +2008_006614__12 +2008_006616__12 +2008_006682__12 +2008_006716__12 +2008_006717__12 +2008_006724__12 +2008_006743__12 +2008_006798__12 +2008_006811__12 +2008_006837__12 +2008_006844__12 +2008_006870__12 +2008_006908__12 +2008_006965__12 +2008_006968__12 +2008_007012__12 +2008_007021__12 +2008_007028__12 +2008_007045__12 +2008_007057__12 +2008_007073__12 +2008_007081__12 +2008_007108__12 +2008_007118__12 +2008_007131__12 +2008_007146__12 +2008_007163__12 +2008_007169__12 +2008_007187__12 +2008_007208__12 +2008_007236__12 +2008_007274__12 +2008_007279__12 +2008_007286__12 +2008_007311__12 +2008_007312__12 +2008_007320__12 +2008_007321__12 +2008_007358__12 +2008_007384__12 +2008_007455__12 +2008_007461__12 +2008_007471__12 +2008_007478__12 +2008_007486__12 +2008_007491__12 +2008_007494__12 +2008_007509__12 +2008_007519__12 +2008_007538__12 +2008_007559__12 +2008_007567__12 +2008_007581__12 +2008_007583__12 +2008_007621__12 +2008_007649__12 +2008_007664__12 +2008_007665__12 +2008_007668__12 +2008_007694__12 +2008_007710__12 +2008_007735__12 +2008_007781__12 +2008_007805__12 +2008_007819__12 +2008_007840__12 +2008_007875__12 +2008_007902__12 +2008_007907__12 +2008_007917__12 +2008_007937__12 +2008_008433__12 +2008_008578__12 +2008_008622__12 +2008_008636__12 +2009_000002__12 +2009_000016__12 +2009_000029__12 +2009_000045__12 +2009_000088__12 +2009_000142__12 +2009_000145__12 +2009_000181__12 +2009_000189__12 +2009_000232__12 +2009_000282__12 +2009_000287__12 +2009_000321__12 +2009_000339__12 +2009_000400__12 +2009_000410__12 +2009_000416__12 +2009_000449__12 +2009_000452__12 +2009_000496__12 +2009_000512__12 +2009_000539__12 +2009_000550__12 +2009_000577__12 +2009_000586__12 +2009_000593__12 +2009_000600__12 +2009_000636__12 +2009_000647__12 +2009_000651__12 +2009_000702__12 +2009_000718__12 +2009_000770__12 +2009_000774__12 +2009_000778__12 +2009_000779__12 +2009_000804__12 +2009_000856__12 +2009_000869__12 +2009_000871__12 +2009_000874__12 +2009_000923__12 +2009_000953__12 +2009_000979__12 +2009_001006__12 +2009_001026__12 +2009_001052__12 +2009_001075__12 +2009_001124__12 +2009_001133__12 +2009_001139__12 +2009_001201__12 +2009_001224__12 +2009_001225__12 +2009_001251__12 +2009_001271__12 +2009_001286__12 +2009_001288__12 +2009_001329__12 +2009_001367__12 +2009_001368__12 +2009_001424__12 +2009_001446__12 +2009_001449__12 +2009_001470__12 +2009_001476__12 +2009_001498__12 +2009_001507__12 +2009_001538__12 +2009_001553__12 +2009_001568__12 +2009_001577__12 +2009_001589__12 +2009_001605__12 +2009_001606__12 +2009_001611__12 +2009_001706__12 +2009_001733__12 +2009_001774__12 +2009_001778__12 +2009_001792__12 +2009_001828__12 +2009_001833__12 +2009_001869__12 +2009_001881__12 +2009_001885__12 +2009_001902__12 +2009_001916__12 +2009_001952__12 +2009_001960__12 +2009_001962__12 +2009_002031__12 +2009_002039__12 +2009_002093__12 +2009_002177__12 +2009_002192__12 +2009_002193__12 +2009_002198__12 +2009_002215__12 +2009_002259__12 +2009_002273__12 +2009_002285__12 +2009_002302__12 +2009_002324__12 +2009_002333__12 +2009_002370__12 +2009_002373__12 +2009_002408__12 +2009_002419__12 +2009_002456__12 +2009_002460__12 +2009_002464__12 +2009_002476__12 +2009_002500__12 +2009_002522__12 +2009_002553__12 +2009_002563__12 +2009_002586__12 +2009_002605__12 +2009_002609__12 +2009_002672__12 +2009_002681__12 +2009_002687__12 +2009_002728__12 +2009_002789__12 +2009_002803__12 +2009_002807__12 +2009_002845__12 +2009_002847__12 +2009_002865__12 +2009_002893__12 +2009_002908__12 +2009_002925__12 +2009_002958__12 +2009_003023__12 +2009_003095__12 +2009_003098__12 +2009_003122__12 +2009_003130__12 +2009_003156__12 +2009_003172__12 +2009_003194__12 +2009_003209__12 +2009_003225__12 +2009_003262__12 +2009_003271__12 +2009_003277__12 +2009_003301__12 +2009_003369__12 +2009_003375__12 +2009_003395__12 +2009_003402__12 +2009_003467__12 +2009_003499__12 +2009_003513__12 +2009_003538__12 +2009_003612__12 +2009_003627__12 +2009_003677__12 +2009_003708__12 +2009_003711__12 +2009_003714__12 +2009_003717__12 +2009_003735__12 +2009_003786__12 +2009_003816__12 +2009_003827__12 +2009_003879__12 +2009_003902__12 +2009_003913__12 +2009_003986__12 +2009_003992__12 +2009_004012__12 +2009_004016__12 +2009_004020__12 +2009_004073__12 +2009_004074__12 +2009_004076__12 +2009_004082__12 +2009_004111__12 +2009_004139__12 +2009_004150__12 +2009_004179__12 +2009_004193__12 +2009_004200__12 +2009_004227__12 +2009_004229__12 +2009_004264__12 +2009_004277__12 +2009_004319__12 +2009_004361__12 +2009_004390__12 +2009_004392__12 +2009_004403__12 +2009_004411__12 +2009_004425__12 +2009_004445__12 +2009_004465__12 +2009_004479__12 +2009_004483__12 +2009_004545__12 +2009_004560__12 +2009_004593__12 +2009_004614__12 +2009_004628__12 +2009_004634__12 +2009_004648__12 +2009_004683__12 +2009_004688__12 +2009_004716__12 +2009_004728__12 +2009_004744__12 +2009_004763__12 +2009_004772__12 +2009_004782__12 +2009_004828__12 +2009_004855__12 +2009_004889__12 +2009_004961__12 +2009_005055__12 +2009_005073__12 +2009_005102__12 +2009_005107__12 +2009_005153__12 +2009_005162__12 +2009_005185__12 +2009_005272__12 +2010_000014__12 +2010_000026__12 +2010_000033__12 +2010_000045__12 +2010_000061__12 +2010_000072__12 +2010_000074__12 +2010_000079__12 +2010_000082__12 +2010_000090__12 +2010_000098__12 +2010_000124__12 +2010_000127__12 +2010_000140__12 +2010_000152__12 +2010_000162__12 +2010_000170__12 +2010_000183__12 +2010_000203__12 +2010_000227__12 +2010_000247__12 +2010_000263__12 +2010_000266__12 +2010_000276__12 +2010_000286__12 +2010_000293__12 +2010_000312__12 +2010_000320__12 +2010_000362__12 +2010_000370__12 +2010_000384__12 +2010_000395__12 +2010_000415__12 +2010_000453__12 +2010_000456__12 +2010_000475__12 +2010_000484__12 +2010_000515__12 +2010_000561__12 +2010_000574__12 +2010_000577__12 +2010_000608__12 +2010_000626__12 +2010_000635__12 +2010_000644__12 +2010_000648__12 +2010_000655__12 +2010_000681__12 +2010_000716__12 +2010_000731__12 +2010_000746__12 +2010_000760__12 +2010_000761__12 +2010_000771__12 +2010_000785__12 +2010_000805__12 +2010_000811__12 +2010_000837__12 +2010_000842__12 +2010_000846__12 +2010_000862__12 +2010_000870__12 +2010_000879__12 +2010_000883__12 +2010_000889__12 +2010_000899__12 +2010_000926__12 +2010_000968__12 +2010_000973__12 +2010_000975__12 +2010_000983__12 +2010_000989__12 +2010_001002__12 +2010_001006__12 +2010_001009__12 +2010_001023__12 +2010_001032__12 +2010_001063__12 +2010_001074__12 +2010_001092__12 +2010_001094__12 +2010_001099__12 +2010_001154__12 +2010_001172__12 +2010_001185__12 +2010_001211__12 +2010_001216__12 +2010_001218__12 +2010_001229__12 +2010_001237__12 +2010_001242__12 +2010_001253__12 +2010_001273__12 +2010_001279__12 +2010_001282__12 +2010_001293__12 +2010_001301__12 +2010_001305__12 +2010_001311__12 +2010_001326__12 +2010_001339__12 +2010_001370__12 +2010_001390__12 +2010_001399__12 +2010_001407__12 +2010_001421__12 +2010_001422__12 +2010_001425__12 +2010_001432__12 +2010_001450__12 +2010_001461__12 +2010_001464__12 +2010_001473__12 +2010_001487__12 +2010_001497__12 +2010_001529__12 +2010_001537__12 +2010_001543__12 +2010_001571__12 +2010_001596__12 +2010_001601__12 +2010_001626__12 +2010_001633__12 +2010_001635__12 +2010_001644__12 +2010_001659__12 +2010_001665__12 +2010_001679__12 +2010_001687__12 +2010_001689__12 +2010_001705__12 +2010_001706__12 +2010_001709__12 +2010_001715__12 +2010_001720__12 +2010_001726__12 +2010_001754__12 +2010_001756__12 +2010_001760__12 +2010_001776__12 +2010_001795__12 +2010_001837__12 +2010_001838__12 +2010_001841__12 +2010_001843__12 +2010_001845__12 +2010_001850__12 +2010_001869__12 +2010_001891__12 +2010_001896__12 +2010_001904__12 +2010_001911__12 +2010_001924__12 +2010_001929__12 +2010_001933__12 +2010_001944__12 +2010_001976__12 +2010_002005__12 +2010_002020__12 +2010_002022__12 +2010_002029__12 +2010_002037__12 +2010_002042__12 +2010_002060__12 +2010_002067__12 +2010_002089__12 +2010_002095__12 +2010_002105__12 +2010_002187__12 +2010_002220__12 +2010_002248__12 +2010_002276__12 +2010_002287__12 +2010_002299__12 +2010_002309__12 +2010_002313__12 +2010_002318__12 +2010_002332__12 +2010_002338__12 +2010_002349__12 +2010_002365__12 +2010_002382__12 +2010_002391__12 +2010_002398__12 +2010_002399__12 +2010_002402__12 +2010_002410__12 +2010_002449__12 +2010_002456__12 +2010_002458__12 +2010_002468__12 +2010_002499__12 +2010_002527__12 +2010_002534__12 +2010_002551__12 +2010_002552__12 +2010_002561__12 +2010_002562__12 +2010_002570__12 +2010_002580__12 +2010_002586__12 +2010_002589__12 +2010_002601__12 +2010_002616__12 +2010_002631__12 +2010_002644__12 +2010_002654__12 +2010_002659__12 +2010_002674__12 +2010_002676__12 +2010_002679__12 +2010_002696__12 +2010_002741__12 +2010_002759__12 +2010_002775__12 +2010_002786__12 +2010_002793__12 +2010_002794__12 +2010_002805__12 +2010_002808__12 +2010_002813__12 +2010_002817__12 +2010_002824__12 +2010_002856__12 +2010_002857__12 +2010_002879__12 +2010_002884__12 +2010_002887__12 +2010_002901__12 +2010_002907__12 +2010_002924__12 +2010_002938__12 +2010_002960__12 +2010_002972__12 +2010_002979__12 +2010_002980__12 +2010_002995__12 +2010_003007__12 +2010_003010__12 +2010_003019__12 +2010_003032__12 +2010_003043__12 +2010_003055__12 +2010_003056__12 +2010_003071__12 +2010_003077__12 +2010_003082__12 +2010_003086__12 +2010_003098__12 +2010_003114__12 +2010_003138__12 +2010_003146__12 +2010_003147__12 +2010_003162__12 +2010_003176__12 +2010_003186__12 +2010_003204__12 +2010_003214__12 +2010_003222__12 +2010_003233__12 +2010_003244__12 +2010_003257__12 +2010_003269__12 +2010_003285__12 +2010_003300__12 +2010_003301__12 +2010_003321__12 +2010_003331__12 +2010_003333__12 +2010_003337__12 +2010_003341__12 +2010_003342__12 +2010_003351__12 +2010_003366__12 +2010_003375__12 +2010_003391__12 +2010_003397__12 +2010_003406__12 +2010_003411__12 +2010_003436__12 +2010_003439__12 +2010_003465__12 +2010_003477__12 +2010_003479__12 +2010_003493__12 +2010_003513__12 +2010_003522__12 +2010_003535__12 +2010_003554__12 +2010_003560__12 +2010_003568__12 +2010_003582__12 +2010_003588__12 +2010_003629__12 +2010_003643__12 +2010_003653__12 +2010_003673__12 +2010_003690__12 +2010_003709__12 +2010_003721__12 +2010_003734__12 +2010_003735__12 +2010_003742__12 +2010_003784__12 +2010_003791__12 +2010_003826__12 +2010_003837__12 +2010_003852__12 +2010_003879__12 +2010_003884__12 +2010_003891__12 +2010_003894__12 +2010_003920__12 +2010_003938__12 +2010_003958__12 +2010_003961__12 +2010_003980__12 +2010_003987__12 +2010_003999__12 +2010_004002__12 +2010_004017__12 +2010_004021__12 +2010_004036__12 +2010_004052__12 +2010_004064__12 +2010_004065__12 +2010_004089__12 +2010_004108__12 +2010_004121__12 +2010_004129__12 +2010_004161__12 +2010_004162__12 +2010_004163__12 +2010_004179__12 +2010_004211__12 +2010_004216__12 +2010_004224__12 +2010_004227__12 +2010_004248__12 +2010_004279__12 +2010_004296__12 +2010_004297__12 +2010_004304__12 +2010_004311__12 +2010_004327__12 +2010_004339__12 +2010_004362__12 +2010_004366__12 +2010_004370__12 +2010_004387__12 +2010_004417__12 +2010_004445__12 +2010_004450__12 +2010_004461__12 +2010_004476__12 +2010_004492__12 +2010_004493__12 +2010_004505__12 +2010_004511__12 +2010_004586__12 +2010_004620__12 +2010_004631__12 +2010_004642__12 +2010_004656__12 +2010_004672__12 +2010_004679__12 +2010_004681__12 +2010_004683__12 +2010_004722__12 +2010_004730__12 +2010_004733__12 +2010_004738__12 +2010_004749__12 +2010_004750__12 +2010_004756__12 +2010_004760__12 +2010_004770__12 +2010_004792__12 +2010_004797__12 +2010_004804__12 +2010_004807__12 +2010_004830__12 +2010_004832__12 +2010_004849__12 +2010_004896__12 +2010_004900__12 +2010_004901__12 +2010_004908__12 +2010_004916__12 +2010_004919__12 +2010_004928__12 +2010_004931__12 +2010_004945__12 +2010_004957__12 +2010_004959__12 +2010_004967__12 +2010_004970__12 +2010_004989__12 +2010_005008__12 +2010_005011__12 +2010_005018__12 +2010_005033__12 +2010_005042__12 +2010_005049__12 +2010_005060__12 +2010_005064__12 +2010_005083__12 +2010_005090__12 +2010_005109__12 +2010_005134__12 +2010_005138__12 +2010_005149__12 +2010_005169__12 +2010_005201__12 +2010_005215__12 +2010_005229__12 +2010_005242__12 +2010_005246__12 +2010_005253__12 +2010_005274__12 +2010_005292__12 +2010_005306__12 +2010_005312__12 +2010_005330__12 +2010_005331__12 +2010_005361__12 +2010_005375__12 +2010_005384__12 +2010_005391__12 +2010_005406__12 +2010_005419__12 +2010_005434__12 +2010_005475__12 +2010_005491__12 +2010_005522__12 +2010_005532__12 +2010_005548__12 +2010_005561__12 +2010_005584__12 +2010_005585__12 +2010_005594__12 +2010_005604__12 +2010_005612__12 +2010_005614__12 +2010_005615__12 +2010_005625__12 +2010_005628__12 +2010_005629__12 +2010_005637__12 +2010_005640__12 +2010_005657__12 +2010_005658__12 +2010_005665__12 +2010_005669__12 +2010_005671__12 +2010_005681__12 +2010_005716__12 +2010_005735__12 +2010_005738__12 +2010_005747__12 +2010_005753__12 +2010_005775__12 +2010_005777__12 +2010_005794__12 +2010_005796__12 +2010_005807__12 +2010_005815__12 +2010_005826__12 +2010_005870__12 +2010_005874__12 +2010_005882__12 +2010_005885__12 +2010_005898__12 +2010_005914__12 +2010_005929__12 +2010_005937__12 +2010_005943__12 +2010_005948__12 +2010_005981__12 +2010_005982__12 +2010_005995__12 +2010_006009__12 +2010_006035__12 +2010_006042__12 +2010_006057__12 +2010_006066__12 +2010_006076__12 +2010_006079__12 +2010_006086__12 +2011_000083__12 +2011_000122__12 +2011_000202__12 +2011_000216__12 +2011_000307__12 +2011_000317__12 +2011_000428__12 +2011_000449__12 +2011_000542__12 +2011_000577__12 +2011_000578__12 +2011_000709__12 +2011_000772__12 +2011_000788__12 +2011_000961__12 +2011_000981__12 +2011_001054__12 +2011_001127__12 +2011_001144__12 +2011_001198__12 +2011_001284__12 +2011_001288__12 +2011_001389__12 +2011_001464__12 +2011_001547__12 +2011_001608__12 +2011_001662__12 +2011_001727__12 +2011_001805__12 +2011_001855__12 +2011_001877__12 +2011_001944__12 +2011_002044__12 +2011_002137__12 +2011_002239__12 +2011_002300__12 +2011_002398__12 +2011_002414__12 +2011_002433__12 +2011_002491__12 +2011_002612__12 +2011_002717__12 +2011_002765__12 +2011_002823__12 +2011_002833__12 +2011_002900__12 +2011_002908__12 +2011_002911__12 +2011_003010__12 +2011_003151__12 +2011_003236__12 +2007_000392__13 +2007_000836__13 +2007_000904__13 +2007_001420__13 +2007_001724__13 +2007_001960__13 +2007_002273__13 +2007_003189__13 +2007_003889__13 +2007_004537__13 +2007_005248__13 +2007_006134__13 +2007_006151__13 +2007_006445__13 +2007_008142__13 +2007_008307__13 +2007_008526__13 +2007_009665__13 +2007_009807__13 +2008_000008__13 +2008_000076__13 +2008_000119__13 +2008_000141__13 +2008_000142__13 +2008_000177__13 +2008_000219__13 +2008_000371__13 +2008_000403__13 +2008_000428__13 +2008_000499__13 +2008_000552__13 +2008_000696__13 +2008_000783__13 +2008_000880__13 +2008_000912__13 +2008_001031__13 +2008_001068__13 +2008_001112__13 +2008_001171__13 +2008_001218__13 +2008_001235__13 +2008_001278__13 +2008_001382__13 +2008_001464__13 +2008_001542__13 +2008_001544__13 +2008_001645__13 +2008_001729__13 +2008_001934__13 +2008_002172__13 +2008_002338__13 +2008_002411__13 +2008_002459__13 +2008_002509__13 +2008_002589__13 +2008_002665__13 +2008_002666__13 +2008_002686__13 +2008_002697__13 +2008_002762__13 +2008_002806__13 +2008_002860__13 +2008_002866__13 +2008_002993__13 +2008_003018__13 +2008_003020__13 +2008_003043__13 +2008_003055__13 +2008_003182__13 +2008_003242__13 +2008_003272__13 +2008_003283__13 +2008_003316__13 +2008_003326__13 +2008_003344__13 +2008_003447__13 +2008_003596__13 +2008_003677__13 +2008_003800__13 +2008_003802__13 +2008_003805__13 +2008_004125__13 +2008_004166__13 +2008_004195__13 +2008_004354__13 +2008_004430__13 +2008_004470__13 +2008_004476__13 +2008_004533__13 +2008_004539__13 +2008_004545__13 +2008_004546__13 +2008_004584__13 +2008_004649__13 +2008_004696__13 +2008_004764__13 +2008_004868__13 +2008_004892__13 +2008_004911__13 +2008_005166__13 +2008_005168__13 +2008_005215__13 +2008_005304__13 +2008_005313__13 +2008_005327__13 +2008_005365__13 +2008_005369__13 +2008_005396__13 +2008_005408__13 +2008_005431__13 +2008_005473__13 +2008_005485__13 +2008_005498__13 +2008_005501__13 +2008_005558__13 +2008_005582__13 +2008_005589__13 +2008_005668__13 +2008_005682__13 +2008_005702__13 +2008_005705__13 +2008_005726__13 +2008_005791__13 +2008_005792__13 +2008_005818__13 +2008_005928__13 +2008_005934__13 +2008_006028__13 +2008_006087__13 +2008_006096__13 +2008_006235__13 +2008_006317__13 +2008_006390__13 +2008_006397__13 +2008_006429__13 +2008_006434__13 +2008_006448__13 +2008_006613__13 +2008_006646__13 +2008_006650__13 +2008_006758__13 +2008_006779__13 +2008_006797__13 +2008_006909__13 +2008_006912__13 +2008_006953__13 +2008_006991__13 +2008_007004__13 +2008_007237__13 +2008_007277__13 +2008_007319__13 +2008_007424__13 +2008_007444__13 +2008_007514__13 +2008_007524__13 +2008_007531__13 +2008_007576__13 +2008_007588__13 +2008_007612__13 +2008_007625__13 +2008_007816__13 +2008_007897__13 +2008_007998__13 +2008_008011__13 +2008_008012__13 +2008_008031__13 +2008_008034__13 +2008_008040__13 +2008_008052__13 +2008_008075__13 +2008_008147__13 +2008_008193__13 +2008_008212__13 +2008_008229__13 +2008_008262__13 +2008_008279__13 +2008_008318__13 +2008_008356__13 +2008_008366__13 +2008_008395__13 +2008_008403__13 +2008_008431__13 +2008_008453__13 +2008_008476__13 +2008_008487__13 +2008_008488__13 +2008_008500__13 +2008_008533__13 +2009_000001__13 +2009_000017__13 +2009_000068__13 +2009_000100__13 +2009_000169__13 +2009_000248__13 +2009_000268__13 +2009_000293__13 +2009_000312__13 +2009_000409__13 +2009_000529__13 +2009_000547__13 +2009_000565__13 +2009_000576__13 +2009_000603__13 +2009_000642__13 +2009_000691__13 +2009_000793__13 +2009_000823__13 +2009_000843__13 +2009_000852__13 +2009_000934__13 +2009_001013__13 +2009_001036__13 +2009_001055__13 +2009_001110__13 +2009_001147__13 +2009_001180__13 +2009_001184__13 +2009_001196__13 +2009_001323__13 +2009_001355__13 +2009_001448__13 +2009_001457__13 +2009_001500__13 +2009_001539__13 +2009_001635__13 +2009_001636__13 +2009_001653__13 +2009_001664__13 +2009_001724__13 +2009_001749__13 +2009_001837__13 +2009_001885__13 +2009_001890__13 +2009_001911__13 +2009_002037__13 +2009_002046__13 +2009_002112__13 +2009_002151__13 +2009_002182__13 +2009_002230__13 +2009_002231__13 +2009_002305__13 +2009_002339__13 +2009_002416__13 +2009_002420__13 +2009_002477__13 +2009_002515__13 +2009_002565__13 +2009_002684__13 +2009_002697__13 +2009_002750__13 +2009_002882__13 +2009_002883__13 +2009_002957__13 +2009_003082__13 +2009_003083__13 +2009_003127__13 +2009_003128__13 +2009_003232__13 +2009_003261__13 +2009_003294__13 +2009_003305__13 +2009_003338__13 +2009_003365__13 +2009_003445__13 +2009_003530__13 +2009_003533__13 +2009_003565__13 +2009_003572__13 +2009_003686__13 +2009_003725__13 +2009_003734__13 +2009_003739__13 +2009_003768__13 +2009_003775__13 +2009_003822__13 +2009_003855__13 +2009_003897__13 +2009_003976__13 +2009_003982__13 +2009_004058__13 +2009_004105__13 +2009_004118__13 +2009_004134__13 +2009_004262__13 +2009_004279__13 +2009_004454__13 +2009_004530__13 +2009_004543__13 +2009_004606__13 +2009_004623__13 +2009_004662__13 +2009_004790__13 +2009_004815__13 +2009_004897__13 +2009_005000__13 +2009_005035__13 +2009_005130__13 +2009_005161__13 +2009_005269__13 +2010_000071__13 +2010_000095__13 +2010_000138__13 +2010_000139__13 +2010_000273__13 +2010_000283__13 +2010_000296__13 +2010_000388__13 +2010_000413__13 +2010_000446__13 +2010_000461__13 +2010_000613__13 +2010_000717__13 +2010_000749__13 +2010_000838__13 +2010_000849__13 +2010_000923__13 +2010_001077__13 +2010_001184__13 +2010_001225__13 +2010_001325__13 +2010_001329__13 +2010_001412__13 +2010_001503__13 +2010_001595__13 +2010_001675__13 +2010_001676__13 +2010_001759__13 +2010_001797__13 +2010_001829__13 +2010_001852__13 +2010_001856__13 +2010_002168__13 +2010_002183__13 +2010_002221__13 +2010_002283__13 +2010_002340__13 +2010_002459__13 +2010_002615__13 +2010_002734__13 +2010_002822__13 +2010_002948__13 +2010_003192__13 +2010_003270__13 +2010_003280__13 +2010_003297__13 +2010_003380__13 +2010_003490__13 +2010_003573__13 +2010_003576__13 +2010_003645__13 +2010_003714__13 +2010_003890__13 +2010_004045__13 +2010_004092__13 +2010_004107__13 +2010_004145__13 +2010_004160__13 +2010_004228__13 +2010_004247__13 +2010_004325__13 +2010_004351__13 +2010_004363__13 +2010_004374__13 +2010_004477__13 +2010_004542__13 +2010_005019__13 +2010_005107__13 +2010_005243__13 +2010_005270__13 +2010_005338__13 +2010_005376__13 +2010_005417__13 +2010_005452__13 +2010_005467__13 +2010_005471__13 +2010_005484__13 +2010_005595__13 +2010_005733__13 +2010_005748__13 +2010_005784__13 +2010_005919__13 +2010_006062__13 +2011_000022__13 +2011_000043__13 +2011_000147__13 +2011_000210__13 +2011_000343__13 +2011_000370__13 +2011_000418__13 +2011_000519__13 +2011_000538__13 +2011_000594__13 +2011_000612__13 +2011_000628__13 +2011_000675__13 +2011_000748__13 +2011_000840__13 +2011_000893__13 +2011_000930__13 +2011_000982__13 +2011_001080__13 +2011_001169__13 +2011_001245__13 +2011_001253__13 +2011_001335__13 +2011_001537__13 +2011_001544__13 +2011_001625__13 +2011_001656__13 +2011_001815__13 +2011_001825__13 +2011_001893__13 +2011_001956__13 +2011_002047__13 +2011_002063__13 +2011_002159__13 +2011_002341__13 +2011_002448__13 +2011_002520__13 +2011_002555__13 +2011_002583__13 +2011_002694__13 +2011_002697__13 +2011_002884__13 +2011_002962__13 +2011_002999__13 +2011_003184__13 +2011_003244__13 +2011_003253__13 +2007_000364__14 +2007_000733__14 +2007_000822__14 +2007_001709__14 +2007_002105__14 +2007_002488__14 +2007_004003__14 +2007_005314__14 +2007_005878__14 +2007_005951__14 +2007_005989__14 +2007_006585__14 +2007_007355__14 +2007_007783__14 +2007_007902__14 +2007_008801__14 +2007_008994__14 +2007_009139__14 +2007_009209__14 +2008_000082__14 +2008_000109__14 +2008_000143__14 +2008_000144__14 +2008_000203__14 +2008_000328__14 +2008_000378__14 +2008_000393__14 +2008_000516__14 +2008_000545__14 +2008_000547__14 +2008_000563__14 +2008_000689__14 +2008_000806__14 +2008_000854__14 +2008_000858__14 +2008_000981__14 +2008_001007__14 +2008_001119__14 +2008_001158__14 +2008_001168__14 +2008_001177__14 +2008_001406__14 +2008_001525__14 +2008_001631__14 +2008_001643__14 +2008_001690__14 +2008_001691__14 +2008_001699__14 +2008_001727__14 +2008_002115__14 +2008_002123__14 +2008_002191__14 +2008_002202__14 +2008_002209__14 +2008_002370__14 +2008_002374__14 +2008_002696__14 +2008_002752__14 +2008_002772__14 +2008_002774__14 +2008_002804__14 +2008_002926__14 +2008_002948__14 +2008_002955__14 +2008_002972__14 +2008_003053__14 +2008_003151__14 +2008_003152__14 +2008_003249__14 +2008_003269__14 +2008_003320__14 +2008_003323__14 +2008_003334__14 +2008_003394__14 +2008_003429__14 +2008_003479__14 +2008_003590__14 +2008_003618__14 +2008_003637__14 +2008_003769__14 +2008_003820__14 +2008_003827__14 +2008_003844__14 +2008_003892__14 +2008_003939__14 +2008_004084__14 +2008_004288__14 +2008_004365__14 +2008_004371__14 +2008_004549__14 +2008_004554__14 +2008_004568__14 +2008_004599__14 +2008_004611__14 +2008_004615__14 +2008_004617__14 +2008_004668__14 +2008_004703__14 +2008_004729__14 +2008_004786__14 +2008_004822__14 +2008_004832__14 +2008_004838__14 +2008_004940__14 +2008_004970__14 +2008_005000__14 +2008_005032__14 +2008_005033__14 +2008_005139__14 +2008_005213__14 +2008_005297__14 +2008_005374__14 +2008_005423__14 +2008_005427__14 +2008_005429__14 +2008_005593__14 +2008_005603__14 +2008_005790__14 +2008_005893__14 +2008_005902__14 +2008_006017__14 +2008_006042__14 +2008_006152__14 +2008_006205__14 +2008_006242__14 +2008_006323__14 +2008_006345__14 +2008_006441__14 +2008_006458__14 +2008_006491__14 +2008_006517__14 +2008_006549__14 +2008_006567__14 +2008_006686__14 +2008_006708__14 +2008_006776__14 +2008_006857__14 +2008_006879__14 +2008_006889__14 +2008_006898__14 +2008_006944__14 +2008_006987__14 +2008_006989__14 +2008_007054__14 +2008_007075__14 +2008_007090__14 +2008_007138__14 +2008_007181__14 +2008_007184__14 +2008_007190__14 +2008_007241__14 +2008_007264__14 +2008_007313__14 +2008_007325__14 +2008_007336__14 +2008_007344__14 +2008_007443__14 +2008_007459__14 +2008_007485__14 +2008_007488__14 +2008_007558__14 +2008_007565__14 +2008_007646__14 +2008_007716__14 +2008_007739__14 +2008_007746__14 +2008_007935__14 +2008_007942__14 +2008_007955__14 +2008_008018__14 +2008_008058__14 +2008_008148__14 +2008_008154__14 +2008_008177__14 +2008_008218__14 +2008_008227__14 +2008_008246__14 +2008_008288__14 +2008_008324__14 +2008_008416__14 +2008_008450__14 +2008_008497__14 +2008_008511__14 +2008_008525__14 +2008_008552__14 +2009_000119__14 +2009_000131__14 +2009_000184__14 +2009_000223__14 +2009_000233__14 +2009_000276__14 +2009_000341__14 +2009_000343__14 +2009_000402__14 +2009_000515__14 +2009_000602__14 +2009_000611__14 +2009_000637__14 +2009_000676__14 +2009_000719__14 +2009_000726__14 +2009_000858__14 +2009_000882__14 +2009_000894__14 +2009_000897__14 +2009_001061__14 +2009_001121__14 +2009_001135__14 +2009_001140__14 +2009_001151__14 +2009_001206__14 +2009_001253__14 +2009_001320__14 +2009_001427__14 +2009_001443__14 +2009_001542__14 +2009_001638__14 +2009_001645__14 +2009_001646__14 +2009_001657__14 +2009_001677__14 +2009_001750__14 +2009_001755__14 +2009_001759__14 +2009_001764__14 +2009_001805__14 +2009_001827__14 +2009_001973__14 +2009_001984__14 +2009_001990__14 +2009_002077__14 +2009_002129__14 +2009_002131__14 +2009_002180__14 +2009_002216__14 +2009_002240__14 +2009_002335__14 +2009_002399__14 +2009_002434__14 +2009_002569__14 +2009_002595__14 +2009_002665__14 +2009_002746__14 +2009_002763__14 +2009_002784__14 +2009_002806__14 +2009_002824__14 +2009_002894__14 +2009_002902__14 +2009_002993__14 +2009_003012__14 +2009_003068__14 +2009_003090__14 +2009_003234__14 +2009_003238__14 +2009_003346__14 +2009_003351__14 +2009_003411__14 +2009_003416__14 +2009_003522__14 +2009_003539__14 +2009_003598__14 +2009_003639__14 +2009_003722__14 +2009_003743__14 +2009_003747__14 +2009_003802__14 +2009_003820__14 +2009_003846__14 +2009_003873__14 +2009_003944__14 +2009_004093__14 +2009_004117__14 +2009_004202__14 +2009_004207__14 +2009_004241__14 +2009_004249__14 +2009_004338__14 +2009_004368__14 +2009_004502__14 +2009_004643__14 +2009_004651__14 +2009_004756__14 +2009_004759__14 +2009_004845__14 +2009_004849__14 +2009_004857__14 +2009_004868__14 +2009_005086__14 +2009_005147__14 +2009_005155__14 +2009_005247__14 +2010_000027__14 +2010_000035__14 +2010_000255__14 +2010_000262__14 +2010_000371__14 +2010_000433__14 +2010_000490__14 +2010_000495__14 +2010_000510__14 +2010_000583__14 +2010_000678__14 +2010_000688__14 +2010_000695__14 +2010_000744__14 +2010_000806__14 +2010_000927__14 +2010_000991__14 +2010_000995__14 +2010_001234__14 +2010_001383__14 +2010_001472__14 +2010_001614__14 +2010_001810__14 +2010_001828__14 +2010_001923__14 +2010_002128__14 +2010_002149__14 +2010_002172__14 +2010_002408__14 +2010_002435__14 +2010_002452__14 +2010_002469__14 +2010_002629__14 +2010_002747__14 +2010_002991__14 +2010_003159__14 +2010_003200__14 +2010_003303__14 +2010_003314__14 +2010_003316__14 +2010_003432__14 +2010_003469__14 +2010_003529__14 +2010_003605__14 +2010_003608__14 +2010_003635__14 +2010_003695__14 +2010_003736__14 +2010_003745__14 +2010_003856__14 +2010_004028__14 +2010_004033__14 +2010_004066__14 +2010_004119__14 +2010_004143__14 +2010_004191__14 +2010_004289__14 +2010_004336__14 +2010_004422__14 +2010_004517__14 +2010_004624__14 +2010_004766__14 +2010_004848__14 +2010_004854__14 +2010_004894__14 +2010_004995__14 +2010_005100__14 +2010_005136__14 +2010_005182__14 +2010_005199__14 +2010_005272__14 +2010_005346__14 +2010_005359__14 +2010_005393__14 +2010_005429__14 +2010_005497__14 +2010_005572__14 +2010_005619__14 +2010_005684__14 +2010_005721__14 +2010_005731__14 +2010_005823__14 +2010_005849__14 +2010_005876__14 +2010_005997__14 +2010_006028__14 +2010_006051__14 +2011_000034__14 +2011_000053__14 +2011_000060__14 +2011_000214__14 +2011_000220__14 +2011_000233__14 +2011_000314__14 +2011_000315__14 +2011_000332__14 +2011_000397__14 +2011_000487__14 +2011_000560__14 +2011_000637__14 +2011_000651__14 +2011_000652__14 +2011_000770__14 +2011_000898__14 +2011_000927__14 +2011_000940__14 +2011_001022__14 +2011_001040__14 +2011_001123__14 +2011_001203__14 +2011_001216__14 +2011_001266__14 +2011_001286__14 +2011_001290__14 +2011_001498__14 +2011_001508__14 +2011_001524__14 +2011_001605__14 +2011_001629__14 +2011_001663__14 +2011_001765__14 +2011_001799__14 +2011_001876__14 +2011_002039__14 +2011_002062__14 +2011_002093__14 +2011_002097__14 +2011_002102__14 +2011_002107__14 +2011_002116__14 +2011_002236__14 +2011_002252__14 +2011_002357__14 +2011_002402__14 +2011_002413__14 +2011_002479__14 +2011_002598__14 +2011_002760__14 +2011_002786__14 +2011_002796__14 +2011_002921__14 +2011_002949__14 +2011_002988__14 +2011_003013__14 +2011_003027__14 +2011_003057__14 +2011_003063__14 +2011_003078__14 +2007_000170__15 +2007_000364__15 +2007_000392__15 +2007_000480__15 +2007_000504__15 +2007_000515__15 +2007_000733__15 +2007_000793__15 +2007_000836__15 +2007_000904__15 +2007_001185__15 +2007_001340__15 +2007_001420__15 +2007_001709__15 +2007_001857__15 +2007_002055__15 +2007_002105__15 +2007_002120__15 +2007_002142__15 +2007_002293__15 +2007_002361__15 +2007_002488__15 +2007_002545__15 +2007_002611__15 +2007_002639__15 +2007_002668__15 +2007_002895__15 +2007_002914__15 +2007_002954__15 +2007_003118__15 +2007_003189__15 +2007_003191__15 +2007_003205__15 +2007_003431__15 +2007_003529__15 +2007_003541__15 +2007_003580__15 +2007_003815__15 +2007_003889__15 +2007_003910__15 +2007_004289__15 +2007_004291__15 +2007_004328__15 +2007_004476__15 +2007_004481__15 +2007_004537__15 +2007_004769__15 +2007_005064__15 +2007_005086__15 +2007_005124__15 +2007_005144__15 +2007_005227__15 +2007_005273__15 +2007_005368__15 +2007_005430__15 +2007_005702__15 +2007_005790__15 +2007_005797__15 +2007_005859__15 +2007_005951__15 +2007_005988__15 +2007_005989__15 +2007_006004__15 +2007_006151__15 +2007_006317__15 +2007_006409__15 +2007_006445__15 +2007_006477__15 +2007_006483__15 +2007_006585__15 +2007_006615__15 +2007_006661__15 +2007_006699__15 +2007_007048__15 +2007_007154__15 +2007_007355__15 +2007_007398__15 +2007_007447__15 +2007_007480__15 +2007_007649__15 +2007_007783__15 +2007_007878__15 +2007_007891__15 +2007_007902__15 +2007_008043__15 +2007_008072__15 +2007_008142__15 +2007_008218__15 +2007_008307__15 +2007_008407__15 +2007_008526__15 +2007_008778__15 +2007_008801__15 +2007_008932__15 +2007_008945__15 +2007_009139__15 +2007_009295__15 +2007_009327__15 +2007_009436__15 +2007_009554__15 +2007_009649__15 +2007_009665__15 +2007_009779__15 +2007_009901__15 +2007_009950__15 +2008_000008__15 +2008_000023__15 +2008_000026__15 +2008_000034__15 +2008_000036__15 +2008_000041__15 +2008_000051__15 +2008_000067__15 +2008_000082__15 +2008_000090__15 +2008_000096__15 +2008_000109__15 +2008_000128__15 +2008_000132__15 +2008_000138__15 +2008_000142__15 +2008_000143__15 +2008_000144__15 +2008_000176__15 +2008_000194__15 +2008_000195__15 +2008_000199__15 +2008_000202__15 +2008_000203__15 +2008_000204__15 +2008_000207__15 +2008_000217__15 +2008_000222__15 +2008_000235__15 +2008_000236__15 +2008_000243__15 +2008_000246__15 +2008_000252__15 +2008_000259__15 +2008_000260__15 +2008_000261__15 +2008_000264__15 +2008_000266__15 +2008_000268__15 +2008_000275__15 +2008_000277__15 +2008_000278__15 +2008_000283__15 +2008_000284__15 +2008_000289__15 +2008_000290__15 +2008_000297__15 +2008_000298__15 +2008_000307__15 +2008_000311__15 +2008_000313__15 +2008_000316__15 +2008_000330__15 +2008_000338__15 +2008_000340__15 +2008_000343__15 +2008_000346__15 +2008_000354__15 +2008_000364__15 +2008_000365__15 +2008_000367__15 +2008_000376__15 +2008_000380__15 +2008_000381__15 +2008_000382__15 +2008_000383__15 +2008_000392__15 +2008_000393__15 +2008_000398__15 +2008_000406__15 +2008_000407__15 +2008_000408__15 +2008_000413__15 +2008_000415__15 +2008_000416__15 +2008_000418__15 +2008_000421__15 +2008_000422__15 +2008_000423__15 +2008_000424__15 +2008_000426__15 +2008_000432__15 +2008_000435__15 +2008_000436__15 +2008_000442__15 +2008_000443__15 +2008_000445__15 +2008_000446__15 +2008_000447__15 +2008_000448__15 +2008_000452__15 +2008_000455__15 +2008_000461__15 +2008_000473__15 +2008_000475__15 +2008_000480__15 +2008_000481__15 +2008_000489__15 +2008_000492__15 +2008_000493__15 +2008_000496__15 +2008_000511__15 +2008_000514__15 +2008_000516__15 +2008_000522__15 +2008_000527__15 +2008_000531__15 +2008_000535__15 +2008_000540__15 +2008_000541__15 +2008_000544__15 +2008_000547__15 +2008_000548__15 +2008_000552__15 +2008_000553__15 +2008_000558__15 +2008_000561__15 +2008_000562__15 +2008_000563__15 +2008_000568__15 +2008_000569__15 +2008_000572__15 +2008_000578__15 +2008_000579__15 +2008_000583__15 +2008_000584__15 +2008_000588__15 +2008_000599__15 +2008_000605__15 +2008_000607__15 +2008_000609__15 +2008_000613__15 +2008_000614__15 +2008_000615__15 +2008_000620__15 +2008_000622__15 +2008_000623__15 +2008_000628__15 +2008_000629__15 +2008_000634__15 +2008_000636__15 +2008_000640__15 +2008_000645__15 +2008_000646__15 +2008_000647__15 +2008_000648__15 +2008_000650__15 +2008_000652__15 +2008_000655__15 +2008_000656__15 +2008_000660__15 +2008_000672__15 +2008_000674__15 +2008_000677__15 +2008_000683__15 +2008_000689__15 +2008_000690__15 +2008_000691__15 +2008_000694__15 +2008_000699__15 +2008_000704__15 +2008_000705__15 +2008_000706__15 +2008_000714__15 +2008_000719__15 +2008_000723__15 +2008_000726__15 +2008_000727__15 +2008_000729__15 +2008_000732__15 +2008_000733__15 +2008_000734__15 +2008_000737__15 +2008_000740__15 +2008_000742__15 +2008_000745__15 +2008_000748__15 +2008_000753__15 +2008_000758__15 +2008_000764__15 +2008_000769__15 +2008_000775__15 +2008_000776__15 +2008_000777__15 +2008_000778__15 +2008_000780__15 +2008_000783__15 +2008_000787__15 +2008_000792__15 +2008_000796__15 +2008_000801__15 +2008_000803__15 +2008_000806__15 +2008_000814__15 +2008_000815__15 +2008_000825__15 +2008_000828__15 +2008_000829__15 +2008_000834__15 +2008_000835__15 +2008_000837__15 +2008_000841__15 +2008_000842__15 +2008_000844__15 +2008_000847__15 +2008_000851__15 +2008_000864__15 +2008_000867__15 +2008_000868__15 +2008_000870__15 +2008_000873__15 +2008_000875__15 +2008_000878__15 +2008_000880__15 +2008_000881__15 +2008_000885__15 +2008_000887__15 +2008_000899__15 +2008_000902__15 +2008_000904__15 +2008_000908__15 +2008_000910__15 +2008_000917__15 +2008_000922__15 +2008_000924__15 +2008_000928__15 +2008_000936__15 +2008_000940__15 +2008_000941__15 +2008_000942__15 +2008_000950__15 +2008_000953__15 +2008_000956__15 +2008_000965__15 +2008_000970__15 +2008_000972__15 +2008_000976__15 +2008_000979__15 +2008_000982__15 +2008_000984__15 +2008_000985__15 +2008_000987__15 +2008_000993__15 +2008_001007__15 +2008_001009__15 +2008_001012__15 +2008_001021__15 +2008_001023__15 +2008_001024__15 +2008_001026__15 +2008_001031__15 +2008_001034__15 +2008_001035__15 +2008_001036__15 +2008_001039__15 +2008_001046__15 +2008_001048__15 +2008_001052__15 +2008_001055__15 +2008_001057__15 +2008_001060__15 +2008_001063__15 +2008_001066__15 +2008_001073__15 +2008_001075__15 +2008_001081__15 +2008_001083__15 +2008_001089__15 +2008_001090__15 +2008_001092__15 +2008_001098__15 +2008_001099__15 +2008_001104__15 +2008_001106__15 +2008_001112__15 +2008_001113__15 +2008_001115__15 +2008_001119__15 +2008_001121__15 +2008_001122__15 +2008_001134__15 +2008_001139__15 +2008_001140__15 +2008_001147__15 +2008_001154__15 +2008_001166__15 +2008_001167__15 +2008_001171__15 +2008_001182__15 +2008_001183__15 +2008_001188__15 +2008_001189__15 +2008_001190__15 +2008_001192__15 +2008_001199__15 +2008_001202__15 +2008_001205__15 +2008_001206__15 +2008_001215__15 +2008_001219__15 +2008_001221__15 +2008_001223__15 +2008_001225__15 +2008_001226__15 +2008_001227__15 +2008_001230__15 +2008_001235__15 +2008_001241__15 +2008_001245__15 +2008_001248__15 +2008_001255__15 +2008_001257__15 +2008_001263__15 +2008_001271__15 +2008_001272__15 +2008_001275__15 +2008_001284__15 +2008_001294__15 +2008_001296__15 +2008_001299__15 +2008_001301__15 +2008_001302__15 +2008_001304__15 +2008_001306__15 +2008_001307__15 +2008_001310__15 +2008_001312__15 +2008_001314__15 +2008_001318__15 +2008_001320__15 +2008_001322__15 +2008_001325__15 +2008_001329__15 +2008_001333__15 +2008_001338__15 +2008_001340__15 +2008_001344__15 +2008_001346__15 +2008_001349__15 +2008_001351__15 +2008_001353__15 +2008_001356__15 +2008_001358__15 +2008_001359__15 +2008_001366__15 +2008_001367__15 +2008_001369__15 +2008_001373__15 +2008_001375__15 +2008_001376__15 +2008_001382__15 +2008_001383__15 +2008_001385__15 +2008_001388__15 +2008_001390__15 +2008_001391__15 +2008_001395__15 +2008_001401__15 +2008_001402__15 +2008_001408__15 +2008_001413__15 +2008_001419__15 +2008_001420__15 +2008_001427__15 +2008_001429__15 +2008_001431__15 +2008_001434__15 +2008_001437__15 +2008_001440__15 +2008_001444__15 +2008_001451__15 +2008_001454__15 +2008_001455__15 +2008_001461__15 +2008_001462__15 +2008_001464__15 +2008_001466__15 +2008_001467__15 +2008_001470__15 +2008_001475__15 +2008_001481__15 +2008_001482__15 +2008_001486__15 +2008_001488__15 +2008_001493__15 +2008_001494__15 +2008_001501__15 +2008_001503__15 +2008_001510__15 +2008_001520__15 +2008_001525__15 +2008_001527__15 +2008_001533__15 +2008_001534__15 +2008_001536__15 +2008_001538__15 +2008_001539__15 +2008_001540__15 +2008_001549__15 +2008_001550__15 +2008_001551__15 +2008_001553__15 +2008_001563__15 +2008_001564__15 +2008_001566__15 +2008_001574__15 +2008_001576__15 +2008_001577__15 +2008_001582__15 +2008_001589__15 +2008_001590__15 +2008_001591__15 +2008_001593__15 +2008_001594__15 +2008_001596__15 +2008_001598__15 +2008_001602__15 +2008_001605__15 +2008_001607__15 +2008_001609__15 +2008_001613__15 +2008_001615__15 +2008_001617__15 +2008_001619__15 +2008_001620__15 +2008_001622__15 +2008_001624__15 +2008_001631__15 +2008_001636__15 +2008_001638__15 +2008_001643__15 +2008_001645__15 +2008_001648__15 +2008_001649__15 +2008_001652__15 +2008_001655__15 +2008_001659__15 +2008_001660__15 +2008_001661__15 +2008_001666__15 +2008_001667__15 +2008_001668__15 +2008_001669__15 +2008_001670__15 +2008_001673__15 +2008_001680__15 +2008_001690__15 +2008_001692__15 +2008_001697__15 +2008_001699__15 +2008_001702__15 +2008_001706__15 +2008_001709__15 +2008_001710__15 +2008_001712__15 +2008_001722__15 +2008_001724__15 +2008_001730__15 +2008_001731__15 +2008_001735__15 +2008_001736__15 +2008_001737__15 +2008_001742__15 +2008_001745__15 +2008_001746__15 +2008_001751__15 +2008_001757__15 +2008_001758__15 +2008_001761__15 +2008_001763__15 +2008_001764__15 +2008_001765__15 +2008_001769__15 +2008_001773__15 +2008_001775__15 +2008_001782__15 +2008_001784__15 +2008_001789__15 +2008_001791__15 +2008_001797__15 +2008_001802__15 +2008_001806__15 +2008_001808__15 +2008_001811__15 +2008_001812__15 +2008_001816__15 +2008_001825__15 +2008_001832__15 +2008_001834__15 +2008_001837__15 +2008_001838__15 +2008_001841__15 +2008_001842__15 +2008_001845__15 +2008_001852__15 +2008_001854__15 +2008_001856__15 +2008_001860__15 +2008_001863__15 +2008_001867__15 +2008_001881__15 +2008_001882__15 +2008_001888__15 +2008_001894__15 +2008_001899__15 +2008_001903__15 +2008_001905__15 +2008_001907__15 +2008_001909__15 +2008_001910__15 +2008_001914__15 +2008_001919__15 +2008_001928__15 +2008_001929__15 +2008_001930__15 +2008_001932__15 +2008_001937__15 +2008_001946__15 +2008_001947__15 +2008_001951__15 +2008_001955__15 +2008_001957__15 +2008_001958__15 +2008_001965__15 +2008_001967__15 +2008_001969__15 +2008_001970__15 +2008_001977__15 +2008_001980__15 +2008_001982__15 +2008_001986__15 +2008_001989__15 +2008_001998__15 +2008_002001__15 +2008_002002__15 +2008_002003__15 +2008_002007__15 +2008_002009__15 +2008_002011__15 +2008_002013__15 +2008_002017__15 +2008_002021__15 +2008_002023__15 +2008_002031__15 +2008_002032__15 +2008_002035__15 +2008_002036__15 +2008_002037__15 +2008_002039__15 +2008_002042__15 +2008_002047__15 +2008_002052__15 +2008_002056__15 +2008_002058__15 +2008_002061__15 +2008_002067__15 +2008_002069__15 +2008_002079__15 +2008_002080__15 +2008_002084__15 +2008_002086__15 +2008_002088__15 +2008_002092__15 +2008_002093__15 +2008_002094__15 +2008_002096__15 +2008_002098__15 +2008_002099__15 +2008_002103__15 +2008_002107__15 +2008_002112__15 +2008_002113__15 +2008_002114__15 +2008_002115__15 +2008_002117__15 +2008_002123__15 +2008_002124__15 +2008_002129__15 +2008_002132__15 +2008_002140__15 +2008_002144__15 +2008_002145__15 +2008_002146__15 +2008_002148__15 +2008_002150__15 +2008_002155__15 +2008_002156__15 +2008_002162__15 +2008_002167__15 +2008_002169__15 +2008_002172__15 +2008_002176__15 +2008_002181__15 +2008_002185__15 +2008_002191__15 +2008_002199__15 +2008_002206__15 +2008_002210__15 +2008_002220__15 +2008_002222__15 +2008_002231__15 +2008_002236__15 +2008_002243__15 +2008_002244__15 +2008_002247__15 +2008_002248__15 +2008_002251__15 +2008_002259__15 +2008_002262__15 +2008_002272__15 +2008_002280__15 +2008_002283__15 +2008_002296__15 +2008_002299__15 +2008_002304__15 +2008_002305__15 +2008_002311__15 +2008_002312__15 +2008_002314__15 +2008_002317__15 +2008_002321__15 +2008_002322__15 +2008_002324__15 +2008_002325__15 +2008_002328__15 +2008_002330__15 +2008_002331__15 +2008_002340__15 +2008_002344__15 +2008_002347__15 +2008_002356__15 +2008_002357__15 +2008_002361__15 +2008_002362__15 +2008_002365__15 +2008_002366__15 +2008_002374__15 +2008_002377__15 +2008_002378__15 +2008_002384__15 +2008_002403__15 +2008_002404__15 +2008_002405__15 +2008_002411__15 +2008_002414__15 +2008_002418__15 +2008_002419__15 +2008_002422__15 +2008_002424__15 +2008_002425__15 +2008_002428__15 +2008_002434__15 +2008_002436__15 +2008_002439__15 +2008_002445__15 +2008_002446__15 +2008_002448__15 +2008_002451__15 +2008_002457__15 +2008_002458__15 +2008_002459__15 +2008_002470__15 +2008_002477__15 +2008_002483__15 +2008_002484__15 +2008_002485__15 +2008_002487__15 +2008_002491__15 +2008_002494__15 +2008_002499__15 +2008_002501__15 +2008_002502__15 +2008_002506__15 +2008_002514__15 +2008_002523__15 +2008_002524__15 +2008_002526__15 +2008_002527__15 +2008_002533__15 +2008_002540__15 +2008_002541__15 +2008_002542__15 +2008_002543__15 +2008_002549__15 +2008_002555__15 +2008_002558__15 +2008_002562__15 +2008_002564__15 +2008_002566__15 +2008_002567__15 +2008_002568__15 +2008_002574__15 +2008_002575__15 +2008_002576__15 +2008_002578__15 +2008_002579__15 +2008_002590__15 +2008_002598__15 +2008_002599__15 +2008_002601__15 +2008_002603__15 +2008_002606__15 +2008_002610__15 +2008_002613__15 +2008_002616__15 +2008_002621__15 +2008_002622__15 +2008_002631__15 +2008_002634__15 +2008_002638__15 +2008_002639__15 +2008_002640__15 +2008_002641__15 +2008_002643__15 +2008_002645__15 +2008_002649__15 +2008_002652__15 +2008_002653__15 +2008_002662__15 +2008_002665__15 +2008_002666__15 +2008_002670__15 +2008_002674__15 +2008_002675__15 +2008_002676__15 +2008_002677__15 +2008_002679__15 +2008_002700__15 +2008_002712__15 +2008_002714__15 +2008_002715__15 +2008_002716__15 +2008_002718__15 +2008_002720__15 +2008_002725__15 +2008_002728__15 +2008_002730__15 +2008_002733__15 +2008_002735__15 +2008_002736__15 +2008_002738__15 +2008_002741__15 +2008_002746__15 +2008_002750__15 +2008_002751__15 +2008_002752__15 +2008_002753__15 +2008_002756__15 +2008_002760__15 +2008_002762__15 +2008_002768__15 +2008_002774__15 +2008_002776__15 +2008_002783__15 +2008_002787__15 +2008_002789__15 +2008_002791__15 +2008_002792__15 +2008_002794__15 +2008_002795__15 +2008_002801__15 +2008_002804__15 +2008_002808__15 +2008_002809__15 +2008_002811__15 +2008_002813__15 +2008_002817__15 +2008_002820__15 +2008_002823__15 +2008_002829__15 +2008_002830__15 +2008_002831__15 +2008_002838__15 +2008_002842__15 +2008_002843__15 +2008_002848__15 +2008_002852__15 +2008_002854__15 +2008_002856__15 +2008_002857__15 +2008_002868__15 +2008_002869__15 +2008_002870__15 +2008_002873__15 +2008_002879__15 +2008_002880__15 +2008_002882__15 +2008_002885__15 +2008_002892__15 +2008_002894__15 +2008_002897__15 +2008_002906__15 +2008_002908__15 +2008_002909__15 +2008_002910__15 +2008_002917__15 +2008_002920__15 +2008_002922__15 +2008_002930__15 +2008_002931__15 +2008_002932__15 +2008_002946__15 +2008_002947__15 +2008_002951__15 +2008_002954__15 +2008_002955__15 +2008_002956__15 +2008_002960__15 +2008_002966__15 +2008_002968__15 +2008_002971__15 +2008_002973__15 +2008_002984__15 +2008_002985__15 +2008_002988__15 +2008_002992__15 +2008_002997__15 +2008_002999__15 +2008_003001__15 +2008_003008__15 +2008_003013__15 +2008_003015__15 +2008_003017__15 +2008_003018__15 +2008_003020__15 +2008_003039__15 +2008_003043__15 +2008_003056__15 +2008_003061__15 +2008_003062__15 +2008_003065__15 +2008_003075__15 +2008_003079__15 +2008_003089__15 +2008_003090__15 +2008_003093__15 +2008_003095__15 +2008_003099__15 +2008_003100__15 +2008_003107__15 +2008_003120__15 +2008_003122__15 +2008_003127__15 +2008_003128__15 +2008_003132__15 +2008_003134__15 +2008_003136__15 +2008_003140__15 +2008_003143__15 +2008_003144__15 +2008_003146__15 +2008_003152__15 +2008_003154__15 +2008_003157__15 +2008_003181__15 +2008_003182__15 +2008_003186__15 +2008_003191__15 +2008_003193__15 +2008_003202__15 +2008_003203__15 +2008_003205__15 +2008_003209__15 +2008_003220__15 +2008_003224__15 +2008_003228__15 +2008_003231__15 +2008_003242__15 +2008_003245__15 +2008_003248__15 +2008_003249__15 +2008_003251__15 +2008_003265__15 +2008_003271__15 +2008_003277__15 +2008_003278__15 +2008_003280__15 +2008_003283__15 +2008_003286__15 +2008_003287__15 +2008_003288__15 +2008_003289__15 +2008_003290__15 +2008_003295__15 +2008_003304__15 +2008_003305__15 +2008_003311__15 +2008_003316__15 +2008_003318__15 +2008_003320__15 +2008_003329__15 +2008_003334__15 +2008_003335__15 +2008_003336__15 +2008_003338__15 +2008_003342__15 +2008_003344__15 +2008_003351__15 +2008_003359__15 +2008_003361__15 +2008_003384__15 +2008_003393__15 +2008_003395__15 +2008_003405__15 +2008_003407__15 +2008_003409__15 +2008_003414__15 +2008_003417__15 +2008_003418__15 +2008_003424__15 +2008_003429__15 +2008_003432__15 +2008_003433__15 +2008_003437__15 +2008_003442__15 +2008_003448__15 +2008_003452__15 +2008_003453__15 +2008_003463__15 +2008_003464__15 +2008_003467__15 +2008_003469__15 +2008_003472__15 +2008_003478__15 +2008_003479__15 +2008_003482__15 +2008_003483__15 +2008_003488__15 +2008_003493__15 +2008_003496__15 +2008_003507__15 +2008_003510__15 +2008_003514__15 +2008_003515__15 +2008_003521__15 +2008_003522__15 +2008_003523__15 +2008_003526__15 +2008_003531__15 +2008_003533__15 +2008_003534__15 +2008_003542__15 +2008_003544__15 +2008_003545__15 +2008_003565__15 +2008_003578__15 +2008_003587__15 +2008_003592__15 +2008_003593__15 +2008_003596__15 +2008_003598__15 +2008_003604__15 +2008_003608__15 +2008_003609__15 +2008_003610__15 +2008_003611__15 +2008_003613__15 +2008_003617__15 +2008_003619__15 +2008_003626__15 +2008_003635__15 +2008_003638__15 +2008_003650__15 +2008_003653__15 +2008_003658__15 +2008_003667__15 +2008_003671__15 +2008_003672__15 +2008_003674__15 +2008_003675__15 +2008_003677__15 +2008_003681__15 +2008_003682__15 +2008_003684__15 +2008_003685__15 +2008_003689__15 +2008_003697__15 +2008_003706__15 +2008_003707__15 +2008_003712__15 +2008_003720__15 +2008_003721__15 +2008_003722__15 +2008_003732__15 +2008_003745__15 +2008_003748__15 +2008_003749__15 +2008_003754__15 +2008_003755__15 +2008_003756__15 +2008_003761__15 +2008_003762__15 +2008_003764__15 +2008_003766__15 +2008_003768__15 +2008_003769__15 +2008_003773__15 +2008_003775__15 +2008_003779__15 +2008_003780__15 +2008_003781__15 +2008_003791__15 +2008_003793__15 +2008_003796__15 +2008_003800__15 +2008_003801__15 +2008_003802__15 +2008_003805__15 +2008_003813__15 +2008_003815__15 +2008_003819__15 +2008_003820__15 +2008_003825__15 +2008_003829__15 +2008_003830__15 +2008_003843__15 +2008_003844__15 +2008_003847__15 +2008_003860__15 +2008_003864__15 +2008_003866__15 +2008_003868__15 +2008_003871__15 +2008_003873__15 +2008_003881__15 +2008_003882__15 +2008_003883__15 +2008_003884__15 +2008_003888__15 +2008_003891__15 +2008_003892__15 +2008_003908__15 +2008_003914__15 +2008_003915__15 +2008_003916__15 +2008_003920__15 +2008_003921__15 +2008_003925__15 +2008_003929__15 +2008_003933__15 +2008_003940__15 +2008_003945__15 +2008_003947__15 +2008_003951__15 +2008_003956__15 +2008_003958__15 +2008_003962__15 +2008_003965__15 +2008_003967__15 +2008_003971__15 +2008_003978__15 +2008_003983__15 +2008_003984__15 +2008_003985__15 +2008_003995__15 +2008_003996__15 +2008_004002__15 +2008_004003__15 +2008_004016__15 +2008_004017__15 +2008_004018__15 +2008_004037__15 +2008_004040__15 +2008_004044__15 +2008_004045__15 +2008_004046__15 +2008_004053__15 +2008_004054__15 +2008_004056__15 +2008_004064__15 +2008_004066__15 +2008_004071__15 +2008_004076__15 +2008_004077__15 +2008_004081__15 +2008_004084__15 +2008_004088__15 +2008_004090__15 +2008_004092__15 +2008_004102__15 +2008_004105__15 +2008_004113__15 +2008_004123__15 +2008_004126__15 +2008_004134__15 +2008_004135__15 +2008_004137__15 +2008_004142__15 +2008_004148__15 +2008_004161__15 +2008_004176__15 +2008_004182__15 +2008_004188__15 +2008_004195__15 +2008_004196__15 +2008_004198__15 +2008_004201__15 +2008_004203__15 +2008_004205__15 +2008_004208__15 +2008_004213__15 +2008_004217__15 +2008_004221__15 +2008_004230__15 +2008_004231__15 +2008_004242__15 +2008_004243__15 +2008_004245__15 +2008_004246__15 +2008_004247__15 +2008_004258__15 +2008_004263__15 +2008_004269__15 +2008_004270__15 +2008_004274__15 +2008_004278__15 +2008_004287__15 +2008_004288__15 +2008_004301__15 +2008_004307__15 +2008_004308__15 +2008_004314__15 +2008_004318__15 +2008_004321__15 +2008_004325__15 +2008_004328__15 +2008_004330__15 +2008_004333__15 +2008_004342__15 +2008_004344__15 +2008_004353__15 +2008_004354__15 +2008_004361__15 +2008_004365__15 +2008_004371__15 +2008_004372__15 +2008_004380__15 +2008_004389__15 +2008_004391__15 +2008_004398__15 +2008_004403__15 +2008_004417__15 +2008_004418__15 +2008_004419__15 +2008_004426__15 +2008_004431__15 +2008_004435__15 +2008_004436__15 +2008_004438__15 +2008_004439__15 +2008_004441__15 +2008_004455__15 +2008_004457__15 +2008_004459__15 +2008_004464__15 +2008_004470__15 +2008_004471__15 +2008_004476__15 +2008_004478__15 +2008_004479__15 +2008_004482__15 +2008_004492__15 +2008_004493__15 +2008_004499__15 +2008_004501__15 +2008_004504__15 +2008_004510__15 +2008_004512__15 +2008_004513__15 +2008_004515__15 +2008_004519__15 +2008_004522__15 +2008_004533__15 +2008_004534__15 +2008_004538__15 +2008_004541__15 +2008_004544__15 +2008_004546__15 +2008_004549__15 +2008_004553__15 +2008_004554__15 +2008_004564__15 +2008_004567__15 +2008_004568__15 +2008_004574__15 +2008_004602__15 +2008_004603__15 +2008_004606__15 +2008_004611__15 +2008_004616__15 +2008_004636__15 +2008_004647__15 +2008_004649__15 +2008_004665__15 +2008_004666__15 +2008_004677__15 +2008_004678__15 +2008_004692__15 +2008_004703__15 +2008_004706__15 +2008_004707__15 +2008_004711__15 +2008_004713__15 +2008_004729__15 +2008_004740__15 +2008_004745__15 +2008_004749__15 +2008_004756__15 +2008_004764__15 +2008_004774__15 +2008_004778__15 +2008_004781__15 +2008_004784__15 +2008_004807__15 +2008_004814__15 +2008_004832__15 +2008_004834__15 +2008_004841__15 +2008_004849__15 +2008_004851__15 +2008_004858__15 +2008_004862__15 +2008_004866__15 +2008_004868__15 +2008_004872__15 +2008_004874__15 +2008_004875__15 +2008_004881__15 +2008_004892__15 +2008_004894__15 +2008_004926__15 +2008_004930__15 +2008_004937__15 +2008_004938__15 +2008_004940__15 +2008_004948__15 +2008_004950__15 +2008_004961__15 +2008_004966__15 +2008_004976__15 +2008_004979__15 +2008_004984__15 +2008_004991__15 +2008_005000__15 +2008_005016__15 +2008_005032__15 +2008_005033__15 +2008_005035__15 +2008_005043__15 +2008_005045__15 +2008_005046__15 +2008_005063__15 +2008_005066__15 +2008_005072__15 +2008_005090__15 +2008_005094__15 +2008_005098__15 +2008_005101__15 +2008_005107__15 +2008_005110__15 +2008_005111__15 +2008_005133__15 +2008_005139__15 +2008_005146__15 +2008_005150__15 +2008_005151__15 +2008_005156__15 +2008_005168__15 +2008_005171__15 +2008_005178__15 +2008_005190__15 +2008_005201__15 +2008_005213__15 +2008_005215__15 +2008_005218__15 +2008_005231__15 +2008_005236__15 +2008_005243__15 +2008_005251__15 +2008_005253__15 +2008_005271__15 +2008_005276__15 +2008_005277__15 +2008_005294__15 +2008_005295__15 +2008_005315__15 +2008_005323__15 +2008_005324__15 +2008_005336__15 +2008_005359__15 +2008_005360__15 +2008_005365__15 +2008_005374__15 +2008_005395__15 +2008_005408__15 +2008_005412__15 +2008_005414__15 +2008_005423__15 +2008_005427__15 +2008_005429__15 +2008_005431__15 +2008_005444__15 +2008_005447__15 +2008_005472__15 +2008_005484__15 +2008_005485__15 +2008_005490__15 +2008_005494__15 +2008_005498__15 +2008_005500__15 +2008_005501__15 +2008_005519__15 +2008_005527__15 +2008_005549__15 +2008_005552__15 +2008_005553__15 +2008_005558__15 +2008_005563__15 +2008_005566__15 +2008_005567__15 +2008_005570__15 +2008_005582__15 +2008_005588__15 +2008_005599__15 +2008_005600__15 +2008_005609__15 +2008_005612__15 +2008_005616__15 +2008_005625__15 +2008_005635__15 +2008_005646__15 +2008_005650__15 +2008_005652__15 +2008_005677__15 +2008_005698__15 +2008_005701__15 +2008_005707__15 +2008_005726__15 +2008_005732__15 +2008_005737__15 +2008_005742__15 +2008_005758__15 +2008_005761__15 +2008_005779__15 +2008_005788__15 +2008_005790__15 +2008_005791__15 +2008_005792__15 +2008_005794__15 +2008_005796__15 +2008_005808__15 +2008_005825__15 +2008_005832__15 +2008_005838__15 +2008_005846__15 +2008_005848__15 +2008_005850__15 +2008_005855__15 +2008_005867__15 +2008_005869__15 +2008_005877__15 +2008_005881__15 +2008_005884__15 +2008_005902__15 +2008_005921__15 +2008_005923__15 +2008_005928__15 +2008_005934__15 +2008_005937__15 +2008_005939__15 +2008_005957__15 +2008_005972__15 +2008_005976__15 +2008_005978__15 +2008_005991__15 +2008_006021__15 +2008_006024__15 +2008_006027__15 +2008_006032__15 +2008_006039__15 +2008_006041__15 +2008_006042__15 +2008_006047__15 +2008_006052__15 +2008_006059__15 +2008_006064__15 +2008_006071__15 +2008_006076__15 +2008_006078__15 +2008_006087__15 +2008_006088__15 +2008_006096__15 +2008_006102__15 +2008_006117__15 +2008_006120__15 +2008_006121__15 +2008_006124__15 +2008_006129__15 +2008_006135__15 +2008_006145__15 +2008_006148__15 +2008_006151__15 +2008_006152__15 +2008_006154__15 +2008_006181__15 +2008_006188__15 +2008_006195__15 +2008_006205__15 +2008_006222__15 +2008_006235__15 +2008_006250__15 +2008_006253__15 +2008_006256__15 +2008_006257__15 +2008_006267__15 +2008_006272__15 +2008_006273__15 +2008_006295__15 +2008_006311__15 +2008_006317__15 +2008_006331__15 +2008_006370__15 +2008_006377__15 +2008_006382__15 +2008_006390__15 +2008_006392__15 +2008_006397__15 +2008_006410__15 +2008_006424__15 +2008_006433__15 +2008_006441__15 +2008_006448__15 +2008_006458__15 +2008_006467__15 +2008_006491__15 +2008_006519__15 +2008_006530__15 +2008_006549__15 +2008_006567__15 +2008_006570__15 +2008_006588__15 +2008_006609__15 +2008_006610__15 +2008_006611__15 +2008_006613__15 +2008_006616__15 +2008_006625__15 +2008_006631__15 +2008_006642__15 +2008_006646__15 +2008_006650__15 +2008_006663__15 +2008_006684__15 +2008_006705__15 +2008_006708__15 +2008_006710__15 +2008_006716__15 +2008_006730__15 +2008_006731__15 +2008_006732__15 +2008_006733__15 +2008_006750__15 +2008_006758__15 +2008_006776__15 +2008_006779__15 +2008_006796__15 +2008_006798__15 +2008_006807__15 +2008_006813__15 +2008_006816__15 +2008_006820__15 +2008_006825__15 +2008_006833__15 +2008_006834__15 +2008_006844__15 +2008_006847__15 +2008_006855__15 +2008_006857__15 +2008_006863__15 +2008_006864__15 +2008_006870__15 +2008_006872__15 +2008_006879__15 +2008_006880__15 +2008_006881__15 +2008_006890__15 +2008_006892__15 +2008_006912__15 +2008_006933__15 +2008_006941__15 +2008_006944__15 +2008_006952__15 +2008_006953__15 +2008_006960__15 +2008_006969__15 +2008_006979__15 +2008_006987__15 +2008_006989__15 +2008_006991__15 +2008_006998__15 +2008_007019__15 +2008_007032__15 +2008_007038__15 +2008_007043__15 +2008_007058__15 +2008_007061__15 +2008_007067__15 +2008_007084__15 +2008_007090__15 +2008_007091__15 +2008_007097__15 +2008_007103__15 +2008_007112__15 +2008_007119__15 +2008_007129__15 +2008_007133__15 +2008_007138__15 +2008_007146__15 +2008_007147__15 +2008_007163__15 +2008_007168__15 +2008_007169__15 +2008_007181__15 +2008_007182__15 +2008_007185__15 +2008_007190__15 +2008_007195__15 +2008_007207__15 +2008_007218__15 +2008_007222__15 +2008_007223__15 +2008_007242__15 +2008_007254__15 +2008_007280__15 +2008_007281__15 +2008_007285__15 +2008_007286__15 +2008_007291__15 +2008_007293__15 +2008_007325__15 +2008_007334__15 +2008_007336__15 +2008_007339__15 +2008_007348__15 +2008_007393__15 +2008_007409__15 +2008_007410__15 +2008_007421__15 +2008_007435__15 +2008_007443__15 +2008_007446__15 +2008_007452__15 +2008_007459__15 +2008_007470__15 +2008_007476__15 +2008_007478__15 +2008_007485__15 +2008_007509__15 +2008_007510__15 +2008_007514__15 +2008_007524__15 +2008_007525__15 +2008_007537__15 +2008_007556__15 +2008_007565__15 +2008_007567__15 +2008_007576__15 +2008_007584__15 +2008_007585__15 +2008_007586__15 +2008_007593__15 +2008_007612__15 +2008_007617__15 +2008_007625__15 +2008_007643__15 +2008_007646__15 +2008_007665__15 +2008_007676__15 +2008_007682__15 +2008_007692__15 +2008_007696__15 +2008_007701__15 +2008_007706__15 +2008_007710__15 +2008_007716__15 +2008_007717__15 +2008_007729__15 +2008_007741__15 +2008_007742__15 +2008_007745__15 +2008_007746__15 +2008_007757__15 +2008_007761__15 +2008_007766__15 +2008_007768__15 +2008_007770__15 +2008_007787__15 +2008_007827__15 +2008_007833__15 +2008_007837__15 +2008_007841__15 +2008_007843__15 +2008_007852__15 +2008_007853__15 +2008_007861__15 +2008_007864__15 +2008_007872__15 +2008_007882__15 +2008_007895__15 +2008_007902__15 +2008_007904__15 +2008_007917__15 +2008_007923__15 +2008_007928__15 +2008_007931__15 +2008_007933__15 +2008_007935__15 +2008_007950__15 +2008_007955__15 +2008_007962__15 +2008_007966__15 +2008_007970__15 +2008_007986__15 +2008_007990__15 +2008_007993__15 +2008_008029__15 +2008_008034__15 +2008_008058__15 +2008_008064__15 +2008_008066__15 +2008_008069__15 +2008_008072__15 +2008_008083__15 +2008_008084__15 +2008_008091__15 +2008_008093__15 +2008_008095__15 +2008_008097__15 +2008_008098__15 +2008_008112__15 +2008_008113__15 +2008_008116__15 +2008_008122__15 +2008_008147__15 +2008_008154__15 +2008_008155__15 +2008_008162__15 +2008_008184__15 +2008_008185__15 +2008_008190__15 +2008_008192__15 +2008_008210__15 +2008_008212__15 +2008_008217__15 +2008_008218__15 +2008_008229__15 +2008_008246__15 +2008_008262__15 +2008_008266__15 +2008_008272__15 +2008_008288__15 +2008_008307__15 +2008_008315__15 +2008_008319__15 +2008_008320__15 +2008_008325__15 +2008_008337__15 +2008_008357__15 +2008_008366__15 +2008_008368__15 +2008_008382__15 +2008_008388__15 +2008_008395__15 +2008_008403__15 +2008_008410__15 +2008_008431__15 +2008_008433__15 +2008_008440__15 +2008_008443__15 +2008_008444__15 +2008_008450__15 +2008_008474__15 +2008_008479__15 +2008_008487__15 +2008_008488__15 +2008_008500__15 +2008_008511__15 +2008_008517__15 +2008_008519__15 +2008_008522__15 +2008_008524__15 +2008_008526__15 +2008_008528__15 +2008_008530__15 +2008_008547__15 +2008_008549__15 +2008_008550__15 +2008_008552__15 +2008_008564__15 +2008_008567__15 +2008_008572__15 +2008_008579__15 +2008_008588__15 +2008_008591__15 +2008_008593__15 +2008_008598__15 +2008_008600__15 +2008_008619__15 +2008_008624__15 +2008_008637__15 +2008_008641__15 +2008_008652__15 +2008_008671__15 +2008_008676__15 +2008_008691__15 +2008_008697__15 +2008_008705__15 +2008_008708__15 +2008_008717__15 +2008_008718__15 +2008_008744__15 +2008_008745__15 +2008_008755__15 +2008_008757__15 +2009_000010__15 +2009_000017__15 +2009_000021__15 +2009_000056__15 +2009_000058__15 +2009_000059__15 +2009_000063__15 +2009_000067__15 +2009_000072__15 +2009_000119__15 +2009_000128__15 +2009_000140__15 +2009_000141__15 +2009_000145__15 +2009_000157__15 +2009_000164__15 +2009_000168__15 +2009_000169__15 +2009_000171__15 +2009_000182__15 +2009_000183__15 +2009_000237__15 +2009_000247__15 +2009_000248__15 +2009_000249__15 +2009_000253__15 +2009_000257__15 +2009_000277__15 +2009_000282__15 +2009_000288__15 +2009_000291__15 +2009_000305__15 +2009_000322__15 +2009_000337__15 +2009_000340__15 +2009_000343__15 +2009_000375__15 +2009_000378__15 +2009_000379__15 +2009_000397__15 +2009_000405__15 +2009_000411__15 +2009_000430__15 +2009_000449__15 +2009_000453__15 +2009_000456__15 +2009_000461__15 +2009_000463__15 +2009_000464__15 +2009_000474__15 +2009_000496__15 +2009_000505__15 +2009_000511__15 +2009_000525__15 +2009_000529__15 +2009_000539__15 +2009_000544__15 +2009_000546__15 +2009_000547__15 +2009_000550__15 +2009_000552__15 +2009_000563__15 +2009_000565__15 +2009_000577__15 +2009_000595__15 +2009_000604__15 +2009_000606__15 +2009_000615__15 +2009_000617__15 +2009_000637__15 +2009_000642__15 +2009_000653__15 +2009_000661__15 +2009_000674__15 +2009_000676__15 +2009_000686__15 +2009_000719__15 +2009_000724__15 +2009_000726__15 +2009_000741__15 +2009_000744__15 +2009_000750__15 +2009_000755__15 +2009_000757__15 +2009_000759__15 +2009_000763__15 +2009_000783__15 +2009_000815__15 +2009_000820__15 +2009_000821__15 +2009_000831__15 +2009_000833__15 +2009_000834__15 +2009_000843__15 +2009_000856__15 +2009_000858__15 +2009_000865__15 +2009_000890__15 +2009_000896__15 +2009_000897__15 +2009_000901__15 +2009_000902__15 +2009_000930__15 +2009_000934__15 +2009_000962__15 +2009_000969__15 +2009_000987__15 +2009_000990__15 +2009_001002__15 +2009_001011__15 +2009_001013__15 +2009_001024__15 +2009_001026__15 +2009_001030__15 +2009_001042__15 +2009_001054__15 +2009_001084__15 +2009_001091__15 +2009_001096__15 +2009_001103__15 +2009_001106__15 +2009_001110__15 +2009_001111__15 +2009_001120__15 +2009_001133__15 +2009_001137__15 +2009_001140__15 +2009_001151__15 +2009_001154__15 +2009_001155__15 +2009_001181__15 +2009_001190__15 +2009_001207__15 +2009_001212__15 +2009_001238__15 +2009_001254__15 +2009_001266__15 +2009_001286__15 +2009_001313__15 +2009_001319__15 +2009_001320__15 +2009_001339__15 +2009_001345__15 +2009_001376__15 +2009_001384__15 +2009_001389__15 +2009_001395__15 +2009_001407__15 +2009_001414__15 +2009_001427__15 +2009_001437__15 +2009_001443__15 +2009_001447__15 +2009_001450__15 +2009_001463__15 +2009_001468__15 +2009_001474__15 +2009_001479__15 +2009_001508__15 +2009_001521__15 +2009_001539__15 +2009_001553__15 +2009_001575__15 +2009_001577__15 +2009_001581__15 +2009_001585__15 +2009_001602__15 +2009_001605__15 +2009_001611__15 +2009_001615__15 +2009_001617__15 +2009_001635__15 +2009_001642__15 +2009_001646__15 +2009_001657__15 +2009_001677__15 +2009_001690__15 +2009_001705__15 +2009_001709__15 +2009_001740__15 +2009_001743__15 +2009_001750__15 +2009_001758__15 +2009_001759__15 +2009_001764__15 +2009_001767__15 +2009_001778__15 +2009_001794__15 +2009_001799__15 +2009_001801__15 +2009_001805__15 +2009_001809__15 +2009_001810__15 +2009_001822__15 +2009_001827__15 +2009_001846__15 +2009_001852__15 +2009_001856__15 +2009_001864__15 +2009_001865__15 +2009_001875__15 +2009_001906__15 +2009_001908__15 +2009_001909__15 +2009_001934__15 +2009_001940__15 +2009_001952__15 +2009_001965__15 +2009_001967__15 +2009_001971__15 +2009_001979__15 +2009_001994__15 +2009_002001__15 +2009_002037__15 +2009_002046__15 +2009_002054__15 +2009_002058__15 +2009_002088__15 +2009_002107__15 +2009_002110__15 +2009_002112__15 +2009_002123__15 +2009_002128__15 +2009_002129__15 +2009_002131__15 +2009_002139__15 +2009_002145__15 +2009_002147__15 +2009_002177__15 +2009_002182__15 +2009_002192__15 +2009_002222__15 +2009_002230__15 +2009_002240__15 +2009_002242__15 +2009_002257__15 +2009_002273__15 +2009_002285__15 +2009_002297__15 +2009_002305__15 +2009_002308__15 +2009_002326__15 +2009_002331__15 +2009_002333__15 +2009_002338__15 +2009_002349__15 +2009_002350__15 +2009_002376__15 +2009_002381__15 +2009_002391__15 +2009_002398__15 +2009_002404__15 +2009_002406__15 +2009_002407__15 +2009_002425__15 +2009_002433__15 +2009_002439__15 +2009_002443__15 +2009_002464__15 +2009_002471__15 +2009_002474__15 +2009_002477__15 +2009_002500__15 +2009_002504__15 +2009_002514__15 +2009_002522__15 +2009_002524__15 +2009_002542__15 +2009_002552__15 +2009_002556__15 +2009_002569__15 +2009_002579__15 +2009_002580__15 +2009_002586__15 +2009_002595__15 +2009_002607__15 +2009_002621__15 +2009_002669__15 +2009_002671__15 +2009_002698__15 +2009_002703__15 +2009_002705__15 +2009_002710__15 +2009_002715__15 +2009_002717__15 +2009_002743__15 +2009_002744__15 +2009_002750__15 +2009_002772__15 +2009_002777__15 +2009_002780__15 +2009_002784__15 +2009_002790__15 +2009_002792__15 +2009_002800__15 +2009_002806__15 +2009_002807__15 +2009_002814__15 +2009_002824__15 +2009_002827__15 +2009_002850__15 +2009_002883__15 +2009_002885__15 +2009_002893__15 +2009_002894__15 +2009_002918__15 +2009_002925__15 +2009_002932__15 +2009_002954__15 +2009_002960__15 +2009_002961__15 +2009_002976__15 +2009_002983__15 +2009_002993__15 +2009_003000__15 +2009_003006__15 +2009_003010__15 +2009_003020__15 +2009_003035__15 +2009_003042__15 +2009_003044__15 +2009_003058__15 +2009_003064__15 +2009_003076__15 +2009_003087__15 +2009_003090__15 +2009_003116__15 +2009_003125__15 +2009_003126__15 +2009_003128__15 +2009_003150__15 +2009_003154__15 +2009_003166__15 +2009_003175__15 +2009_003191__15 +2009_003238__15 +2009_003251__15 +2009_003257__15 +2009_003261__15 +2009_003262__15 +2009_003265__15 +2009_003266__15 +2009_003272__15 +2009_003282__15 +2009_003290__15 +2009_003294__15 +2009_003310__15 +2009_003320__15 +2009_003338__15 +2009_003340__15 +2009_003347__15 +2009_003349__15 +2009_003351__15 +2009_003353__15 +2009_003361__15 +2009_003363__15 +2009_003365__15 +2009_003379__15 +2009_003409__15 +2009_003415__15 +2009_003425__15 +2009_003441__15 +2009_003453__15 +2009_003454__15 +2009_003459__15 +2009_003468__15 +2009_003482__15 +2009_003488__15 +2009_003490__15 +2009_003499__15 +2009_003519__15 +2009_003540__15 +2009_003563__15 +2009_003577__15 +2009_003581__15 +2009_003594__15 +2009_003598__15 +2009_003609__15 +2009_003627__15 +2009_003633__15 +2009_003636__15 +2009_003646__15 +2009_003656__15 +2009_003668__15 +2009_003683__15 +2009_003689__15 +2009_003690__15 +2009_003702__15 +2009_003705__15 +2009_003710__15 +2009_003720__15 +2009_003722__15 +2009_003725__15 +2009_003734__15 +2009_003736__15 +2009_003738__15 +2009_003739__15 +2009_003751__15 +2009_003753__15 +2009_003768__15 +2009_003776__15 +2009_003785__15 +2009_003795__15 +2009_003800__15 +2009_003816__15 +2009_003829__15 +2009_003840__15 +2009_003846__15 +2009_003855__15 +2009_003874__15 +2009_003888__15 +2009_003892__15 +2009_003896__15 +2009_003897__15 +2009_003900__15 +2009_003908__15 +2009_003920__15 +2009_003921__15 +2009_003965__15 +2009_003976__15 +2009_003986__15 +2009_004004__15 +2009_004007__15 +2009_004020__15 +2009_004032__15 +2009_004040__15 +2009_004042__15 +2009_004050__15 +2009_004052__15 +2009_004055__15 +2009_004069__15 +2009_004074__15 +2009_004085__15 +2009_004092__15 +2009_004094__15 +2009_004100__15 +2009_004102__15 +2009_004112__15 +2009_004113__15 +2009_004118__15 +2009_004126__15 +2009_004131__15 +2009_004133__15 +2009_004142__15 +2009_004148__15 +2009_004154__15 +2009_004161__15 +2009_004179__15 +2009_004181__15 +2009_004188__15 +2009_004202__15 +2009_004205__15 +2009_004218__15 +2009_004227__15 +2009_004228__15 +2009_004276__15 +2009_004279__15 +2009_004285__15 +2009_004300__15 +2009_004301__15 +2009_004303__15 +2009_004309__15 +2009_004322__15 +2009_004323__15 +2009_004332__15 +2009_004347__15 +2009_004359__15 +2009_004368__15 +2009_004377__15 +2009_004410__15 +2009_004429__15 +2009_004432__15 +2009_004435__15 +2009_004457__15 +2009_004486__15 +2009_004502__15 +2009_004503__15 +2009_004508__15 +2009_004514__15 +2009_004518__15 +2009_004525__15 +2009_004570__15 +2009_004580__15 +2009_004582__15 +2009_004587__15 +2009_004619__15 +2009_004643__15 +2009_004645__15 +2009_004648__15 +2009_004664__15 +2009_004674__15 +2009_004679__15 +2009_004694__15 +2009_004697__15 +2009_004701__15 +2009_004718__15 +2009_004731__15 +2009_004756__15 +2009_004758__15 +2009_004761__15 +2009_004781__15 +2009_004782__15 +2009_004784__15 +2009_004786__15 +2009_004794__15 +2009_004798__15 +2009_004806__15 +2009_004822__15 +2009_004830__15 +2009_004847__15 +2009_004849__15 +2009_004865__15 +2009_004868__15 +2009_004890__15 +2009_004899__15 +2009_004919__15 +2009_004926__15 +2009_004933__15 +2009_004939__15 +2009_004975__15 +2009_004982__15 +2009_005001__15 +2009_005016__15 +2009_005035__15 +2009_005045__15 +2009_005056__15 +2009_005057__15 +2009_005082__15 +2009_005098__15 +2009_005103__15 +2009_005140__15 +2009_005141__15 +2009_005147__15 +2009_005155__15 +2009_005172__15 +2009_005181__15 +2009_005183__15 +2009_005185__15 +2009_005191__15 +2009_005203__15 +2009_005205__15 +2009_005216__15 +2009_005222__15 +2009_005229__15 +2009_005247__15 +2009_005257__15 +2009_005265__15 +2009_005269__15 +2009_005279__15 +2009_005300__15 +2009_005309__15 +2009_005311__15 +2010_000018__15 +2010_000052__15 +2010_000055__15 +2010_000074__15 +2010_000089__15 +2010_000095__15 +2010_000097__15 +2010_000120__15 +2010_000131__15 +2010_000133__15 +2010_000139__15 +2010_000140__15 +2010_000148__15 +2010_000162__15 +2010_000177__15 +2010_000204__15 +2010_000224__15 +2010_000234__15 +2010_000245__15 +2010_000262__15 +2010_000296__15 +2010_000308__15 +2010_000313__15 +2010_000324__15 +2010_000344__15 +2010_000352__15 +2010_000358__15 +2010_000361__15 +2010_000371__15 +2010_000375__15 +2010_000377__15 +2010_000386__15 +2010_000419__15 +2010_000420__15 +2010_000431__15 +2010_000432__15 +2010_000444__15 +2010_000446__15 +2010_000448__15 +2010_000449__15 +2010_000462__15 +2010_000474__15 +2010_000490__15 +2010_000492__15 +2010_000500__15 +2010_000503__15 +2010_000510__15 +2010_000511__15 +2010_000548__15 +2010_000564__15 +2010_000574__15 +2010_000577__15 +2010_000578__15 +2010_000617__15 +2010_000645__15 +2010_000648__15 +2010_000671__15 +2010_000691__15 +2010_000695__15 +2010_000697__15 +2010_000717__15 +2010_000722__15 +2010_000739__15 +2010_000744__15 +2010_000759__15 +2010_000760__15 +2010_000772__15 +2010_000778__15 +2010_000792__15 +2010_000806__15 +2010_000866__15 +2010_000883__15 +2010_000891__15 +2010_000910__15 +2010_000912__15 +2010_000915__15 +2010_000922__15 +2010_000942__15 +2010_000944__15 +2010_000945__15 +2010_000954__15 +2010_000959__15 +2010_000968__15 +2010_000970__15 +2010_000974__15 +2010_000991__15 +2010_000993__15 +2010_001006__15 +2010_001008__15 +2010_001077__15 +2010_001092__15 +2010_001094__15 +2010_001106__15 +2010_001107__15 +2010_001111__15 +2010_001112__15 +2010_001113__15 +2010_001119__15 +2010_001120__15 +2010_001130__15 +2010_001131__15 +2010_001142__15 +2010_001148__15 +2010_001184__15 +2010_001188__15 +2010_001192__15 +2010_001210__15 +2010_001218__15 +2010_001224__15 +2010_001241__15 +2010_001245__15 +2010_001250__15 +2010_001254__15 +2010_001287__15 +2010_001288__15 +2010_001289__15 +2010_001312__15 +2010_001325__15 +2010_001337__15 +2010_001338__15 +2010_001385__15 +2010_001397__15 +2010_001401__15 +2010_001408__15 +2010_001411__15 +2010_001412__15 +2010_001418__15 +2010_001422__15 +2010_001430__15 +2010_001431__15 +2010_001479__15 +2010_001486__15 +2010_001502__15 +2010_001528__15 +2010_001533__15 +2010_001535__15 +2010_001543__15 +2010_001552__15 +2010_001576__15 +2010_001580__15 +2010_001586__15 +2010_001619__15 +2010_001635__15 +2010_001636__15 +2010_001645__15 +2010_001649__15 +2010_001669__15 +2010_001674__15 +2010_001682__15 +2010_001689__15 +2010_001720__15 +2010_001739__15 +2010_001748__15 +2010_001749__15 +2010_001756__15 +2010_001757__15 +2010_001762__15 +2010_001785__15 +2010_001810__15 +2010_001814__15 +2010_001817__15 +2010_001828__15 +2010_001837__15 +2010_001846__15 +2010_001850__15 +2010_001856__15 +2010_001857__15 +2010_001870__15 +2010_001891__15 +2010_001892__15 +2010_001907__15 +2010_001921__15 +2010_001923__15 +2010_001940__15 +2010_001941__15 +2010_001954__15 +2010_001960__15 +2010_001978__15 +2010_002002__15 +2010_002006__15 +2010_002018__15 +2010_002022__15 +2010_002039__15 +2010_002044__15 +2010_002058__15 +2010_002070__15 +2010_002094__15 +2010_002096__15 +2010_002100__15 +2010_002107__15 +2010_002121__15 +2010_002127__15 +2010_002128__15 +2010_002139__15 +2010_002154__15 +2010_002166__15 +2010_002179__15 +2010_002183__15 +2010_002185__15 +2010_002211__15 +2010_002219__15 +2010_002221__15 +2010_002226__15 +2010_002245__15 +2010_002247__15 +2010_002263__15 +2010_002269__15 +2010_002274__15 +2010_002276__15 +2010_002278__15 +2010_002301__15 +2010_002303__15 +2010_002312__15 +2010_002340__15 +2010_002346__15 +2010_002353__15 +2010_002356__15 +2010_002363__15 +2010_002373__15 +2010_002400__15 +2010_002405__15 +2010_002408__15 +2010_002431__15 +2010_002435__15 +2010_002440__15 +2010_002445__15 +2010_002446__15 +2010_002457__15 +2010_002461__15 +2010_002462__15 +2010_002484__15 +2010_002492__15 +2010_002497__15 +2010_002498__15 +2010_002501__15 +2010_002504__15 +2010_002509__15 +2010_002510__15 +2010_002516__15 +2010_002520__15 +2010_002526__15 +2010_002539__15 +2010_002552__15 +2010_002561__15 +2010_002570__15 +2010_002597__15 +2010_002603__15 +2010_002618__15 +2010_002625__15 +2010_002628__15 +2010_002629__15 +2010_002632__15 +2010_002642__15 +2010_002652__15 +2010_002675__15 +2010_002679__15 +2010_002696__15 +2010_002702__15 +2010_002704__15 +2010_002708__15 +2010_002714__15 +2010_002720__15 +2010_002725__15 +2010_002737__15 +2010_002742__15 +2010_002747__15 +2010_002767__15 +2010_002770__15 +2010_002783__15 +2010_002807__15 +2010_002817__15 +2010_002820__15 +2010_002821__15 +2010_002822__15 +2010_002844__15 +2010_002853__15 +2010_002860__15 +2010_002864__15 +2010_002865__15 +2010_002871__15 +2010_002877__15 +2010_002879__15 +2010_002880__15 +2010_002881__15 +2010_002914__15 +2010_002915__15 +2010_002927__15 +2010_002931__15 +2010_002946__15 +2010_002962__15 +2010_002980__15 +2010_002982__15 +2010_002987__15 +2010_002990__15 +2010_003003__15 +2010_003010__15 +2010_003016__15 +2010_003017__15 +2010_003034__15 +2010_003035__15 +2010_003040__15 +2010_003043__15 +2010_003054__15 +2010_003056__15 +2010_003062__15 +2010_003078__15 +2010_003084__15 +2010_003091__15 +2010_003093__15 +2010_003094__15 +2010_003107__15 +2010_003108__15 +2010_003112__15 +2010_003122__15 +2010_003185__15 +2010_003212__15 +2010_003219__15 +2010_003220__15 +2010_003241__15 +2010_003248__15 +2010_003250__15 +2010_003257__15 +2010_003263__15 +2010_003264__15 +2010_003280__15 +2010_003287__15 +2010_003297__15 +2010_003303__15 +2010_003304__15 +2010_003305__15 +2010_003326__15 +2010_003332__15 +2010_003355__15 +2010_003361__15 +2010_003370__15 +2010_003374__15 +2010_003375__15 +2010_003401__15 +2010_003461__15 +2010_003469__15 +2010_003526__15 +2010_003537__15 +2010_003538__15 +2010_003540__15 +2010_003556__15 +2010_003560__15 +2010_003563__15 +2010_003567__15 +2010_003573__15 +2010_003585__15 +2010_003599__15 +2010_003603__15 +2010_003608__15 +2010_003613__15 +2010_003618__15 +2010_003625__15 +2010_003628__15 +2010_003630__15 +2010_003632__15 +2010_003643__15 +2010_003649__15 +2010_003670__15 +2010_003671__15 +2010_003673__15 +2010_003674__15 +2010_003677__15 +2010_003695__15 +2010_003703__15 +2010_003719__15 +2010_003724__15 +2010_003729__15 +2010_003731__15 +2010_003744__15 +2010_003752__15 +2010_003755__15 +2010_003804__15 +2010_003844__15 +2010_003859__15 +2010_003861__15 +2010_003871__15 +2010_003877__15 +2010_003878__15 +2010_003884__15 +2010_003892__15 +2010_003894__15 +2010_003906__15 +2010_003914__15 +2010_003936__15 +2010_003944__15 +2010_003949__15 +2010_003955__15 +2010_003957__15 +2010_003961__15 +2010_003983__15 +2010_003995__15 +2010_003996__15 +2010_004025__15 +2010_004031__15 +2010_004033__15 +2010_004050__15 +2010_004067__15 +2010_004069__15 +2010_004073__15 +2010_004094__15 +2010_004102__15 +2010_004109__15 +2010_004111__15 +2010_004124__15 +2010_004125__15 +2010_004130__15 +2010_004141__15 +2010_004160__15 +2010_004161__15 +2010_004168__15 +2010_004186__15 +2010_004191__15 +2010_004198__15 +2010_004207__15 +2010_004222__15 +2010_004223__15 +2010_004225__15 +2010_004249__15 +2010_004254__15 +2010_004264__15 +2010_004280__15 +2010_004282__15 +2010_004289__15 +2010_004297__15 +2010_004318__15 +2010_004325__15 +2010_004332__15 +2010_004341__15 +2010_004360__15 +2010_004370__15 +2010_004374__15 +2010_004387__15 +2010_004400__15 +2010_004439__15 +2010_004441__15 +2010_004447__15 +2010_004469__15 +2010_004475__15 +2010_004481__15 +2010_004491__15 +2010_004499__15 +2010_004506__15 +2010_004523__15 +2010_004542__15 +2010_004554__15 +2010_004560__15 +2010_004567__15 +2010_004577__15 +2010_004581__15 +2010_004594__15 +2010_004596__15 +2010_004597__15 +2010_004618__15 +2010_004665__15 +2010_004666__15 +2010_004672__15 +2010_004686__15 +2010_004692__15 +2010_004698__15 +2010_004703__15 +2010_004704__15 +2010_004708__15 +2010_004710__15 +2010_004722__15 +2010_004728__15 +2010_004741__15 +2010_004743__15 +2010_004775__15 +2010_004778__15 +2010_004782__15 +2010_004786__15 +2010_004838__15 +2010_004844__15 +2010_004854__15 +2010_004877__15 +2010_004878__15 +2010_004889__15 +2010_004901__15 +2010_004910__15 +2010_004922__15 +2010_004937__15 +2010_004942__15 +2010_004952__15 +2010_004973__15 +2010_004983__15 +2010_004987__15 +2010_004995__15 +2010_005005__15 +2010_005018__15 +2010_005019__15 +2010_005026__15 +2010_005031__15 +2010_005044__15 +2010_005049__15 +2010_005071__15 +2010_005079__15 +2010_005110__15 +2010_005111__15 +2010_005127__15 +2010_005141__15 +2010_005148__15 +2010_005170__15 +2010_005188__15 +2010_005199__15 +2010_005222__15 +2010_005226__15 +2010_005236__15 +2010_005243__15 +2010_005273__15 +2010_005276__15 +2010_005293__15 +2010_005299__15 +2010_005303__15 +2010_005314__15 +2010_005318__15 +2010_005327__15 +2010_005338__15 +2010_005346__15 +2010_005349__15 +2010_005350__15 +2010_005352__15 +2010_005369__15 +2010_005374__15 +2010_005379__15 +2010_005384__15 +2010_005393__15 +2010_005403__15 +2010_005405__15 +2010_005409__15 +2010_005410__15 +2010_005466__15 +2010_005467__15 +2010_005471__15 +2010_005480__15 +2010_005489__15 +2010_005493__15 +2010_005500__15 +2010_005512__15 +2010_005518__15 +2010_005527__15 +2010_005532__15 +2010_005538__15 +2010_005542__15 +2010_005543__15 +2010_005570__15 +2010_005572__15 +2010_005578__15 +2010_005585__15 +2010_005610__15 +2010_005619__15 +2010_005620__15 +2010_005628__15 +2010_005629__15 +2010_005646__15 +2010_005654__15 +2010_005657__15 +2010_005666__15 +2010_005681__15 +2010_005684__15 +2010_005697__15 +2010_005700__15 +2010_005723__15 +2010_005733__15 +2010_005748__15 +2010_005758__15 +2010_005764__15 +2010_005768__15 +2010_005794__15 +2010_005810__15 +2010_005826__15 +2010_005830__15 +2010_005835__15 +2010_005838__15 +2010_005840__15 +2010_005847__15 +2010_005867__15 +2010_005875__15 +2010_005876__15 +2010_005884__15 +2010_005897__15 +2010_005898__15 +2010_005904__15 +2010_005906__15 +2010_005936__15 +2010_005954__15 +2010_005960__15 +2010_005967__15 +2010_005972__15 +2010_005996__15 +2010_006000__15 +2010_006004__15 +2010_006010__15 +2010_006037__15 +2010_006050__15 +2010_006051__15 +2010_006061__15 +2010_006079__15 +2011_000003__15 +2011_000006__15 +2011_000007__15 +2011_000009__15 +2011_000016__15 +2011_000030__15 +2011_000038__15 +2011_000041__15 +2011_000053__15 +2011_000069__15 +2011_000071__15 +2011_000072__15 +2011_000084__15 +2011_000137__15 +2011_000142__15 +2011_000146__15 +2011_000147__15 +2011_000161__15 +2011_000182__15 +2011_000192__15 +2011_000194__15 +2011_000196__15 +2011_000206__15 +2011_000208__15 +2011_000213__15 +2011_000214__15 +2011_000220__15 +2011_000241__15 +2011_000252__15 +2011_000258__15 +2011_000273__15 +2011_000278__15 +2011_000282__15 +2011_000304__15 +2011_000314__15 +2011_000317__15 +2011_000320__15 +2011_000321__15 +2011_000322__15 +2011_000332__15 +2011_000342__15 +2011_000344__15 +2011_000361__15 +2011_000362__15 +2011_000370__15 +2011_000375__15 +2011_000391__15 +2011_000398__15 +2011_000408__15 +2011_000413__15 +2011_000416__15 +2011_000420__15 +2011_000427__15 +2011_000434__15 +2011_000445__15 +2011_000453__15 +2011_000471__15 +2011_000475__15 +2011_000477__15 +2011_000485__15 +2011_000492__15 +2011_000505__15 +2011_000509__15 +2011_000520__15 +2011_000534__15 +2011_000538__15 +2011_000557__15 +2011_000559__15 +2011_000567__15 +2011_000575__15 +2011_000577__15 +2011_000589__15 +2011_000594__15 +2011_000600__15 +2011_000628__15 +2011_000630__15 +2011_000637__15 +2011_000642__15 +2011_000646__15 +2011_000657__15 +2011_000675__15 +2011_000679__15 +2011_000683__15 +2011_000684__15 +2011_000688__15 +2011_000689__15 +2011_000692__15 +2011_000711__15 +2011_000713__15 +2011_000724__15 +2011_000725__15 +2011_000731__15 +2011_000745__15 +2011_000748__15 +2011_000753__15 +2011_000767__15 +2011_000768__15 +2011_000772__15 +2011_000774__15 +2011_000785__15 +2011_000800__15 +2011_000806__15 +2011_000823__15 +2011_000827__15 +2011_000828__15 +2011_000829__15 +2011_000851__15 +2011_000858__15 +2011_000859__15 +2011_000882__15 +2011_000885__15 +2011_000899__15 +2011_000908__15 +2011_000927__15 +2011_000932__15 +2011_000933__15 +2011_000934__15 +2011_000947__15 +2011_000954__15 +2011_000957__15 +2011_000977__15 +2011_000979__15 +2011_000983__15 +2011_000990__15 +2011_000991__15 +2011_000996__15 +2011_001001__15 +2011_001008__15 +2011_001009__15 +2011_001010__15 +2011_001023__15 +2011_001027__15 +2011_001030__15 +2011_001032__15 +2011_001036__15 +2011_001055__15 +2011_001056__15 +2011_001058__15 +2011_001062__15 +2011_001066__15 +2011_001079__15 +2011_001080__15 +2011_001091__15 +2011_001093__15 +2011_001100__15 +2011_001107__15 +2011_001116__15 +2011_001123__15 +2011_001124__15 +2011_001149__15 +2011_001152__15 +2011_001158__15 +2011_001168__15 +2011_001201__15 +2011_001203__15 +2011_001213__15 +2011_001215__15 +2011_001220__15 +2011_001221__15 +2011_001253__15 +2011_001260__15 +2011_001266__15 +2011_001271__15 +2011_001277__15 +2011_001283__15 +2011_001285__15 +2011_001286__15 +2011_001295__15 +2011_001305__15 +2011_001310__15 +2011_001311__15 +2011_001327__15 +2011_001336__15 +2011_001370__15 +2011_001382__15 +2011_001387__15 +2011_001388__15 +2011_001400__15 +2011_001406__15 +2011_001422__15 +2011_001456__15 +2011_001467__15 +2011_001475__15 +2011_001501__15 +2011_001503__15 +2011_001521__15 +2011_001526__15 +2011_001535__15 +2011_001538__15 +2011_001557__15 +2011_001571__15 +2011_001599__15 +2011_001605__15 +2011_001608__15 +2011_001612__15 +2011_001616__15 +2011_001618__15 +2011_001621__15 +2011_001625__15 +2011_001628__15 +2011_001629__15 +2011_001632__15 +2011_001641__15 +2011_001649__15 +2011_001650__15 +2011_001662__15 +2011_001678__15 +2011_001679__15 +2011_001691__15 +2011_001700__15 +2011_001707__15 +2011_001710__15 +2011_001730__15 +2011_001732__15 +2011_001747__15 +2011_001755__15 +2011_001765__15 +2011_001769__15 +2011_001790__15 +2011_001800__15 +2011_001815__15 +2011_001819__15 +2011_001820__15 +2011_001842__15 +2011_001845__15 +2011_001870__15 +2011_001885__15 +2011_001895__15 +2011_001906__15 +2011_001914__15 +2011_001919__15 +2011_001920__15 +2011_001922__15 +2011_001928__15 +2011_001937__15 +2011_001956__15 +2011_001961__15 +2011_001966__15 +2011_001975__15 +2011_001980__15 +2011_002005__15 +2011_002006__15 +2011_002019__15 +2011_002021__15 +2011_002033__15 +2011_002039__15 +2011_002045__15 +2011_002046__15 +2011_002047__15 +2011_002049__15 +2011_002063__15 +2011_002074__15 +2011_002085__15 +2011_002097__15 +2011_002106__15 +2011_002109__15 +2011_002113__15 +2011_002135__15 +2011_002137__15 +2011_002142__15 +2011_002144__15 +2011_002148__15 +2011_002149__15 +2011_002158__15 +2011_002160__15 +2011_002163__15 +2011_002167__15 +2011_002179__15 +2011_002184__15 +2011_002186__15 +2011_002192__15 +2011_002193__15 +2011_002227__15 +2011_002234__15 +2011_002236__15 +2011_002237__15 +2011_002239__15 +2011_002245__15 +2011_002248__15 +2011_002273__15 +2011_002291__15 +2011_002301__15 +2011_002303__15 +2011_002312__15 +2011_002335__15 +2011_002341__15 +2011_002348__15 +2011_002359__15 +2011_002365__15 +2011_002386__15 +2011_002393__15 +2011_002394__15 +2011_002397__15 +2011_002413__15 +2011_002418__15 +2011_002429__15 +2011_002435__15 +2011_002436__15 +2011_002443__15 +2011_002455__15 +2011_002462__15 +2011_002474__15 +2011_002479__15 +2011_002482__15 +2011_002488__15 +2011_002490__15 +2011_002495__15 +2011_002505__15 +2011_002511__15 +2011_002533__15 +2011_002536__15 +2011_002542__15 +2011_002551__15 +2011_002553__15 +2011_002554__15 +2011_002555__15 +2011_002556__15 +2011_002561__15 +2011_002566__15 +2011_002579__15 +2011_002588__15 +2011_002590__15 +2011_002598__15 +2011_002606__15 +2011_002610__15 +2011_002614__15 +2011_002624__15 +2011_002639__15 +2011_002640__15 +2011_002650__15 +2011_002658__15 +2011_002661__15 +2011_002677__15 +2011_002678__15 +2011_002694__15 +2011_002706__15 +2011_002709__15 +2011_002725__15 +2011_002748__15 +2011_002760__15 +2011_002765__15 +2011_002770__15 +2011_002795__15 +2011_002796__15 +2011_002798__15 +2011_002802__15 +2011_002803__15 +2011_002811__15 +2011_002821__15 +2011_002830__15 +2011_002831__15 +2011_002838__15 +2011_002842__15 +2011_002852__15 +2011_002872__15 +2011_002880__15 +2011_002881__15 +2011_002884__15 +2011_002900__15 +2011_002920__15 +2011_002921__15 +2011_002935__15 +2011_002940__15 +2011_002947__15 +2011_002949__15 +2011_002958__15 +2011_002962__15 +2011_002966__15 +2011_002967__15 +2011_002969__15 +2011_002971__15 +2011_002974__15 +2011_002978__15 +2011_002985__15 +2011_002999__15 +2011_003010__15 +2011_003013__15 +2011_003038__15 +2011_003044__15 +2011_003048__15 +2011_003050__15 +2011_003054__15 +2011_003066__15 +2011_003078__15 +2011_003079__15 +2011_003089__15 +2011_003109__15 +2011_003138__15 +2011_003141__15 +2011_003152__15 +2011_003162__15 +2011_003167__15 +2011_003168__15 +2011_003185__15 +2011_003188__15 +2011_003220__15 +2011_003230__15 +2011_003236__15 +2011_003238__15 +2011_003242__15 +2011_003246__15 +2011_003255__15 +2011_003259__15 +2011_003260__15 +2011_003262__15 diff --git a/submodules/lang_seg/fewshot_data/data/splits/pascal/trn/fold3.txt b/submodules/lang_seg/fewshot_data/data/splits/pascal/trn/fold3.txt new file mode 100644 index 0000000000000000000000000000000000000000..25ee0e8644a8764849bcf49047c87cc526d9920a --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/pascal/trn/fold3.txt @@ -0,0 +1,2086 @@ +2007_001149__16 +2007_001420__16 +2007_002361__16 +2007_002967__16 +2007_003189__16 +2007_003778__16 +2007_004081__16 +2007_004707__16 +2007_004948__16 +2007_006303__16 +2007_006605__16 +2007_007890__16 +2007_008043__16 +2007_008140__16 +2007_008821__16 +2007_009630__16 +2008_000188__16 +2008_000196__16 +2008_000274__16 +2008_000287__16 +2008_000491__16 +2008_000564__16 +2008_000790__16 +2008_000841__16 +2008_000857__16 +2008_000916__16 +2008_000923__16 +2008_000960__16 +2008_001133__16 +2008_001451__16 +2008_001460__16 +2008_001467__16 +2008_001784__16 +2008_001862__16 +2008_001865__16 +2008_002026__16 +2008_002191__16 +2008_002317__16 +2008_002653__16 +2008_003225__16 +2008_003320__16 +2008_003434__16 +2008_003466__16 +2008_003523__16 +2008_003547__16 +2008_003636__16 +2008_003665__16 +2008_003726__16 +2008_003767__16 +2008_003768__16 +2008_003815__16 +2008_003838__16 +2008_003978__16 +2008_004002__16 +2008_004003__16 +2008_004092__16 +2008_004171__16 +2008_004380__16 +2008_004428__16 +2008_004435__16 +2008_004497__16 +2008_004615__16 +2008_004619__16 +2008_004634__16 +2008_004661__16 +2008_004756__16 +2008_004977__16 +2008_005001__16 +2008_005040__16 +2008_005042__16 +2008_005111__16 +2008_005146__16 +2008_005214__16 +2008_005345__16 +2008_005417__16 +2008_005501__16 +2008_005511__16 +2008_005519__16 +2008_005608__16 +2008_005794__16 +2008_005798__16 +2008_005847__16 +2008_005874__16 +2008_005897__16 +2008_005914__16 +2008_005954__16 +2008_006068__16 +2008_006112__16 +2008_006203__16 +2008_006207__16 +2008_006262__16 +2008_006295__16 +2008_006337__16 +2008_006441__16 +2008_006524__16 +2008_006534__16 +2008_006543__16 +2008_006562__16 +2008_006751__16 +2008_006796__16 +2008_006807__16 +2008_006816__16 +2008_006828__16 +2008_006864__16 +2008_006881__16 +2008_006950__16 +2008_007042__16 +2008_007108__16 +2008_007223__16 +2008_007226__16 +2008_007281__16 +2008_007388__16 +2008_007461__16 +2008_007525__16 +2008_007621__16 +2008_007701__16 +2008_007823__16 +2008_007831__16 +2008_007835__16 +2008_007973__16 +2008_007977__16 +2008_008024__16 +2008_008070__16 +2008_008096__16 +2008_008184__16 +2008_008208__16 +2008_008235__16 +2008_008237__16 +2008_008310__16 +2008_008330__16 +2008_008331__16 +2008_008341__16 +2008_008363__16 +2008_008517__16 +2008_008531__16 +2008_008608__16 +2008_008621__16 +2008_008641__16 +2008_008689__16 +2009_000158__16 +2009_000198__16 +2009_000297__16 +2009_000419__16 +2009_000526__16 +2009_000590__16 +2009_000624__16 +2009_000635__16 +2009_000760__16 +2009_000867__16 +2009_000926__16 +2009_001085__16 +2009_001100__16 +2009_001137__16 +2009_001229__16 +2009_001249__16 +2009_001417__16 +2009_001440__16 +2009_001514__16 +2009_001627__16 +2009_001667__16 +2009_001704__16 +2009_001806__16 +2009_001864__16 +2009_001888__16 +2009_001922__16 +2009_001934__16 +2009_001975__16 +2009_002088__16 +2009_002123__16 +2009_002386__16 +2009_002433__16 +2009_002444__16 +2009_002628__16 +2009_002670__16 +2009_002688__16 +2009_002698__16 +2009_002741__16 +2009_002755__16 +2009_002817__16 +2009_002935__16 +2009_002952__16 +2009_003039__16 +2009_003074__16 +2009_003077__16 +2009_003238__16 +2009_003288__16 +2009_003301__16 +2009_003351__16 +2009_003384__16 +2009_003440__16 +2009_003476__16 +2009_003642__16 +2009_003654__16 +2009_003677__16 +2009_003679__16 +2009_003697__16 +2009_003815__16 +2009_003888__16 +2009_003929__16 +2009_004025__16 +2009_004050__16 +2009_004051__16 +2009_004093__16 +2009_004191__16 +2009_004263__16 +2009_004274__16 +2009_004283__16 +2009_004394__16 +2009_004404__16 +2009_004426__16 +2009_004438__16 +2009_004514__16 +2009_004532__16 +2009_004537__16 +2009_004554__16 +2009_004631__16 +2009_004642__16 +2009_004645__16 +2009_004745__16 +2009_004794__16 +2009_004869__16 +2009_004885__16 +2009_004901__16 +2009_004919__16 +2009_004983__16 +2009_004990__16 +2009_005008__16 +2009_005069__16 +2009_005070__16 +2009_005165__16 +2009_005170__16 +2009_005278__16 +2009_005286__16 +2009_005310__16 +2010_000015__16 +2010_000027__16 +2010_000132__16 +2010_000202__16 +2010_000399__16 +2010_000470__16 +2010_000567__16 +2010_000601__16 +2010_000669__16 +2010_000737__16 +2010_000773__16 +2010_000800__16 +2010_000871__16 +2010_000876__16 +2010_000973__16 +2010_001111__16 +2010_001134__16 +2010_001219__16 +2010_001310__16 +2010_001479__16 +2010_001544__16 +2010_001717__16 +2010_001743__16 +2010_001787__16 +2010_001843__16 +2010_001864__16 +2010_001940__16 +2010_002193__16 +2010_002195__16 +2010_002316__16 +2010_002366__16 +2010_002379__16 +2010_002462__16 +2010_002537__16 +2010_002605__16 +2010_002661__16 +2010_002676__16 +2010_002742__16 +2010_002830__16 +2010_002982__16 +2010_003017__16 +2010_003101__16 +2010_003103__16 +2010_003203__16 +2010_003218__16 +2010_003390__16 +2010_003556__16 +2010_003651__16 +2010_003667__16 +2010_003674__16 +2010_003728__16 +2010_003729__16 +2010_003910__16 +2010_004062__16 +2010_004072__16 +2010_004224__16 +2010_004358__16 +2010_004466__16 +2010_004683__16 +2010_004908__16 +2010_004910__16 +2010_004944__16 +2010_004974__16 +2010_004982__16 +2010_005158__16 +2010_005274__16 +2010_005279__16 +2010_005455__16 +2010_005536__16 +2010_005593__16 +2010_005758__16 +2010_005830__16 +2010_005930__16 +2010_005932__16 +2010_005975__16 +2011_000072__16 +2011_000145__16 +2011_000196__16 +2011_000361__16 +2011_000388__16 +2011_000468__16 +2011_000514__16 +2011_000530__16 +2011_000572__16 +2011_000731__16 +2011_000743__16 +2011_000823__16 +2011_000875__16 +2011_000885__16 +2011_000919__16 +2011_000934__16 +2011_000957__16 +2011_001009__16 +2011_001011__16 +2011_001022__16 +2011_001034__16 +2011_001055__16 +2011_001221__16 +2011_001226__16 +2011_001360__16 +2011_001369__16 +2011_001382__16 +2011_001440__16 +2011_001456__16 +2011_001600__16 +2011_001611__16 +2011_001689__16 +2011_001766__16 +2011_001820__16 +2011_001845__16 +2011_001946__16 +2011_002022__16 +2011_002031__16 +2011_002318__16 +2011_002386__16 +2011_002443__16 +2011_002614__16 +2011_002808__16 +2011_002810__16 +2011_002924__16 +2011_002978__16 +2011_003002__16 +2011_003047__16 +2011_003162__16 +2007_001416__17 +2007_001872__17 +2007_002845__17 +2007_003190__17 +2007_003593__17 +2007_004423__17 +2007_004768__17 +2007_006136__17 +2007_006832__17 +2007_006899__17 +2007_006944__17 +2007_007048__17 +2007_007230__17 +2007_007621__17 +2008_000084__17 +2008_000099__17 +2008_000669__17 +2008_001601__17 +2008_002061__17 +2008_002150__17 +2008_002343__17 +2008_002430__17 +2008_003147__17 +2008_004007__17 +2008_004629__17 +2008_005447__17 +2008_005494__17 +2008_005505__17 +2008_005635__17 +2008_005706__17 +2008_005736__17 +2008_005938__17 +2008_005987__17 +2008_006059__17 +2008_006070__17 +2008_006100__17 +2008_006221__17 +2008_006339__17 +2008_006477__17 +2008_006570__17 +2008_006892__17 +2008_006939__17 +2008_007069__17 +2008_007070__17 +2008_007245__17 +2008_007334__17 +2008_007430__17 +2008_007693__17 +2008_007806__17 +2008_007890__17 +2008_007909__17 +2008_007985__17 +2008_008109__17 +2008_008319__17 +2008_008322__17 +2008_008323__17 +2008_008601__17 +2008_008613__17 +2008_008623__17 +2008_008665__17 +2008_008666__17 +2008_008714__17 +2008_008744__17 +2009_000168__17 +2009_000223__17 +2009_000289__17 +2009_000356__17 +2009_000670__17 +2009_000725__17 +2009_000750__17 +2009_000837__17 +2009_001172__17 +2009_001177__17 +2009_001203__17 +2009_001236__17 +2009_001263__17 +2009_001349__17 +2009_001403__17 +2009_001537__17 +2009_001618__17 +2009_001643__17 +2009_001699__17 +2009_001732__17 +2009_001738__17 +2009_001783__17 +2009_001959__17 +2009_002133__17 +2009_002245__17 +2009_002282__17 +2009_002391__17 +2009_002719__17 +2009_002921__17 +2009_002988__17 +2009_003076__17 +2009_003249__17 +2009_003254__17 +2009_003271__17 +2009_003425__17 +2009_003430__17 +2009_003460__17 +2009_003541__17 +2009_003618__17 +2009_003624__17 +2009_003784__17 +2009_004164__17 +2009_004171__17 +2009_004181__17 +2009_004222__17 +2009_004513__17 +2009_004547__17 +2009_004706__17 +2009_004768__17 +2009_004805__17 +2009_004834__17 +2009_004943__17 +2009_004945__17 +2009_005005__17 +2009_005193__17 +2010_000002__17 +2010_000052__17 +2010_000089__17 +2010_000117__17 +2010_000139__17 +2010_000189__17 +2010_000190__17 +2010_000307__17 +2010_000327__17 +2010_000390__17 +2010_000436__17 +2010_000483__17 +2010_000527__17 +2010_000562__17 +2010_000641__17 +2010_000667__17 +2010_000727__17 +2010_000735__17 +2010_000822__17 +2010_000831__17 +2010_000847__17 +2010_000866__17 +2010_000920__17 +2010_000970__17 +2010_000978__17 +2010_001076__17 +2010_001082__17 +2010_001160__17 +2010_001175__17 +2010_001245__17 +2010_001257__17 +2010_001286__17 +2010_001356__17 +2010_001607__17 +2010_001746__17 +2010_001796__17 +2010_001881__17 +2010_001987__17 +2010_002039__17 +2010_002041__17 +2010_002058__17 +2010_002113__17 +2010_002124__17 +2010_002130__17 +2010_002176__17 +2010_002215__17 +2010_002294__17 +2010_002346__17 +2010_002353__17 +2010_002378__17 +2010_002501__17 +2010_002507__17 +2010_002518__17 +2010_002582__17 +2010_002624__17 +2010_002628__17 +2010_002665__17 +2010_002705__17 +2010_002736__17 +2010_002737__17 +2010_002821__17 +2010_003013__17 +2010_003028__17 +2010_003074__17 +2010_003094__17 +2010_003102__17 +2010_003153__17 +2010_003253__17 +2010_003343__17 +2010_003372__17 +2010_003376__17 +2010_003429__17 +2010_003491__17 +2010_003567__17 +2010_003725__17 +2010_003742__17 +2010_003754__17 +2010_003761__17 +2010_003774__17 +2010_003792__17 +2010_003806__17 +2010_003865__17 +2010_003919__17 +2010_003939__17 +2010_003954__17 +2010_003996__17 +2010_004061__17 +2010_004065__17 +2010_004067__17 +2010_004074__17 +2010_004089__17 +2010_004105__17 +2010_004188__17 +2010_004225__17 +2010_004259__17 +2010_004332__17 +2010_004428__17 +2010_004431__17 +2010_004436__17 +2010_004499__17 +2010_004514__17 +2010_004560__17 +2010_004629__17 +2010_004659__17 +2010_004694__17 +2010_004704__17 +2010_004710__17 +2010_004812__17 +2010_004868__17 +2010_005002__17 +2010_005026__17 +2010_005066__17 +2010_005098__17 +2010_005120__17 +2010_005183__17 +2010_005260__17 +2010_005285__17 +2010_005310__17 +2010_005385__17 +2010_005416__17 +2010_005466__17 +2010_005514__17 +2010_005519__17 +2010_005566__17 +2010_005567__17 +2010_005601__17 +2010_005635__17 +2010_005688__17 +2010_005736__17 +2010_005740__17 +2010_005767__17 +2010_005986__17 +2011_000102__17 +2011_000332__17 +2011_000404__17 +2011_000450__17 +2011_000454__17 +2011_000641__17 +2011_000759__17 +2011_000829__17 +2011_000834__17 +2011_001240__17 +2011_001246__17 +2011_001329__17 +2011_001373__17 +2011_001549__17 +2011_001757__17 +2011_001822__17 +2011_001986__17 +2011_002027__17 +2011_002119__17 +2011_002169__17 +2011_002447__17 +2011_002464__17 +2011_002553__17 +2011_002571__17 +2011_002817__17 +2011_003023__17 +2011_003223__17 +2007_000584__18 +2007_001027__18 +2007_001149__18 +2007_001901__18 +2007_002055__18 +2007_002368__18 +2007_002545__18 +2007_003451__18 +2007_004166__18 +2007_005212__18 +2007_005266__18 +2007_005647__18 +2007_006066__18 +2007_006530__18 +2007_008203__18 +2007_008468__18 +2007_008821__18 +2007_009435__18 +2007_009554__18 +2008_000093__18 +2008_000128__18 +2008_000321__18 +2008_000419__18 +2008_000421__18 +2008_000465__18 +2008_000493__18 +2008_000541__18 +2008_000636__18 +2008_000648__18 +2008_000704__18 +2008_000857__18 +2008_001030__18 +2008_001092__18 +2008_001133__18 +2008_001238__18 +2008_001333__18 +2008_001366__18 +2008_001390__18 +2008_001399__18 +2008_001461__18 +2008_001589__18 +2008_001660__18 +2008_001694__18 +2008_001781__18 +2008_001787__18 +2008_001838__18 +2008_001869__18 +2008_001896__18 +2008_002082__18 +2008_002092__18 +2008_002119__18 +2008_002434__18 +2008_002508__18 +2008_002533__18 +2008_002776__18 +2008_002801__18 +2008_002916__18 +2008_002920__18 +2008_002922__18 +2008_002948__18 +2008_003271__18 +2008_003393__18 +2008_003562__18 +2008_003607__18 +2008_003814__18 +2008_004269__18 +2008_004271__18 +2008_004321__18 +2008_004416__18 +2008_004435__18 +2008_004492__18 +2008_004497__18 +2008_004632__18 +2008_004661__18 +2008_004670__18 +2008_004697__18 +2008_004774__18 +2008_004881__18 +2008_004887__18 +2008_004938__18 +2008_004964__18 +2008_005090__18 +2008_005323__18 +2008_005395__18 +2008_005444__18 +2008_005623__18 +2008_005627__18 +2008_005788__18 +2008_005850__18 +2008_005882__18 +2008_005926__18 +2008_006038__18 +2008_006117__18 +2008_006276__18 +2008_006370__18 +2008_006389__18 +2008_006436__18 +2008_006616__18 +2008_006665__18 +2008_006737__18 +2008_006773__18 +2008_006843__18 +2008_006868__18 +2008_006979__18 +2008_007021__18 +2008_007043__18 +2008_007050__18 +2008_007169__18 +2008_007182__18 +2008_007218__18 +2008_007282__18 +2008_007285__18 +2008_007511__18 +2008_007682__18 +2008_007733__18 +2008_007837__18 +2008_008029__18 +2008_008106__18 +2008_008162__18 +2008_008190__18 +2008_008206__18 +2008_008271__18 +2008_008276__18 +2008_008313__18 +2008_008410__18 +2008_008433__18 +2008_008470__18 +2008_008517__18 +2008_008522__18 +2008_008526__18 +2008_008538__18 +2008_008550__18 +2008_008554__18 +2008_008560__18 +2008_008567__18 +2008_008574__18 +2008_008578__18 +2008_008588__18 +2008_008590__18 +2008_008606__18 +2008_008608__18 +2008_008621__18 +2008_008622__18 +2008_008628__18 +2008_008642__18 +2008_008649__18 +2008_008658__18 +2008_008772__18 +2009_000014__18 +2009_000016__18 +2009_000142__18 +2009_000189__18 +2009_000217__18 +2009_000251__18 +2009_000300__18 +2009_000316__18 +2009_000342__18 +2009_000375__18 +2009_000379__18 +2009_000416__18 +2009_000422__18 +2009_000449__18 +2009_000474__18 +2009_000505__18 +2009_000563__18 +2009_000577__18 +2009_000615__18 +2009_000653__18 +2009_000672__18 +2009_000674__18 +2009_000779__18 +2009_000925__18 +2009_000926__18 +2009_000937__18 +2009_000939__18 +2009_000973__18 +2009_000995__18 +2009_001021__18 +2009_001081__18 +2009_001107__18 +2009_001146__18 +2009_001190__18 +2009_001212__18 +2009_001241__18 +2009_001243__18 +2009_001249__18 +2009_001268__18 +2009_001313__18 +2009_001343__18 +2009_001357__18 +2009_001376__18 +2009_001437__18 +2009_001440__18 +2009_001446__18 +2009_001470__18 +2009_001577__18 +2009_001581__18 +2009_001605__18 +2009_001608__18 +2009_001631__18 +2009_001719__18 +2009_001743__18 +2009_001746__18 +2009_001774__18 +2009_001871__18 +2009_001874__18 +2009_001888__18 +2009_001906__18 +2009_001908__18 +2009_001961__18 +2009_001980__18 +2009_002083__18 +2009_002192__18 +2009_002208__18 +2009_002253__18 +2009_002325__18 +2009_002370__18 +2009_002408__18 +2009_002522__18 +2009_002523__18 +2009_002558__18 +2009_002611__18 +2009_002612__18 +2009_002663__18 +2009_002673__18 +2009_002681__18 +2009_002683__18 +2009_002713__18 +2009_002717__18 +2009_002893__18 +2009_002972__18 +2009_002998__18 +2009_003087__18 +2009_003129__18 +2009_003156__18 +2009_003208__18 +2009_003373__18 +2009_003377__18 +2009_003394__18 +2009_003409__18 +2009_003441__18 +2009_003459__18 +2009_003581__18 +2009_003605__18 +2009_003613__18 +2009_003642__18 +2009_003646__18 +2009_003656__18 +2009_003671__18 +2009_003695__18 +2009_003711__18 +2009_003785__18 +2009_003795__18 +2009_003819__18 +2009_003835__18 +2009_003843__18 +2009_003848__18 +2009_003965__18 +2009_003966__18 +2009_003995__18 +2009_004073__18 +2009_004076__18 +2009_004088__18 +2009_004091__18 +2009_004161__18 +2009_004165__18 +2009_004177__18 +2009_004180__18 +2009_004188__18 +2009_004193__18 +2009_004264__18 +2009_004283__18 +2009_004291__18 +2009_004301__18 +2009_004322__18 +2009_004419__18 +2009_004426__18 +2009_004449__18 +2009_004452__18 +2009_004456__18 +2009_004457__18 +2009_004464__18 +2009_004560__18 +2009_004582__18 +2009_004588__18 +2009_004593__18 +2009_004674__18 +2009_004701__18 +2009_004718__18 +2009_004782__18 +2009_004823__18 +2009_004839__18 +2009_004901__18 +2009_004983__18 +2009_005070__18 +2009_005240__18 +2009_005299__18 +2010_000018__18 +2010_000097__18 +2010_000329__18 +2010_000344__18 +2010_000588__18 +2010_000671__18 +2010_000691__18 +2010_000694__18 +2010_000821__18 +2010_000830__18 +2010_000922__18 +2010_000968__18 +2010_001051__18 +2010_001066__18 +2010_001111__18 +2010_001127__18 +2010_001148__18 +2010_001189__18 +2010_001277__18 +2010_001287__18 +2010_001434__18 +2010_001514__18 +2010_001547__18 +2010_001586__18 +2010_001636__18 +2010_001743__18 +2010_001763__18 +2010_001933__18 +2010_002015__18 +2010_002045__18 +2010_002191__18 +2010_002193__18 +2010_002312__18 +2010_002337__18 +2010_002461__18 +2010_002527__18 +2010_002659__18 +2010_002710__18 +2010_002811__18 +2010_002817__18 +2010_002860__18 +2010_002962__18 +2010_003027__18 +2010_003114__18 +2010_003143__18 +2010_003149__18 +2010_003169__18 +2010_003174__18 +2010_003203__18 +2010_003278__18 +2010_003305__18 +2010_003331__18 +2010_003401__18 +2010_003451__18 +2010_003556__18 +2010_003613__18 +2010_003717__18 +2010_003804__18 +2010_003822__18 +2010_003861__18 +2010_003864__18 +2010_004025__18 +2010_004043__18 +2010_004062__18 +2010_004095__18 +2010_004109__18 +2010_004111__18 +2010_004125__18 +2010_004358__18 +2010_004409__18 +2010_004447__18 +2010_004481__18 +2010_004741__18 +2010_004765__18 +2010_004805__18 +2010_005049__18 +2010_005054__18 +2010_005170__18 +2010_005193__18 +2010_005388__18 +2010_005398__18 +2010_005532__18 +2010_005610__18 +2010_005614__18 +2010_005681__18 +2010_005692__18 +2010_005734__18 +2010_005770__18 +2010_005830__18 +2010_005898__18 +2010_005937__18 +2010_005980__18 +2010_006056__18 +2010_006073__18 +2011_000006__18 +2011_000037__18 +2011_000082__18 +2011_000122__18 +2011_000142__18 +2011_000146__18 +2011_000182__18 +2011_000224__18 +2011_000304__18 +2011_000364__18 +2011_000379__18 +2011_000386__18 +2011_000399__18 +2011_000434__18 +2011_000457__18 +2011_000475__18 +2011_000477__18 +2011_000499__18 +2011_000550__18 +2011_000565__18 +2011_000572__18 +2011_000608__18 +2011_000630__18 +2011_000646__18 +2011_000657__18 +2011_000689__18 +2011_000765__18 +2011_000820__18 +2011_000947__18 +2011_001027__18 +2011_001031__18 +2011_001167__18 +2011_001175__18 +2011_001192__18 +2011_001198__18 +2011_001215__18 +2011_001283__18 +2011_001304__18 +2011_001330__18 +2011_001402__18 +2011_001404__18 +2011_001412__18 +2011_001440__18 +2011_001451__18 +2011_001518__18 +2011_001531__18 +2011_001547__18 +2011_001600__18 +2011_001662__18 +2011_001691__18 +2011_001733__18 +2011_001739__18 +2011_001751__18 +2011_001811__18 +2011_001820__18 +2011_001845__18 +2011_001856__18 +2011_001895__18 +2011_001914__18 +2011_001922__18 +2011_001932__18 +2011_001974__18 +2011_001977__18 +2011_001980__18 +2011_002109__18 +2011_002184__18 +2011_002186__18 +2011_002268__18 +2011_002291__18 +2011_002335__18 +2011_002359__18 +2011_002395__18 +2011_002414__18 +2011_002507__18 +2011_002554__18 +2011_002561__18 +2011_002594__18 +2011_002714__18 +2011_002726__18 +2011_002752__18 +2011_002756__18 +2011_002775__18 +2011_002784__18 +2011_002810__18 +2011_002814__18 +2011_002834__18 +2011_002852__18 +2011_002953__18 +2011_002965__18 +2011_003038__18 +2011_003039__18 +2011_003044__18 +2011_003049__18 +2011_003188__18 +2011_003201__18 +2011_003212__18 +2007_000333__19 +2007_002462__19 +2007_003178__19 +2007_003286__19 +2007_004627__19 +2007_004663__19 +2007_004951__19 +2007_005360__19 +2007_006254__19 +2007_006400__19 +2007_006803__19 +2007_007387__19 +2007_007726__19 +2007_007947__19 +2007_009436__19 +2007_009580__19 +2007_009597__19 +2007_009950__19 +2008_000003__19 +2008_000045__19 +2008_000343__19 +2008_000373__19 +2008_000470__19 +2008_000916__19 +2008_001105__19 +2008_001114__19 +2008_001118__19 +2008_001164__19 +2008_001169__19 +2008_001358__19 +2008_001625__19 +2008_001710__19 +2008_001850__19 +2008_001866__19 +2008_001905__19 +2008_001926__19 +2008_001956__19 +2008_002158__19 +2008_002193__19 +2008_002222__19 +2008_002279__19 +2008_002325__19 +2008_002344__19 +2008_002452__19 +2008_002457__19 +2008_002465__19 +2008_002965__19 +2008_003025__19 +2008_003068__19 +2008_003083__19 +2008_003263__19 +2008_003414__19 +2008_003571__19 +2008_003578__19 +2008_003826__19 +2008_003992__19 +2008_004110__19 +2008_004214__19 +2008_004235__19 +2008_004357__19 +2008_004358__19 +2008_004547__19 +2008_004663__19 +2008_004770__19 +2008_004852__19 +2008_004869__19 +2008_004946__19 +2008_005085__19 +2008_005185__19 +2008_005269__19 +2008_005282__19 +2008_005354__19 +2008_005446__19 +2008_005653__19 +2008_005742__19 +2008_005763__19 +2008_005801__19 +2008_005825__19 +2008_005968__19 +2008_006010__19 +2008_006158__19 +2008_006365__19 +2008_006368__19 +2008_006655__19 +2008_006818__19 +2008_006849__19 +2008_006865__19 +2008_006900__19 +2008_006919__19 +2008_007011__19 +2008_007084__19 +2008_007105__19 +2008_007115__19 +2008_007185__19 +2008_007189__19 +2008_007201__19 +2008_007231__19 +2008_007247__19 +2008_007280__19 +2008_007383__19 +2008_007521__19 +2008_007648__19 +2008_007749__19 +2008_007759__19 +2008_007760__19 +2008_007779__19 +2008_007787__19 +2008_007829__19 +2008_007887__19 +2008_007999__19 +2008_008001__19 +2008_008020__19 +2008_008055__19 +2008_008074__19 +2008_008123__19 +2008_008152__19 +2008_008192__19 +2008_008200__19 +2008_008203__19 +2008_008223__19 +2008_008275__19 +2008_008297__19 +2008_008302__19 +2008_008321__19 +2008_008336__19 +2008_008342__19 +2008_008364__19 +2008_008379__19 +2008_008382__19 +2008_008527__19 +2008_008545__19 +2008_008583__19 +2008_008615__19 +2008_008618__19 +2008_008632__19 +2008_008637__19 +2008_008641__19 +2008_008662__19 +2008_008673__19 +2008_008676__19 +2008_008681__19 +2008_008690__19 +2008_008696__19 +2008_008697__19 +2008_008726__19 +2008_008732__19 +2008_008735__19 +2008_008739__19 +2008_008749__19 +2008_008751__19 +2008_008757__19 +2008_008767__19 +2008_008770__19 +2009_000011__19 +2009_000051__19 +2009_000073__19 +2009_000090__19 +2009_000105__19 +2009_000137__19 +2009_000177__19 +2009_000244__19 +2009_000283__19 +2009_000347__19 +2009_000443__19 +2009_000476__19 +2009_000501__19 +2009_000592__19 +2009_000597__19 +2009_000658__19 +2009_000663__19 +2009_000689__19 +2009_000789__19 +2009_000824__19 +2009_000890__19 +2009_000910__19 +2009_000920__19 +2009_000974__19 +2009_001007__19 +2009_001042__19 +2009_001078__19 +2009_001118__19 +2009_001152__19 +2009_001164__19 +2009_001192__19 +2009_001245__19 +2009_001259__19 +2009_001291__19 +2009_001350__19 +2009_001359__19 +2009_001412__19 +2009_001468__19 +2009_001493__19 +2009_001516__19 +2009_001519__19 +2009_001534__19 +2009_001648__19 +2009_001651__19 +2009_001671__19 +2009_001735__19 +2009_001747__19 +2009_001802__19 +2009_001823__19 +2009_001831__19 +2009_001853__19 +2009_001865__19 +2009_001868__19 +2009_001904__19 +2009_001977__19 +2009_002009__19 +2009_002116__19 +2009_002144__19 +2009_002175__19 +2009_002197__19 +2009_002214__19 +2009_002219__19 +2009_002225__19 +2009_002274__19 +2009_002281__19 +2009_002377__19 +2009_002441__19 +2009_002557__19 +2009_002616__19 +2009_002624__19 +2009_002669__19 +2009_002676__19 +2009_002689__19 +2009_002695__19 +2009_002712__19 +2009_002725__19 +2009_002734__19 +2009_002774__19 +2009_002838__19 +2009_002867__19 +2009_002938__19 +2009_002947__19 +2009_003022__19 +2009_003054__19 +2009_003185__19 +2009_003230__19 +2009_003233__19 +2009_003333__19 +2009_003348__19 +2009_003407__19 +2009_003436__19 +2009_003453__19 +2009_003492__19 +2009_003497__19 +2009_003534__19 +2009_003543__19 +2009_003583__19 +2009_003638__19 +2009_003650__19 +2009_003758__19 +2009_003765__19 +2009_003790__19 +2009_003821__19 +2009_003863__19 +2009_003892__19 +2009_003942__19 +2009_003951__19 +2009_004019__19 +2009_004109__19 +2009_004159__19 +2009_004163__19 +2009_004170__19 +2009_004211__19 +2009_004213__19 +2009_004329__19 +2009_004336__19 +2009_004371__19 +2009_004406__19 +2009_004453__19 +2009_004468__19 +2009_004511__19 +2009_004527__19 +2009_004559__19 +2009_004619__19 +2009_004624__19 +2009_004669__19 +2009_004671__19 +2009_004677__19 +2009_004708__19 +2009_004766__19 +2009_004771__19 +2009_004804__19 +2009_004880__19 +2009_004956__19 +2009_004958__19 +2009_004977__19 +2009_004988__19 +2009_005024__19 +2009_005061__19 +2009_005084__19 +2009_005126__19 +2009_005128__19 +2009_005149__19 +2009_005246__19 +2009_005287__19 +2009_005292__19 +2009_005303__19 +2010_000080__19 +2010_000085__19 +2010_000136__19 +2010_000199__19 +2010_000233__19 +2010_000249__19 +2010_000313__19 +2010_000321__19 +2010_000406__19 +2010_000513__19 +2010_000537__19 +2010_000581__19 +2010_000633__19 +2010_000651__19 +2010_000740__19 +2010_000743__19 +2010_000786__19 +2010_000860__19 +2010_000865__19 +2010_000955__19 +2010_000959__19 +2010_000984__19 +2010_001052__19 +2010_001105__19 +2010_001143__19 +2010_001250__19 +2010_001272__19 +2010_001374__19 +2010_001395__19 +2010_001405__19 +2010_001408__19 +2010_001502__19 +2010_001515__19 +2010_001539__19 +2010_001560__19 +2010_001586__19 +2010_001625__19 +2010_001719__19 +2010_001748__19 +2010_001788__19 +2010_001801__19 +2010_001941__19 +2010_002073__19 +2010_002080__19 +2010_002179__19 +2010_002182__19 +2010_002208__19 +2010_002223__19 +2010_002261__19 +2010_002369__19 +2010_002420__19 +2010_002487__19 +2010_002556__19 +2010_002618__19 +2010_002667__19 +2010_002697__19 +2010_002722__19 +2010_002838__19 +2010_002840__19 +2010_002851__19 +2010_002896__19 +2010_002937__19 +2010_002946__19 +2010_003129__19 +2010_003160__19 +2010_003274__19 +2010_003335__19 +2010_003384__19 +2010_003470__19 +2010_003482__19 +2010_003601__19 +2010_003609__19 +2010_003628__19 +2010_003630__19 +2010_003788__19 +2010_003847__19 +2010_003900__19 +2010_003944__19 +2010_004075__19 +2010_004148__19 +2010_004168__19 +2010_004193__19 +2010_004256__19 +2010_004313__19 +2010_004350__19 +2010_004412__19 +2010_004469__19 +2010_004475__19 +2010_004478__19 +2010_004536__19 +2010_004604__19 +2010_004669__19 +2010_004677__19 +2010_004779__19 +2010_004826__19 +2010_005055__19 +2010_005130__19 +2010_005309__19 +2010_005463__19 +2010_005506__19 +2010_005515__19 +2010_005559__19 +2010_005565__19 +2010_005643__19 +2010_005768__19 +2010_005810__19 +2010_005816__19 +2010_005934__19 +2010_005996__19 +2011_000012__19 +2011_000058__19 +2011_000105__19 +2011_000195__19 +2011_000197__19 +2011_000210__19 +2011_000221__19 +2011_000241__19 +2011_000250__19 +2011_000277__19 +2011_000346__19 +2011_000398__19 +2011_000442__19 +2011_000491__19 +2011_000498__19 +2011_000513__19 +2011_000558__19 +2011_000627__19 +2011_000688__19 +2011_000819__19 +2011_000848__19 +2011_000858__19 +2011_000895__19 +2011_000909__19 +2011_000944__19 +2011_000979__19 +2011_000987__19 +2011_000997__19 +2011_001019__19 +2011_001052__19 +2011_001086__19 +2011_001126__19 +2011_001152__19 +2011_001166__19 +2011_001217__19 +2011_001337__19 +2011_001375__19 +2011_001381__19 +2011_001525__19 +2011_001560__19 +2011_001602__19 +2011_001655__19 +2011_001671__19 +2011_001741__19 +2011_001776__19 +2011_001796__19 +2011_001827__19 +2011_001889__19 +2011_001904__19 +2011_001927__19 +2011_001951__19 +2011_002053__19 +2011_002105__19 +2011_002173__19 +2011_002251__19 +2011_002278__19 +2011_002280__19 +2011_002294__19 +2011_002385__19 +2011_002396__19 +2011_002492__19 +2011_002516__19 +2011_002528__19 +2011_002636__19 +2011_002738__19 +2011_002779__19 +2011_002821__19 +2011_002917__19 +2011_002932__19 +2011_002987__19 +2011_003028__19 +2011_003059__19 +2011_003124__19 +2011_003132__19 +2011_003149__19 +2011_003187__19 +2011_003228__19 +2011_003260__19 +2011_003274__19 +2007_000039__20 +2007_000121__20 +2007_001027__20 +2007_001149__20 +2007_001704__20 +2007_002227__20 +2007_002953__20 +2007_003451__20 +2007_003604__20 +2007_005210__20 +2007_005902__20 +2007_006066__20 +2007_006704__20 +2007_007250__20 +2007_007432__20 +2007_007530__20 +2007_008407__20 +2007_008948__20 +2007_009216__20 +2007_009295__20 +2007_009594__20 +2008_000002__20 +2008_000023__20 +2008_000093__20 +2008_000145__20 +2008_000202__20 +2008_000244__20 +2008_000305__20 +2008_000309__20 +2008_000348__20 +2008_000383__20 +2008_000495__20 +2008_000566__20 +2008_000578__20 +2008_000904__20 +2008_001021__20 +2008_001073__20 +2008_001130__20 +2008_001401__20 +2008_001428__20 +2008_001481__20 +2008_001576__20 +2008_001641__20 +2008_001704__20 +2008_001781__20 +2008_001815__20 +2008_001880__20 +2008_001888__20 +2008_001896__20 +2008_001920__20 +2008_001997__20 +2008_002066__20 +2008_002082__20 +2008_002140__20 +2008_002218__20 +2008_002328__20 +2008_002547__20 +2008_002650__20 +2008_002676__20 +2008_002776__20 +2008_002817__20 +2008_002826__20 +2008_002831__20 +2008_002954__20 +2008_003200__20 +2008_003213__20 +2008_003248__20 +2008_003280__20 +2008_003348__20 +2008_003432__20 +2008_003434__20 +2008_003435__20 +2008_003466__20 +2008_003500__20 +2008_003585__20 +2008_003589__20 +2008_003609__20 +2008_003667__20 +2008_003712__20 +2008_003814__20 +2008_003825__20 +2008_003883__20 +2008_003948__20 +2008_003995__20 +2008_004004__20 +2008_004006__20 +2008_004008__20 +2008_004093__20 +2008_004097__20 +2008_004217__20 +2008_004259__20 +2008_004297__20 +2008_004301__20 +2008_004321__20 +2008_004330__20 +2008_004333__20 +2008_004501__20 +2008_004506__20 +2008_004526__20 +2008_004541__20 +2008_004550__20 +2008_004606__20 +2008_004719__20 +2008_004720__20 +2008_004781__20 +2008_004807__20 +2008_004881__20 +2008_004898__20 +2008_004908__20 +2008_004930__20 +2008_004961__20 +2008_005006__20 +2008_005008__20 +2008_005037__20 +2008_005064__20 +2008_005066__20 +2008_005090__20 +2008_005094__20 +2008_005191__20 +2008_005231__20 +2008_005255__20 +2008_005329__20 +2008_005342__20 +2008_005393__20 +2008_005569__20 +2008_005609__20 +2008_005625__20 +2008_005639__20 +2008_005660__20 +2008_005678__20 +2008_005732__20 +2008_005817__20 +2008_005877__20 +2008_005918__20 +2008_005929__20 +2008_005954__20 +2008_005957__20 +2008_005962__20 +2008_005967__20 +2008_005976__20 +2008_006031__20 +2008_006047__20 +2008_006062__20 +2008_006135__20 +2008_006136__20 +2008_006147__20 +2008_006233__20 +2008_006267__20 +2008_006271__20 +2008_006273__20 +2008_006288__20 +2008_006295__20 +2008_006366__20 +2008_006373__20 +2008_006409__20 +2008_006433__20 +2008_006591__20 +2008_006605__20 +2008_006606__20 +2008_006617__20 +2008_006624__20 +2008_006662__20 +2008_006668__20 +2008_006710__20 +2008_006719__20 +2008_006733__20 +2008_006946__20 +2008_007010__20 +2008_007038__20 +2008_007114__20 +2008_007196__20 +2008_007217__20 +2008_007242__20 +2008_007246__20 +2008_007324__20 +2008_007332__20 +2008_007361__20 +2008_007446__20 +2008_007476__20 +2008_007536__20 +2008_007561__20 +2008_007567__20 +2008_007685__20 +2008_007696__20 +2008_007798__20 +2008_007864__20 +2008_007916__20 +2008_007933__20 +2008_007962__20 +2008_007987__20 +2008_008269__20 +2008_008429__20 +2008_008439__20 +2008_008524__20 +2008_008590__20 +2008_008608__20 +2008_008649__20 +2009_000010__20 +2009_000014__20 +2009_000041__20 +2009_000157__20 +2009_000214__20 +2009_000216__20 +2009_000336__20 +2009_000398__20 +2009_000439__20 +2009_000444__20 +2009_000544__20 +2009_000549__20 +2009_000552__20 +2009_000585__20 +2009_000629__20 +2009_000679__20 +2009_000722__20 +2009_000791__20 +2009_000848__20 +2009_000895__20 +2009_000981__20 +2009_000987__20 +2009_001069__20 +2009_001103__20 +2009_001106__20 +2009_001111__20 +2009_001133__20 +2009_001188__20 +2009_001357__20 +2009_001393__20 +2009_001452__20 +2009_001526__20 +2009_001553__20 +2009_001555__20 +2009_001608__20 +2009_001615__20 +2009_001682__20 +2009_001779__20 +2009_001809__20 +2009_001812__20 +2009_001839__20 +2009_001852__20 +2009_001864__20 +2009_001874__20 +2009_001875__20 +2009_001961__20 +2009_001964__20 +2009_002110__20 +2009_002139__20 +2009_002232__20 +2009_002409__20 +2009_002537__20 +2009_002652__20 +2009_002663__20 +2009_002705__20 +2009_002733__20 +2009_002755__20 +2009_002758__20 +2009_002820__20 +2009_002827__20 +2009_002849__20 +2009_002872__20 +2009_002932__20 +2009_002967__20 +2009_002970__20 +2009_002984__20 +2009_002995__20 +2009_003078__20 +2009_003093__20 +2009_003140__20 +2009_003191__20 +2009_003204__20 +2009_003214__20 +2009_003316__20 +2009_003367__20 +2009_003537__20 +2009_003554__20 +2009_003646__20 +2009_003720__20 +2009_003753__20 +2009_003852__20 +2009_003920__20 +2009_004062__20 +2009_004128__20 +2009_004138__20 +2009_004176__20 +2009_004243__20 +2009_004301__20 +2009_004341__20 +2009_004357__20 +2009_004359__20 +2009_004478__20 +2009_004503__20 +2009_004519__20 +2009_004631__20 +2009_004719__20 +2009_004760__20 +2009_004763__20 +2009_004902__20 +2009_004905__20 +2009_004922__20 +2009_004965__20 +2009_005030__20 +2009_005042__20 +2009_005062__20 +2009_005070__20 +2009_005221__20 +2009_005240__20 +2009_005256__20 +2010_000053__20 +2010_000141__20 +2010_000291__20 +2010_000375__20 +2010_000379__20 +2010_000442__20 +2010_000449__20 +2010_000578__20 +2010_000658__20 +2010_000669__20 +2010_000705__20 +2010_000773__20 +2010_000787__20 +2010_000800__20 +2010_000807__20 +2010_000931__20 +2010_000944__20 +2010_000974__20 +2010_001099__20 +2010_001127__20 +2010_001270__20 +2010_001277__20 +2010_001363__20 +2010_001533__20 +2010_001562__20 +2010_001580__20 +2010_001690__20 +2010_001780__20 +2010_001860__20 +2010_001918__20 +2010_001939__20 +2010_002002__20 +2010_002015__20 +2010_002094__20 +2010_002097__20 +2010_002152__20 +2010_002167__20 +2010_002193__20 +2010_002245__20 +2010_002247__20 +2010_002327__20 +2010_002427__20 +2010_002513__20 +2010_002526__20 +2010_002561__20 +2010_002567__20 +2010_002586__20 +2010_002652__20 +2010_002686__20 +2010_002770__20 +2010_002791__20 +2010_002843__20 +2010_002982__20 +2010_003035__20 +2010_003103__20 +2010_003137__20 +2010_003236__20 +2010_003241__20 +2010_003287__20 +2010_003405__20 +2010_003437__20 +2010_003461__20 +2010_003674__20 +2010_003688__20 +2010_003719__20 +2010_003728__20 +2010_003770__20 +2010_003844__20 +2010_003857__20 +2010_003864__20 +2010_003874__20 +2010_003892__20 +2010_003942__20 +2010_004009__20 +2010_004050__20 +2010_004095__20 +2010_004102__20 +2010_004109__20 +2010_004137__20 +2010_004249__20 +2010_004254__20 +2010_004295__20 +2010_004306__20 +2010_004368__20 +2010_004460__20 +2010_004503__20 +2010_004523__20 +2010_004545__20 +2010_004586__20 +2010_004591__20 +2010_004816__20 +2010_004836__20 +2010_004944__20 +2010_004982__20 +2010_005049__20 +2010_005071__20 +2010_005133__20 +2010_005158__20 +2010_005190__20 +2010_005239__20 +2010_005345__20 +2010_005372__20 +2010_005450__20 +2010_005676__20 +2010_005678__20 +2010_005744__20 +2010_005805__20 +2010_005827__20 +2010_005841__20 +2010_006050__20 +2011_000009__20 +2011_000036__20 +2011_000037__20 +2011_000038__20 +2011_000061__20 +2011_000071__20 +2011_000077__20 +2011_000192__20 +2011_000253__20 +2011_000288__20 +2011_000290__20 +2011_000364__20 +2011_000382__20 +2011_000399__20 +2011_000400__20 +2011_000434__20 +2011_000444__20 +2011_000608__20 +2011_000685__20 +2011_000755__20 +2011_000965__20 +2011_001149__20 +2011_001223__20 +2011_001302__20 +2011_001357__20 +2011_001387__20 +2011_001394__20 +2011_001456__20 +2011_001466__20 +2011_001507__20 +2011_001573__20 +2011_001689__20 +2011_001705__20 +2011_001730__20 +2011_001833__20 +2011_001837__20 +2011_001856__20 +2011_001926__20 +2011_001928__20 +2011_002005__20 +2011_002154__20 +2011_002218__20 +2011_002292__20 +2011_002418__20 +2011_002462__20 +2011_002511__20 +2011_002514__20 +2011_002554__20 +2011_002560__20 +2011_002656__20 +2011_002756__20 +2011_002775__20 +2011_002802__20 +2011_002942__20 +2011_002966__20 +2011_002970__20 +2011_003047__20 +2011_003079__20 +2011_003194__20 +2011_003254__20 diff --git a/submodules/lang_seg/fewshot_data/data/splits/pascal/val/fold0.txt b/submodules/lang_seg/fewshot_data/data/splits/pascal/val/fold0.txt new file mode 100644 index 0000000000000000000000000000000000000000..3170e6873ab3ace104b4ad2525f6a232f1012fc1 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/pascal/val/fold0.txt @@ -0,0 +1,346 @@ +2007_000033__01 +2007_000061__04 +2007_000129__02 +2007_000346__05 +2007_000529__04 +2007_000559__05 +2007_000572__02 +2007_000762__05 +2007_001288__01 +2007_001289__03 +2007_001311__02 +2007_001408__05 +2007_001568__01 +2007_001630__02 +2007_001761__01 +2007_001884__01 +2007_002094__03 +2007_002266__01 +2007_002376__01 +2007_002400__03 +2007_002619__01 +2007_002719__04 +2007_003088__05 +2007_003131__04 +2007_003188__02 +2007_003349__03 +2007_003571__04 +2007_003621__02 +2007_003682__03 +2007_003861__04 +2007_004052__01 +2007_004143__03 +2007_004241__04 +2007_004468__05 +2007_005074__04 +2007_005107__02 +2007_005294__05 +2007_005304__05 +2007_005428__05 +2007_005509__01 +2007_005600__01 +2007_005705__04 +2007_005828__01 +2007_006076__03 +2007_006086__05 +2007_006449__02 +2007_006946__01 +2007_007084__03 +2007_007235__02 +2007_007341__01 +2007_007470__01 +2007_007477__04 +2007_007836__02 +2007_008051__03 +2007_008084__03 +2007_008204__05 +2007_008670__03 +2007_009088__03 +2007_009258__02 +2007_009323__03 +2007_009458__05 +2007_009687__05 +2007_009817__03 +2007_009911__01 +2008_000120__04 +2008_000123__03 +2008_000533__03 +2008_000725__02 +2008_000911__05 +2008_001013__04 +2008_001040__04 +2008_001135__04 +2008_001260__04 +2008_001404__02 +2008_001514__03 +2008_001531__02 +2008_001546__01 +2008_001580__04 +2008_001966__03 +2008_001971__01 +2008_002043__03 +2008_002269__02 +2008_002358__01 +2008_002429__03 +2008_002467__05 +2008_002504__04 +2008_002775__05 +2008_002864__05 +2008_003034__04 +2008_003076__05 +2008_003108__02 +2008_003110__03 +2008_003155__01 +2008_003270__02 +2008_003369__01 +2008_003858__04 +2008_003876__01 +2008_003886__04 +2008_003926__01 +2008_003976__01 +2008_004363__02 +2008_004654__02 +2008_004659__05 +2008_004704__01 +2008_004758__02 +2008_004995__02 +2008_005262__05 +2008_005338__01 +2008_005628__04 +2008_005727__02 +2008_005812__05 +2008_005904__05 +2008_006216__01 +2008_006229__04 +2008_006254__02 +2008_006703__01 +2008_007120__03 +2008_007143__04 +2008_007219__05 +2008_007350__01 +2008_007498__03 +2008_007811__05 +2008_007994__03 +2008_008268__03 +2008_008629__02 +2008_008711__02 +2008_008746__03 +2009_000032__01 +2009_000037__03 +2009_000121__05 +2009_000149__02 +2009_000201__05 +2009_000205__01 +2009_000318__03 +2009_000354__02 +2009_000387__01 +2009_000421__04 +2009_000440__01 +2009_000446__04 +2009_000457__02 +2009_000469__04 +2009_000573__02 +2009_000619__03 +2009_000664__03 +2009_000723__04 +2009_000828__04 +2009_000840__05 +2009_000879__03 +2009_000991__03 +2009_000998__03 +2009_001108__03 +2009_001160__03 +2009_001255__02 +2009_001278__05 +2009_001314__03 +2009_001332__01 +2009_001565__03 +2009_001607__03 +2009_001683__03 +2009_001718__02 +2009_001765__03 +2009_001818__05 +2009_001850__01 +2009_001851__01 +2009_001941__04 +2009_002185__05 +2009_002295__02 +2009_002320__01 +2009_002372__05 +2009_002521__05 +2009_002594__05 +2009_002604__03 +2009_002649__05 +2009_002727__04 +2009_002732__05 +2009_002749__05 +2009_002808__01 +2009_002856__05 +2009_002888__01 +2009_002928__02 +2009_003003__05 +2009_003005__01 +2009_003043__04 +2009_003080__04 +2009_003193__02 +2009_003224__02 +2009_003269__05 +2009_003273__03 +2009_003343__02 +2009_003378__03 +2009_003450__03 +2009_003498__03 +2009_003504__04 +2009_003517__05 +2009_003640__03 +2009_003696__01 +2009_003707__04 +2009_003806__01 +2009_003858__03 +2009_003971__02 +2009_004021__03 +2009_004084__03 +2009_004125__04 +2009_004247__05 +2009_004324__05 +2009_004509__03 +2009_004540__03 +2009_004568__03 +2009_004579__05 +2009_004635__04 +2009_004653__01 +2009_004848__02 +2009_004882__02 +2009_004886__03 +2009_004895__03 +2009_004969__01 +2009_005038__05 +2009_005137__03 +2009_005156__02 +2009_005189__01 +2009_005190__05 +2009_005260__03 +2009_005262__03 +2009_005302__05 +2010_000065__02 +2010_000083__02 +2010_000084__04 +2010_000238__01 +2010_000241__03 +2010_000272__04 +2010_000342__02 +2010_000426__05 +2010_000572__01 +2010_000622__01 +2010_000814__03 +2010_000906__04 +2010_000961__03 +2010_001016__03 +2010_001017__01 +2010_001024__01 +2010_001036__04 +2010_001061__03 +2010_001069__03 +2010_001174__01 +2010_001367__02 +2010_001367__05 +2010_001448__01 +2010_001830__05 +2010_001995__03 +2010_002017__05 +2010_002030__02 +2010_002142__03 +2010_002147__01 +2010_002150__04 +2010_002200__01 +2010_002310__01 +2010_002536__02 +2010_002546__04 +2010_002693__02 +2010_002939__01 +2010_003127__01 +2010_003132__01 +2010_003168__03 +2010_003362__03 +2010_003365__01 +2010_003418__03 +2010_003468__05 +2010_003473__03 +2010_003495__01 +2010_003547__04 +2010_003716__01 +2010_003771__03 +2010_003781__05 +2010_003820__03 +2010_003912__02 +2010_003915__01 +2010_004041__04 +2010_004056__05 +2010_004208__04 +2010_004314__01 +2010_004419__01 +2010_004520__05 +2010_004529__05 +2010_004551__05 +2010_004556__03 +2010_004559__03 +2010_004662__04 +2010_004772__04 +2010_004828__05 +2010_004994__03 +2010_005252__04 +2010_005401__04 +2010_005428__03 +2010_005496__05 +2010_005531__03 +2010_005534__01 +2010_005582__05 +2010_005664__02 +2010_005705__04 +2010_005718__01 +2010_005762__05 +2010_005877__01 +2010_005888__01 +2010_006034__01 +2010_006070__02 +2011_000066__05 +2011_000112__03 +2011_000185__03 +2011_000234__04 +2011_000238__04 +2011_000412__02 +2011_000435__04 +2011_000456__03 +2011_000482__03 +2011_000585__02 +2011_000669__03 +2011_000747__05 +2011_000874__01 +2011_001114__01 +2011_001161__04 +2011_001263__01 +2011_001287__03 +2011_001407__01 +2011_001421__03 +2011_001434__01 +2011_001589__04 +2011_001624__01 +2011_001793__04 +2011_001880__01 +2011_001988__02 +2011_002064__02 +2011_002098__05 +2011_002223__02 +2011_002295__03 +2011_002327__01 +2011_002515__01 +2011_002675__01 +2011_002713__02 +2011_002754__04 +2011_002863__05 +2011_002929__01 +2011_002975__04 +2011_003003__02 +2011_003030__03 +2011_003145__03 +2011_003271__05 diff --git a/submodules/lang_seg/fewshot_data/data/splits/pascal/val/fold1.txt b/submodules/lang_seg/fewshot_data/data/splits/pascal/val/fold1.txt new file mode 100644 index 0000000000000000000000000000000000000000..f63aa822c7dcbfea07cdfc6c42d976699429745f --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/pascal/val/fold1.txt @@ -0,0 +1,451 @@ +2007_000452__09 +2007_000464__10 +2007_000491__10 +2007_000663__06 +2007_000663__07 +2007_000727__06 +2007_000727__07 +2007_000804__09 +2007_000830__09 +2007_001299__10 +2007_001321__07 +2007_001457__09 +2007_001677__09 +2007_001717__09 +2007_001763__08 +2007_001774__08 +2007_001884__06 +2007_002268__08 +2007_002387__10 +2007_002445__08 +2007_002470__08 +2007_002539__06 +2007_002597__08 +2007_002643__07 +2007_002903__10 +2007_003011__09 +2007_003051__07 +2007_003101__06 +2007_003106__08 +2007_003137__06 +2007_003143__07 +2007_003169__08 +2007_003195__06 +2007_003201__10 +2007_003503__06 +2007_003503__07 +2007_003621__06 +2007_003711__06 +2007_003786__06 +2007_003841__10 +2007_003917__07 +2007_003991__08 +2007_004193__09 +2007_004392__09 +2007_004405__09 +2007_004510__09 +2007_004712__09 +2007_004856__08 +2007_004866__08 +2007_005074__07 +2007_005114__10 +2007_005296__07 +2007_005331__07 +2007_005460__08 +2007_005547__07 +2007_005547__10 +2007_005844__09 +2007_005845__08 +2007_005911__06 +2007_005978__06 +2007_006035__07 +2007_006086__09 +2007_006241__09 +2007_006260__08 +2007_006277__07 +2007_006348__09 +2007_006553__09 +2007_006761__10 +2007_006841__10 +2007_007414__07 +2007_007417__08 +2007_007524__08 +2007_007815__07 +2007_007818__07 +2007_007996__09 +2007_008106__09 +2007_008110__09 +2007_008543__09 +2007_008722__10 +2007_008747__06 +2007_008815__08 +2007_008897__09 +2007_008973__10 +2007_009015__06 +2007_009015__07 +2007_009068__09 +2007_009084__09 +2007_009096__07 +2007_009221__08 +2007_009245__10 +2007_009346__08 +2007_009392__06 +2007_009392__07 +2007_009413__09 +2007_009521__09 +2007_009764__06 +2007_009794__08 +2007_009897__10 +2007_009923__08 +2007_009938__07 +2008_000009__10 +2008_000073__10 +2008_000075__06 +2008_000107__09 +2008_000149__09 +2008_000182__08 +2008_000345__08 +2008_000401__08 +2008_000464__08 +2008_000501__07 +2008_000673__09 +2008_000853__08 +2008_000919__10 +2008_001078__08 +2008_001433__08 +2008_001439__09 +2008_001513__08 +2008_001640__08 +2008_001715__09 +2008_001885__08 +2008_002152__08 +2008_002205__06 +2008_002212__07 +2008_002379__09 +2008_002521__09 +2008_002623__08 +2008_002681__08 +2008_002778__10 +2008_002958__07 +2008_003141__06 +2008_003141__07 +2008_003333__07 +2008_003477__09 +2008_003499__08 +2008_003577__07 +2008_003777__06 +2008_003821__09 +2008_003846__07 +2008_004069__07 +2008_004339__07 +2008_004552__07 +2008_004612__09 +2008_004701__10 +2008_005097__10 +2008_005105__10 +2008_005245__07 +2008_005676__06 +2008_006008__09 +2008_006063__10 +2008_006254__07 +2008_006325__08 +2008_006341__08 +2008_006480__08 +2008_006528__10 +2008_006554__06 +2008_006986__07 +2008_007025__10 +2008_007031__10 +2008_007048__09 +2008_007123__10 +2008_007194__09 +2008_007273__10 +2008_007378__09 +2008_007402__09 +2008_007527__09 +2008_007548__08 +2008_007596__10 +2008_007737__09 +2008_007797__06 +2008_007804__07 +2008_007828__09 +2008_008252__06 +2008_008301__06 +2008_008469__06 +2008_008682__06 +2009_000013__08 +2009_000080__08 +2009_000219__10 +2009_000309__10 +2009_000335__06 +2009_000335__07 +2009_000426__06 +2009_000455__06 +2009_000457__07 +2009_000523__07 +2009_000641__10 +2009_000716__08 +2009_000731__10 +2009_000771__10 +2009_000825__07 +2009_000964__08 +2009_001008__08 +2009_001082__06 +2009_001240__07 +2009_001255__07 +2009_001299__09 +2009_001391__08 +2009_001411__08 +2009_001536__07 +2009_001775__09 +2009_001804__06 +2009_001816__06 +2009_001854__06 +2009_002035__10 +2009_002122__10 +2009_002150__10 +2009_002164__07 +2009_002171__10 +2009_002221__10 +2009_002238__06 +2009_002238__07 +2009_002239__07 +2009_002268__08 +2009_002346__09 +2009_002415__09 +2009_002487__09 +2009_002527__08 +2009_002535__06 +2009_002549__10 +2009_002571__09 +2009_002618__07 +2009_002635__10 +2009_002753__08 +2009_002936__08 +2009_002990__07 +2009_003003__07 +2009_003059__10 +2009_003071__09 +2009_003269__07 +2009_003304__06 +2009_003387__07 +2009_003406__07 +2009_003494__09 +2009_003507__09 +2009_003542__10 +2009_003549__07 +2009_003569__10 +2009_003589__07 +2009_003703__06 +2009_003771__08 +2009_003773__10 +2009_003849__09 +2009_003895__09 +2009_003904__08 +2009_004072__06 +2009_004140__09 +2009_004217__09 +2009_004248__08 +2009_004455__07 +2009_004504__08 +2009_004590__06 +2009_004594__07 +2009_004687__09 +2009_004721__08 +2009_004732__06 +2009_004748__07 +2009_004789__06 +2009_004859__09 +2009_004867__06 +2009_005158__08 +2009_005219__08 +2009_005231__06 +2010_000003__09 +2010_000160__07 +2010_000163__08 +2010_000372__07 +2010_000427__10 +2010_000530__07 +2010_000552__08 +2010_000573__06 +2010_000628__07 +2010_000639__09 +2010_000682__06 +2010_000683__08 +2010_000724__08 +2010_000907__10 +2010_000941__08 +2010_000952__07 +2010_001000__10 +2010_001010__10 +2010_001070__08 +2010_001206__06 +2010_001292__08 +2010_001331__08 +2010_001351__08 +2010_001403__06 +2010_001403__07 +2010_001534__08 +2010_001553__07 +2010_001579__09 +2010_001646__06 +2010_001656__08 +2010_001692__10 +2010_001699__09 +2010_001767__07 +2010_001851__09 +2010_001913__08 +2010_002017__07 +2010_002017__09 +2010_002025__08 +2010_002137__08 +2010_002146__08 +2010_002305__08 +2010_002336__09 +2010_002348__08 +2010_002361__07 +2010_002390__10 +2010_002422__08 +2010_002512__08 +2010_002531__08 +2010_002546__06 +2010_002623__09 +2010_002693__08 +2010_002693__09 +2010_002763__08 +2010_002763__10 +2010_002868__06 +2010_002900__08 +2010_002902__07 +2010_002921__09 +2010_002929__07 +2010_002988__07 +2010_003123__07 +2010_003183__10 +2010_003231__07 +2010_003239__10 +2010_003275__08 +2010_003276__07 +2010_003293__06 +2010_003302__09 +2010_003325__09 +2010_003381__07 +2010_003402__08 +2010_003409__09 +2010_003446__07 +2010_003453__07 +2010_003468__08 +2010_003531__09 +2010_003675__08 +2010_003746__07 +2010_003758__08 +2010_003764__08 +2010_003768__07 +2010_003772__06 +2010_003781__08 +2010_003813__07 +2010_003854__07 +2010_003971__08 +2010_003971__09 +2010_004104__08 +2010_004120__08 +2010_004320__08 +2010_004322__10 +2010_004348__06 +2010_004369__08 +2010_004472__07 +2010_004479__08 +2010_004635__10 +2010_004763__09 +2010_004783__09 +2010_004789__10 +2010_004815__08 +2010_004825__09 +2010_004861__08 +2010_004946__07 +2010_005013__07 +2010_005021__08 +2010_005021__09 +2010_005063__06 +2010_005108__08 +2010_005118__06 +2010_005160__06 +2010_005166__10 +2010_005284__06 +2010_005344__08 +2010_005421__08 +2010_005432__07 +2010_005501__07 +2010_005508__08 +2010_005606__08 +2010_005709__08 +2010_005718__07 +2010_005860__07 +2010_005899__08 +2010_006070__07 +2011_000178__06 +2011_000226__09 +2011_000239__06 +2011_000248__06 +2011_000312__06 +2011_000338__09 +2011_000419__08 +2011_000503__07 +2011_000548__10 +2011_000566__10 +2011_000607__09 +2011_000661__08 +2011_000661__09 +2011_000780__08 +2011_000789__08 +2011_000809__09 +2011_000813__08 +2011_000813__09 +2011_000830__06 +2011_000843__09 +2011_000888__06 +2011_000900__07 +2011_000969__06 +2011_001047__10 +2011_001064__06 +2011_001071__09 +2011_001110__07 +2011_001159__10 +2011_001232__10 +2011_001292__08 +2011_001341__06 +2011_001346__09 +2011_001447__09 +2011_001530__10 +2011_001534__08 +2011_001546__10 +2011_001567__09 +2011_001597__08 +2011_001601__08 +2011_001607__08 +2011_001665__09 +2011_001708__10 +2011_001775__08 +2011_001782__10 +2011_001812__09 +2011_002041__09 +2011_002064__07 +2011_002124__09 +2011_002200__09 +2011_002298__09 +2011_002322__07 +2011_002343__09 +2011_002358__09 +2011_002391__09 +2011_002509__09 +2011_002592__07 +2011_002644__09 +2011_002685__08 +2011_002812__07 +2011_002885__10 +2011_003011__09 +2011_003019__07 +2011_003019__10 +2011_003055__07 +2011_003103__09 +2011_003114__06 diff --git a/submodules/lang_seg/fewshot_data/data/splits/pascal/val/fold2.txt b/submodules/lang_seg/fewshot_data/data/splits/pascal/val/fold2.txt new file mode 100644 index 0000000000000000000000000000000000000000..3033f09423f705528eeaca4b4f3744c7582e6194 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/pascal/val/fold2.txt @@ -0,0 +1,725 @@ +2007_000129__15 +2007_000323__15 +2007_000332__13 +2007_000346__15 +2007_000762__11 +2007_000762__15 +2007_000783__13 +2007_000783__15 +2007_000799__13 +2007_000799__15 +2007_000830__11 +2007_000847__11 +2007_000847__15 +2007_000999__15 +2007_001175__15 +2007_001239__12 +2007_001284__15 +2007_001311__15 +2007_001408__15 +2007_001423__15 +2007_001430__11 +2007_001430__15 +2007_001526__15 +2007_001585__15 +2007_001586__13 +2007_001586__15 +2007_001594__15 +2007_001630__15 +2007_001677__11 +2007_001678__15 +2007_001717__15 +2007_001763__12 +2007_001955__13 +2007_002046__13 +2007_002119__15 +2007_002260__14 +2007_002268__12 +2007_002378__15 +2007_002426__15 +2007_002539__15 +2007_002565__15 +2007_002597__12 +2007_002624__11 +2007_002624__15 +2007_002643__15 +2007_002728__15 +2007_002823__14 +2007_002823__15 +2007_002824__15 +2007_002852__12 +2007_003011__11 +2007_003020__15 +2007_003022__13 +2007_003022__15 +2007_003088__15 +2007_003106__15 +2007_003110__12 +2007_003134__15 +2007_003188__15 +2007_003194__12 +2007_003367__14 +2007_003367__15 +2007_003373__12 +2007_003373__15 +2007_003530__15 +2007_003621__15 +2007_003742__11 +2007_003742__15 +2007_003872__12 +2007_004033__14 +2007_004033__15 +2007_004112__12 +2007_004112__15 +2007_004121__15 +2007_004189__12 +2007_004275__14 +2007_004275__15 +2007_004281__15 +2007_004380__14 +2007_004380__15 +2007_004392__15 +2007_004405__11 +2007_004538__13 +2007_004538__15 +2007_004644__12 +2007_004712__11 +2007_004712__15 +2007_004722__13 +2007_004722__15 +2007_004902__13 +2007_004902__15 +2007_005114__13 +2007_005114__15 +2007_005149__12 +2007_005173__14 +2007_005173__15 +2007_005281__15 +2007_005304__15 +2007_005331__13 +2007_005331__15 +2007_005354__14 +2007_005354__15 +2007_005509__15 +2007_005547__15 +2007_005608__14 +2007_005608__15 +2007_005696__12 +2007_005759__14 +2007_005803__11 +2007_005844__11 +2007_005845__15 +2007_006028__15 +2007_006076__15 +2007_006086__11 +2007_006117__15 +2007_006171__12 +2007_006171__15 +2007_006241__11 +2007_006364__13 +2007_006364__15 +2007_006373__15 +2007_006444__12 +2007_006444__15 +2007_006560__15 +2007_006647__14 +2007_006647__15 +2007_006698__15 +2007_006802__15 +2007_006841__15 +2007_006864__15 +2007_006866__13 +2007_006866__15 +2007_007007__11 +2007_007007__15 +2007_007109__13 +2007_007109__15 +2007_007195__15 +2007_007203__15 +2007_007211__14 +2007_007235__15 +2007_007417__12 +2007_007493__15 +2007_007498__11 +2007_007498__15 +2007_007651__11 +2007_007651__15 +2007_007688__14 +2007_007748__13 +2007_007748__15 +2007_007795__15 +2007_007810__11 +2007_007810__15 +2007_007815__15 +2007_007836__15 +2007_007849__15 +2007_007996__15 +2007_008110__15 +2007_008204__15 +2007_008222__12 +2007_008256__13 +2007_008256__15 +2007_008260__12 +2007_008374__15 +2007_008415__12 +2007_008430__15 +2007_008596__13 +2007_008596__15 +2007_008708__15 +2007_008802__13 +2007_008897__15 +2007_008944__15 +2007_008964__12 +2007_008964__15 +2007_008980__12 +2007_009068__15 +2007_009084__12 +2007_009084__14 +2007_009251__13 +2007_009251__15 +2007_009258__15 +2007_009320__15 +2007_009331__12 +2007_009331__13 +2007_009331__15 +2007_009413__11 +2007_009413__15 +2007_009521__11 +2007_009562__12 +2007_009592__12 +2007_009654__15 +2007_009655__15 +2007_009684__15 +2007_009687__15 +2007_009691__14 +2007_009691__15 +2007_009706__11 +2007_009750__15 +2007_009756__14 +2007_009756__15 +2007_009841__13 +2007_009938__14 +2008_000080__12 +2008_000213__15 +2008_000215__15 +2008_000223__15 +2008_000233__15 +2008_000234__15 +2008_000239__12 +2008_000270__12 +2008_000270__15 +2008_000271__15 +2008_000359__15 +2008_000474__15 +2008_000510__15 +2008_000573__11 +2008_000573__15 +2008_000602__13 +2008_000630__15 +2008_000661__12 +2008_000661__15 +2008_000662__15 +2008_000666__15 +2008_000673__15 +2008_000700__15 +2008_000725__15 +2008_000731__15 +2008_000763__11 +2008_000763__15 +2008_000765__13 +2008_000782__14 +2008_000795__15 +2008_000811__14 +2008_000811__15 +2008_000863__12 +2008_000943__12 +2008_000992__15 +2008_001013__15 +2008_001028__15 +2008_001070__12 +2008_001074__15 +2008_001076__15 +2008_001150__14 +2008_001170__15 +2008_001231__15 +2008_001249__15 +2008_001283__15 +2008_001308__15 +2008_001379__12 +2008_001404__15 +2008_001478__12 +2008_001491__15 +2008_001504__15 +2008_001531__15 +2008_001547__15 +2008_001629__15 +2008_001682__13 +2008_001821__15 +2008_001874__15 +2008_001895__12 +2008_001895__15 +2008_001992__13 +2008_001992__15 +2008_002212__15 +2008_002239__12 +2008_002240__14 +2008_002241__15 +2008_002379__11 +2008_002383__14 +2008_002495__15 +2008_002536__12 +2008_002588__15 +2008_002775__11 +2008_002775__15 +2008_002835__13 +2008_002835__15 +2008_002859__12 +2008_002864__11 +2008_002864__15 +2008_002904__12 +2008_002929__15 +2008_002936__12 +2008_002942__15 +2008_002958__12 +2008_003034__15 +2008_003076__15 +2008_003108__15 +2008_003141__15 +2008_003210__15 +2008_003238__12 +2008_003238__15 +2008_003330__15 +2008_003333__14 +2008_003333__15 +2008_003379__13 +2008_003451__14 +2008_003451__15 +2008_003461__13 +2008_003461__15 +2008_003477__11 +2008_003492__15 +2008_003511__12 +2008_003511__15 +2008_003546__15 +2008_003576__12 +2008_003676__15 +2008_003733__15 +2008_003782__13 +2008_003856__15 +2008_003874__15 +2008_004101__15 +2008_004140__11 +2008_004140__15 +2008_004175__13 +2008_004345__14 +2008_004396__13 +2008_004399__14 +2008_004399__15 +2008_004575__11 +2008_004575__15 +2008_004624__13 +2008_004654__15 +2008_004687__13 +2008_004705__13 +2008_005049__14 +2008_005089__15 +2008_005145__11 +2008_005197__12 +2008_005197__15 +2008_005245__14 +2008_005245__15 +2008_005399__15 +2008_005422__14 +2008_005445__15 +2008_005525__13 +2008_005637__14 +2008_005642__13 +2008_005691__13 +2008_005738__15 +2008_005812__15 +2008_005915__14 +2008_006008__11 +2008_006036__13 +2008_006108__11 +2008_006108__15 +2008_006130__12 +2008_006216__15 +2008_006219__13 +2008_006254__15 +2008_006275__15 +2008_006341__15 +2008_006408__11 +2008_006408__15 +2008_006526__14 +2008_006526__15 +2008_006554__15 +2008_006722__12 +2008_006722__15 +2008_006874__14 +2008_006874__15 +2008_006981__12 +2008_007048__11 +2008_007219__15 +2008_007378__11 +2008_007378__12 +2008_007392__13 +2008_007392__15 +2008_007402__11 +2008_007402__15 +2008_007513__12 +2008_007737__15 +2008_007828__15 +2008_007945__13 +2008_007994__15 +2008_008051__11 +2008_008127__14 +2008_008127__15 +2008_008221__15 +2008_008335__11 +2008_008335__15 +2008_008362__11 +2008_008362__15 +2008_008392__13 +2008_008393__13 +2008_008421__13 +2008_008469__15 +2009_000012__13 +2009_000074__14 +2009_000074__15 +2009_000156__12 +2009_000219__15 +2009_000309__15 +2009_000412__13 +2009_000418__15 +2009_000421__15 +2009_000457__15 +2009_000704__15 +2009_000705__13 +2009_000727__13 +2009_000730__14 +2009_000730__15 +2009_000825__14 +2009_000825__15 +2009_000839__12 +2009_000892__12 +2009_000931__13 +2009_000935__12 +2009_001215__11 +2009_001215__15 +2009_001299__15 +2009_001433__13 +2009_001433__15 +2009_001535__12 +2009_001663__15 +2009_001687__12 +2009_001687__15 +2009_001718__15 +2009_001768__15 +2009_001854__15 +2009_002012__12 +2009_002042__15 +2009_002097__13 +2009_002155__12 +2009_002165__13 +2009_002185__15 +2009_002239__14 +2009_002239__15 +2009_002317__14 +2009_002317__15 +2009_002346__12 +2009_002346__15 +2009_002372__15 +2009_002382__14 +2009_002382__15 +2009_002415__11 +2009_002445__12 +2009_002487__11 +2009_002539__12 +2009_002571__11 +2009_002584__15 +2009_002649__15 +2009_002651__14 +2009_002651__15 +2009_002732__15 +2009_002975__13 +2009_003003__11 +2009_003003__15 +2009_003063__12 +2009_003065__15 +2009_003071__11 +2009_003071__15 +2009_003123__11 +2009_003196__14 +2009_003217__12 +2009_003241__12 +2009_003269__15 +2009_003323__13 +2009_003323__15 +2009_003466__12 +2009_003481__13 +2009_003494__15 +2009_003507__11 +2009_003576__14 +2009_003576__15 +2009_003756__12 +2009_003804__13 +2009_003810__12 +2009_003849__11 +2009_003849__15 +2009_003903__13 +2009_003928__12 +2009_003991__11 +2009_003991__15 +2009_004033__12 +2009_004043__14 +2009_004043__15 +2009_004140__11 +2009_004221__15 +2009_004455__14 +2009_004497__13 +2009_004507__12 +2009_004507__15 +2009_004581__12 +2009_004592__12 +2009_004738__14 +2009_004738__15 +2009_004848__15 +2009_004859__11 +2009_004859__15 +2009_004942__13 +2009_004987__14 +2009_004987__15 +2009_004994__12 +2009_004994__15 +2009_005038__11 +2009_005038__15 +2009_005078__14 +2009_005087__15 +2009_005217__13 +2009_005217__15 +2010_000003__12 +2010_000038__13 +2010_000038__15 +2010_000087__14 +2010_000087__15 +2010_000110__12 +2010_000110__15 +2010_000159__12 +2010_000174__11 +2010_000174__15 +2010_000216__12 +2010_000238__15 +2010_000256__15 +2010_000422__12 +2010_000530__15 +2010_000559__15 +2010_000639__12 +2010_000666__13 +2010_000666__15 +2010_000738__15 +2010_000788__12 +2010_000874__13 +2010_000904__12 +2010_001024__15 +2010_001124__12 +2010_001251__14 +2010_001264__12 +2010_001313__14 +2010_001313__15 +2010_001367__15 +2010_001376__12 +2010_001451__13 +2010_001553__14 +2010_001563__12 +2010_001563__15 +2010_001579__11 +2010_001579__15 +2010_001692__15 +2010_001699__15 +2010_001734__15 +2010_001767__15 +2010_001851__11 +2010_001908__12 +2010_001956__12 +2010_002017__15 +2010_002137__15 +2010_002161__13 +2010_002161__15 +2010_002228__12 +2010_002251__14 +2010_002251__15 +2010_002271__14 +2010_002336__11 +2010_002396__14 +2010_002396__15 +2010_002480__12 +2010_002623__15 +2010_002691__13 +2010_002763__15 +2010_002792__15 +2010_002902__15 +2010_002929__15 +2010_003014__15 +2010_003060__12 +2010_003187__12 +2010_003207__14 +2010_003239__15 +2010_003325__11 +2010_003325__15 +2010_003381__15 +2010_003409__15 +2010_003446__15 +2010_003506__12 +2010_003531__11 +2010_003532__13 +2010_003597__11 +2010_003597__15 +2010_003746__12 +2010_003746__15 +2010_003947__14 +2010_003971__11 +2010_004042__14 +2010_004165__12 +2010_004165__15 +2010_004219__14 +2010_004219__15 +2010_004337__15 +2010_004355__14 +2010_004432__15 +2010_004472__15 +2010_004479__15 +2010_004519__13 +2010_004550__12 +2010_004559__15 +2010_004628__12 +2010_004697__14 +2010_004697__15 +2010_004795__12 +2010_004815__15 +2010_004825__11 +2010_004828__15 +2010_004856__13 +2010_004941__14 +2010_004951__15 +2010_005046__11 +2010_005046__15 +2010_005118__15 +2010_005159__12 +2010_005160__14 +2010_005166__15 +2010_005174__13 +2010_005206__12 +2010_005245__12 +2010_005245__15 +2010_005252__14 +2010_005252__15 +2010_005284__15 +2010_005366__14 +2010_005433__14 +2010_005501__14 +2010_005575__12 +2010_005582__15 +2010_005606__15 +2010_005626__11 +2010_005626__15 +2010_005644__12 +2010_005709__15 +2010_005871__15 +2010_005991__12 +2010_005991__15 +2010_005992__12 +2011_000045__12 +2011_000051__15 +2011_000054__15 +2011_000178__15 +2011_000226__11 +2011_000248__15 +2011_000338__11 +2011_000396__13 +2011_000435__15 +2011_000438__15 +2011_000455__14 +2011_000455__15 +2011_000479__15 +2011_000512__14 +2011_000526__13 +2011_000536__12 +2011_000566__15 +2011_000585__15 +2011_000598__11 +2011_000618__14 +2011_000618__15 +2011_000638__15 +2011_000780__15 +2011_000809__11 +2011_000809__15 +2011_000843__15 +2011_000953__11 +2011_000953__15 +2011_001014__12 +2011_001060__15 +2011_001069__15 +2011_001071__15 +2011_001159__15 +2011_001276__11 +2011_001276__12 +2011_001276__15 +2011_001346__15 +2011_001416__15 +2011_001447__15 +2011_001530__15 +2011_001567__15 +2011_001619__15 +2011_001642__12 +2011_001665__11 +2011_001674__15 +2011_001714__12 +2011_001714__15 +2011_001722__13 +2011_001745__12 +2011_001794__15 +2011_001862__11 +2011_001862__12 +2011_001868__12 +2011_001984__12 +2011_001988__15 +2011_002002__15 +2011_002040__12 +2011_002075__11 +2011_002075__15 +2011_002098__12 +2011_002110__12 +2011_002110__15 +2011_002121__12 +2011_002124__15 +2011_002156__12 +2011_002200__11 +2011_002200__15 +2011_002247__15 +2011_002279__12 +2011_002298__12 +2011_002308__15 +2011_002317__15 +2011_002322__14 +2011_002322__15 +2011_002343__15 +2011_002358__11 +2011_002358__15 +2011_002371__12 +2011_002498__15 +2011_002509__15 +2011_002532__15 +2011_002575__15 +2011_002578__15 +2011_002589__12 +2011_002623__15 +2011_002641__15 +2011_002675__15 +2011_002951__13 +2011_002997__15 +2011_003019__14 +2011_003019__15 +2011_003085__13 +2011_003114__15 +2011_003240__15 +2011_003256__12 diff --git a/submodules/lang_seg/fewshot_data/data/splits/pascal/val/fold3.txt b/submodules/lang_seg/fewshot_data/data/splits/pascal/val/fold3.txt new file mode 100644 index 0000000000000000000000000000000000000000..8a797acfd48c4c3626318201a852184059ddac66 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/data/splits/pascal/val/fold3.txt @@ -0,0 +1,346 @@ +2007_000042__19 +2007_000123__19 +2007_000175__17 +2007_000187__20 +2007_000452__18 +2007_000559__20 +2007_000629__19 +2007_000636__19 +2007_000661__18 +2007_000676__17 +2007_000804__18 +2007_000925__17 +2007_001154__18 +2007_001175__20 +2007_001408__16 +2007_001430__16 +2007_001430__20 +2007_001457__18 +2007_001458__18 +2007_001585__18 +2007_001594__17 +2007_001678__20 +2007_001717__20 +2007_001733__17 +2007_001763__18 +2007_001763__20 +2007_002119__20 +2007_002132__20 +2007_002268__18 +2007_002284__16 +2007_002378__16 +2007_002426__18 +2007_002427__18 +2007_002565__19 +2007_002618__17 +2007_002648__17 +2007_002728__19 +2007_003011__18 +2007_003011__20 +2007_003169__18 +2007_003367__16 +2007_003499__19 +2007_003506__16 +2007_003530__18 +2007_003587__19 +2007_003714__17 +2007_003848__19 +2007_003957__19 +2007_004190__20 +2007_004193__20 +2007_004275__16 +2007_004281__19 +2007_004483__19 +2007_004510__20 +2007_004558__16 +2007_004649__19 +2007_004712__16 +2007_004969__17 +2007_005469__17 +2007_005626__19 +2007_005689__19 +2007_005813__16 +2007_005857__16 +2007_005915__17 +2007_006171__18 +2007_006348__20 +2007_006373__18 +2007_006678__17 +2007_006680__19 +2007_006802__19 +2007_007130__20 +2007_007165__17 +2007_007168__19 +2007_007195__19 +2007_007196__20 +2007_007203__20 +2007_007417__18 +2007_007534__17 +2007_007624__16 +2007_007795__16 +2007_007881__19 +2007_007996__18 +2007_008204__20 +2007_008260__18 +2007_008339__19 +2007_008374__20 +2007_008543__18 +2007_008547__16 +2007_009068__18 +2007_009252__18 +2007_009320__17 +2007_009419__16 +2007_009446__20 +2007_009521__18 +2007_009521__20 +2007_009592__18 +2007_009655__18 +2007_009684__18 +2007_009750__16 +2008_000016__20 +2008_000149__18 +2008_000270__18 +2008_000391__16 +2008_000589__18 +2008_000657__19 +2008_001078__16 +2008_001283__16 +2008_001688__16 +2008_001688__20 +2008_001966__16 +2008_002273__16 +2008_002379__16 +2008_002464__20 +2008_002536__17 +2008_002680__20 +2008_002900__19 +2008_002929__18 +2008_003003__20 +2008_003026__20 +2008_003105__19 +2008_003135__16 +2008_003676__16 +2008_003709__18 +2008_003733__18 +2008_003885__20 +2008_004172__18 +2008_004212__19 +2008_004279__20 +2008_004367__19 +2008_004453__17 +2008_004477__16 +2008_004562__18 +2008_004610__19 +2008_004621__17 +2008_004754__20 +2008_004854__17 +2008_004910__20 +2008_005089__20 +2008_005217__16 +2008_005242__16 +2008_005254__20 +2008_005439__20 +2008_005445__20 +2008_005544__19 +2008_005633__17 +2008_005680__16 +2008_006055__19 +2008_006159__20 +2008_006327__17 +2008_006523__19 +2008_006553__19 +2008_006752__19 +2008_006784__18 +2008_006835__17 +2008_007497__17 +2008_007527__20 +2008_007677__17 +2008_007814__17 +2008_007828__20 +2008_008103__18 +2008_008221__19 +2008_008434__16 +2009_000022__19 +2009_000039__17 +2009_000087__18 +2009_000096__18 +2009_000136__20 +2009_000242__18 +2009_000391__20 +2009_000418__16 +2009_000418__18 +2009_000487__18 +2009_000488__16 +2009_000488__20 +2009_000628__19 +2009_000675__17 +2009_000704__20 +2009_000712__19 +2009_000732__18 +2009_000845__19 +2009_000924__17 +2009_001300__19 +2009_001333__19 +2009_001363__20 +2009_001505__17 +2009_001644__16 +2009_001644__18 +2009_001644__20 +2009_001684__16 +2009_001731__18 +2009_001768__17 +2009_001775__16 +2009_001775__18 +2009_001991__17 +2009_002082__17 +2009_002094__20 +2009_002202__19 +2009_002265__19 +2009_002291__19 +2009_002346__18 +2009_002366__20 +2009_002390__18 +2009_002487__16 +2009_002562__20 +2009_002568__19 +2009_002571__16 +2009_002571__18 +2009_002573__20 +2009_002584__16 +2009_002638__19 +2009_002732__18 +2009_002887__19 +2009_002982__19 +2009_003105__19 +2009_003123__18 +2009_003299__19 +2009_003311__19 +2009_003433__19 +2009_003523__20 +2009_003551__20 +2009_003564__16 +2009_003564__18 +2009_003607__18 +2009_003666__17 +2009_003857__20 +2009_003895__18 +2009_003895__20 +2009_003938__19 +2009_004099__18 +2009_004140__18 +2009_004255__19 +2009_004298__18 +2009_004687__18 +2009_004730__19 +2009_004799__19 +2009_004993__18 +2009_004993__20 +2009_005148__19 +2009_005220__19 +2010_000256__18 +2010_000284__18 +2010_000309__17 +2010_000318__20 +2010_000330__16 +2010_000639__16 +2010_000738__20 +2010_000764__19 +2010_001011__17 +2010_001079__17 +2010_001104__19 +2010_001149__18 +2010_001151__19 +2010_001246__16 +2010_001256__17 +2010_001327__18 +2010_001367__20 +2010_001522__17 +2010_001557__17 +2010_001577__17 +2010_001699__16 +2010_001734__19 +2010_001752__20 +2010_001767__18 +2010_001773__16 +2010_001851__16 +2010_001951__19 +2010_001962__18 +2010_002106__17 +2010_002137__16 +2010_002137__18 +2010_002232__17 +2010_002531__18 +2010_002682__19 +2010_002921__20 +2010_003014__18 +2010_003123__16 +2010_003302__16 +2010_003514__19 +2010_003541__17 +2010_003597__18 +2010_003781__16 +2010_003956__19 +2010_004149__19 +2010_004226__17 +2010_004382__16 +2010_004479__20 +2010_004757__16 +2010_004757__18 +2010_004783__18 +2010_004825__16 +2010_004857__20 +2010_004951__19 +2010_004980__19 +2010_005180__18 +2010_005187__16 +2010_005305__20 +2010_005606__18 +2010_005706__19 +2010_005719__17 +2010_005727__19 +2010_005788__17 +2010_005860__16 +2010_005871__19 +2010_005991__18 +2010_006054__19 +2011_000070__18 +2011_000173__18 +2011_000283__19 +2011_000291__19 +2011_000310__18 +2011_000436__17 +2011_000521__19 +2011_000747__16 +2011_001005__18 +2011_001060__19 +2011_001281__19 +2011_001350__17 +2011_001567__18 +2011_001601__18 +2011_001614__19 +2011_001674__18 +2011_001713__16 +2011_001713__18 +2011_001726__20 +2011_001794__18 +2011_001862__18 +2011_001863__16 +2011_001910__20 +2011_002124__18 +2011_002156__20 +2011_002178__17 +2011_002247__19 +2011_002379__19 +2011_002391__18 +2011_002532__20 +2011_002535__19 +2011_002644__18 +2011_002644__20 +2011_002879__18 +2011_002879__20 +2011_003103__16 +2011_003103__18 +2011_003146__19 +2011_003182__18 +2011_003197__19 +2011_003256__18 diff --git a/submodules/lang_seg/fewshot_data/model/base/conv4d.py b/submodules/lang_seg/fewshot_data/model/base/conv4d.py new file mode 100644 index 0000000000000000000000000000000000000000..fd59f37091f448159cdb211d69728867d1df44f9 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/model/base/conv4d.py @@ -0,0 +1,58 @@ +r""" Implementation of center-pivot 4D convolution """ + +import torch +import torch.nn as nn + + +class CenterPivotConv4d(nn.Module): + r""" CenterPivot 4D conv""" + def __init__(self, in_channels, out_channels, kernel_size, stride, padding, bias=True): + super(CenterPivotConv4d, self).__init__() + + self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size[:2], stride=stride[:2], + bias=bias, padding=padding[:2]) + self.conv2 = nn.Conv2d(in_channels, out_channels, kernel_size[2:], stride=stride[2:], + bias=bias, padding=padding[2:]) + + self.stride34 = stride[2:] + self.kernel_size = kernel_size + self.stride = stride + self.padding = padding + self.idx_initialized = False + + def prune(self, ct): + bsz, ch, ha, wa, hb, wb = ct.size() + if not self.idx_initialized: + idxh = torch.arange(start=0, end=hb, step=self.stride[2:][0], device=ct.device) + idxw = torch.arange(start=0, end=wb, step=self.stride[2:][1], device=ct.device) + self.len_h = len(idxh) + self.len_w = len(idxw) + self.idx = (idxw.repeat(self.len_h, 1) + idxh.repeat(self.len_w, 1).t() * wb).view(-1) + self.idx_initialized = True + ct_pruned = ct.view(bsz, ch, ha, wa, -1).index_select(4, self.idx).view(bsz, ch, ha, wa, self.len_h, self.len_w) + + return ct_pruned + + def forward(self, x): + if self.stride[2:][-1] > 1: + out1 = self.prune(x) + else: + out1 = x + bsz, inch, ha, wa, hb, wb = out1.size() + out1 = out1.permute(0, 4, 5, 1, 2, 3).contiguous().view(-1, inch, ha, wa) + out1 = self.conv1(out1) + outch, o_ha, o_wa = out1.size(-3), out1.size(-2), out1.size(-1) + out1 = out1.view(bsz, hb, wb, outch, o_ha, o_wa).permute(0, 3, 4, 5, 1, 2).contiguous() + + bsz, inch, ha, wa, hb, wb = x.size() + out2 = x.permute(0, 2, 3, 1, 4, 5).contiguous().view(-1, inch, hb, wb) + out2 = self.conv2(out2) + outch, o_hb, o_wb = out2.size(-3), out2.size(-2), out2.size(-1) + out2 = out2.view(bsz, ha, wa, outch, o_hb, o_wb).permute(0, 3, 1, 2, 4, 5).contiguous() + + if out1.size()[-2:] != out2.size()[-2:] and self.padding[-2:] == (0, 0): + out1 = out1.view(bsz, outch, o_ha, o_wa, -1).sum(dim=-1) + out2 = out2.squeeze() + + y = out1 + out2 + return y diff --git a/submodules/lang_seg/fewshot_data/model/base/correlation.py b/submodules/lang_seg/fewshot_data/model/base/correlation.py new file mode 100644 index 0000000000000000000000000000000000000000..7400dbba8d6ee9bd98b524c1b1cf63da415bd1dc --- /dev/null +++ b/submodules/lang_seg/fewshot_data/model/base/correlation.py @@ -0,0 +1,29 @@ +r""" Provides functions that builds/manipulates correlation tensors """ +import torch + + +class Correlation: + + @classmethod + def multilayer_correlation(cls, query_feats, support_feats, stack_ids): + eps = 1e-5 + + corrs = [] + for idx, (query_feat, support_feat) in enumerate(zip(query_feats, support_feats)): + bsz, ch, hb, wb = support_feat.size() + support_feat = support_feat.view(bsz, ch, -1) + support_feat = support_feat / (support_feat.norm(dim=1, p=2, keepdim=True) + eps) + + bsz, ch, ha, wa = query_feat.size() + query_feat = query_feat.view(bsz, ch, -1) + query_feat = query_feat / (query_feat.norm(dim=1, p=2, keepdim=True) + eps) + + corr = torch.bmm(query_feat.transpose(1, 2), support_feat).view(bsz, ha, wa, hb, wb) + corr = corr.clamp(min=0) + corrs.append(corr) + + corr_l4 = torch.stack(corrs[-stack_ids[0]:]).transpose(0, 1).contiguous() + corr_l3 = torch.stack(corrs[-stack_ids[1]:-stack_ids[0]]).transpose(0, 1).contiguous() + corr_l2 = torch.stack(corrs[-stack_ids[2]:-stack_ids[1]]).transpose(0, 1).contiguous() + + return [corr_l4, corr_l3, corr_l2] diff --git a/submodules/lang_seg/fewshot_data/model/base/feature.py b/submodules/lang_seg/fewshot_data/model/base/feature.py new file mode 100644 index 0000000000000000000000000000000000000000..a516f9ded9d9ca72376ae184ef793a23a6d6979b --- /dev/null +++ b/submodules/lang_seg/fewshot_data/model/base/feature.py @@ -0,0 +1,47 @@ +r""" Extracts intermediate features from given backbone network & layer ids """ + + +def extract_feat_vgg(img, backbone, feat_ids, bottleneck_ids=None, lids=None): + r""" Extract intermediate features from VGG """ + feats = [] + feat = img + for lid, module in enumerate(backbone.features): + feat = module(feat) + if lid in feat_ids: + feats.append(feat.clone()) + return feats + + +def extract_feat_res(img, backbone, feat_ids, bottleneck_ids, lids): + r""" Extract intermediate features from ResNet""" + feats = [] + + # Layer 0 + feat = backbone.conv1.forward(img) + feat = backbone.bn1.forward(feat) + feat = backbone.relu.forward(feat) + feat = backbone.maxpool.forward(feat) + + # Layer 1-4 + for hid, (bid, lid) in enumerate(zip(bottleneck_ids, lids)): + res = feat + feat = backbone.__getattr__('layer%d' % lid)[bid].conv1.forward(feat) + feat = backbone.__getattr__('layer%d' % lid)[bid].bn1.forward(feat) + feat = backbone.__getattr__('layer%d' % lid)[bid].relu.forward(feat) + feat = backbone.__getattr__('layer%d' % lid)[bid].conv2.forward(feat) + feat = backbone.__getattr__('layer%d' % lid)[bid].bn2.forward(feat) + feat = backbone.__getattr__('layer%d' % lid)[bid].relu.forward(feat) + feat = backbone.__getattr__('layer%d' % lid)[bid].conv3.forward(feat) + feat = backbone.__getattr__('layer%d' % lid)[bid].bn3.forward(feat) + + if bid == 0: + res = backbone.__getattr__('layer%d' % lid)[bid].downsample.forward(res) + + feat += res + + if hid + 1 in feat_ids: + feats.append(feat.clone()) + + feat = backbone.__getattr__('layer%d' % lid)[bid].relu.forward(feat) + + return feats \ No newline at end of file diff --git a/submodules/lang_seg/fewshot_data/model/hsnet.py b/submodules/lang_seg/fewshot_data/model/hsnet.py new file mode 100644 index 0000000000000000000000000000000000000000..def69910d3cf84d7e5c1948aa9075b2c4b251dd4 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/model/hsnet.py @@ -0,0 +1,101 @@ +r""" Hypercorrelation Squeeze Network """ +from functools import reduce +from operator import add + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchvision.models import resnet +from torchvision.models import vgg + +from fewshot_data.model.base.feature import extract_feat_vgg, extract_feat_res +from fewshot_data.model.base.correlation import Correlation +from fewshot_data.model.learner import HPNLearner + + +class HypercorrSqueezeNetwork(nn.Module): + def __init__(self, backbone, use_original_imgsize): + super(HypercorrSqueezeNetwork, self).__init__() + + # 1. Backbone network initialization + self.backbone_type = backbone + self.use_original_imgsize = use_original_imgsize + if backbone == 'vgg16': + self.backbone = vgg.vgg16(pretrained=True) + self.feat_ids = [17, 19, 21, 24, 26, 28, 30] + self.extract_feats = extract_feat_vgg + nbottlenecks = [2, 2, 3, 3, 3, 1] + elif backbone == 'resnet50': + self.backbone = resnet.resnet50(pretrained=True) + self.feat_ids = list(range(4, 17)) + self.extract_feats = extract_feat_res + nbottlenecks = [3, 4, 6, 3] + elif backbone == 'resnet101': + self.backbone = resnet.resnet101(pretrained=True) + self.feat_ids = list(range(4, 34)) + self.extract_feats = extract_feat_res + nbottlenecks = [3, 4, 23, 3] + else: + raise Exception('Unavailable backbone: %s' % backbone) + + self.bottleneck_ids = reduce(add, list(map(lambda x: list(range(x)), nbottlenecks))) + self.lids = reduce(add, [[i + 1] * x for i, x in enumerate(nbottlenecks)]) + self.stack_ids = torch.tensor(self.lids).bincount().__reversed__().cumsum(dim=0)[:3] + self.backbone.eval() + self.hpn_learner = HPNLearner(list(reversed(nbottlenecks[-3:]))) + self.cross_entropy_loss = nn.CrossEntropyLoss() + + def forward(self, query_img, support_img, support_mask): + with torch.no_grad(): + query_feats = self.extract_feats(query_img, self.backbone, self.feat_ids, self.bottleneck_ids, self.lids) + support_feats = self.extract_feats(support_img, self.backbone, self.feat_ids, self.bottleneck_ids, self.lids) + support_feats = self.mask_feature(support_feats, support_mask.clone()) + corr = Correlation.multilayer_correlation(query_feats, support_feats, self.stack_ids) + + logit_mask = self.hpn_learner(corr) + if not self.use_original_imgsize: + logit_mask = F.interpolate(logit_mask, support_img.size()[2:], mode='bilinear', align_corners=True) + + return logit_mask + + def mask_feature(self, features, support_mask): + for idx, feature in enumerate(features): + mask = F.interpolate(support_mask.unsqueeze(1).float(), feature.size()[2:], mode='bilinear', align_corners=True) + features[idx] = features[idx] * mask + return features + + def predict_mask_nshot(self, batch, nshot): + + # Perform multiple prediction given (nshot) number of different support sets + logit_mask_agg = 0 + for s_idx in range(nshot): + logit_mask = self(batch['query_img'], batch['support_imgs'][:, s_idx], batch['support_masks'][:, s_idx]) + + if self.use_original_imgsize: + org_qry_imsize = tuple([batch['org_query_imsize'][1].item(), batch['org_query_imsize'][0].item()]) + logit_mask = F.interpolate(logit_mask, org_qry_imsize, mode='bilinear', align_corners=True) + + logit_mask_agg += logit_mask.argmax(dim=1).clone() + if nshot == 1: return logit_mask_agg + + # Average & quantize predictions given threshold (=0.5) + bsz = logit_mask_agg.size(0) + max_vote = logit_mask_agg.view(bsz, -1).max(dim=1)[0] + max_vote = torch.stack([max_vote, torch.ones_like(max_vote).long()]) + max_vote = max_vote.max(dim=0)[0].view(bsz, 1, 1) + pred_mask = logit_mask_agg.float() / max_vote + pred_mask[pred_mask < 0.5] = 0 + pred_mask[pred_mask >= 0.5] = 1 + + return pred_mask + + def compute_objective(self, logit_mask, gt_mask): + bsz = logit_mask.size(0) + logit_mask = logit_mask.view(bsz, 2, -1) + gt_mask = gt_mask.view(bsz, -1).long() + + return self.cross_entropy_loss(logit_mask, gt_mask) + + def train_mode(self): + self.train() + self.backbone.eval() # to prevent BN from learning data statistics with exponential averaging diff --git a/submodules/lang_seg/fewshot_data/model/learner.py b/submodules/lang_seg/fewshot_data/model/learner.py new file mode 100644 index 0000000000000000000000000000000000000000..fc003d147c9f1f135f4b13dabdac17f2b49478ce --- /dev/null +++ b/submodules/lang_seg/fewshot_data/model/learner.py @@ -0,0 +1,82 @@ + +import torch.nn as nn +import torch.nn.functional as F + +from fewshot_data.model.base.conv4d import CenterPivotConv4d as Conv4d + + +class HPNLearner(nn.Module): + def __init__(self, inch): + super(HPNLearner, self).__init__() + + def make_building_block(in_channel, out_channels, kernel_sizes, spt_strides, group=4): + assert len(out_channels) == len(kernel_sizes) == len(spt_strides) + + building_block_layers = [] + for idx, (outch, ksz, stride) in enumerate(zip(out_channels, kernel_sizes, spt_strides)): + inch = in_channel if idx == 0 else out_channels[idx - 1] + ksz4d = (ksz,) * 4 + str4d = (1, 1) + (stride,) * 2 + pad4d = (ksz // 2,) * 4 + + building_block_layers.append(Conv4d(inch, outch, ksz4d, str4d, pad4d)) + building_block_layers.append(nn.GroupNorm(group, outch)) + building_block_layers.append(nn.ReLU(inplace=True)) + + return nn.Sequential(*building_block_layers) + + outch1, outch2, outch3 = 16, 64, 128 + + # Squeezing building blocks + self.encoder_layer4 = make_building_block(inch[0], [outch1, outch2, outch3], [3, 3, 3], [2, 2, 2]) + self.encoder_layer3 = make_building_block(inch[1], [outch1, outch2, outch3], [5, 3, 3], [4, 2, 2]) + self.encoder_layer2 = make_building_block(inch[2], [outch1, outch2, outch3], [5, 5, 3], [4, 4, 2]) + + # Mixing building blocks + self.encoder_layer4to3 = make_building_block(outch3, [outch3, outch3, outch3], [3, 3, 3], [1, 1, 1]) + self.encoder_layer3to2 = make_building_block(outch3, [outch3, outch3, outch3], [3, 3, 3], [1, 1, 1]) + + # Decoder layers + self.decoder1 = nn.Sequential(nn.Conv2d(outch3, outch3, (3, 3), padding=(1, 1), bias=True), + nn.ReLU(), + nn.Conv2d(outch3, outch2, (3, 3), padding=(1, 1), bias=True), + nn.ReLU()) + + self.decoder2 = nn.Sequential(nn.Conv2d(outch2, outch2, (3, 3), padding=(1, 1), bias=True), + nn.ReLU(), + nn.Conv2d(outch2, 2, (3, 3), padding=(1, 1), bias=True)) + + def interpolate_support_dims(self, hypercorr, spatial_size=None): + bsz, ch, ha, wa, hb, wb = hypercorr.size() + hypercorr = hypercorr.permute(0, 4, 5, 1, 2, 3).contiguous().view(bsz * hb * wb, ch, ha, wa) + hypercorr = F.interpolate(hypercorr, spatial_size, mode='bilinear', align_corners=True) + o_hb, o_wb = spatial_size + hypercorr = hypercorr.view(bsz, hb, wb, ch, o_hb, o_wb).permute(0, 3, 4, 5, 1, 2).contiguous() + return hypercorr + + def forward(self, hypercorr_pyramid): + + # Encode hypercorrelations from each layer (Squeezing building blocks) + hypercorr_sqz4 = self.encoder_layer4(hypercorr_pyramid[0]) + hypercorr_sqz3 = self.encoder_layer3(hypercorr_pyramid[1]) + hypercorr_sqz2 = self.encoder_layer2(hypercorr_pyramid[2]) + + # Propagate encoded 4D-tensor (Mixing building blocks) + hypercorr_sqz4 = self.interpolate_support_dims(hypercorr_sqz4, hypercorr_sqz3.size()[-4:-2]) + hypercorr_mix43 = hypercorr_sqz4 + hypercorr_sqz3 + hypercorr_mix43 = self.encoder_layer4to3(hypercorr_mix43) + + hypercorr_mix43 = self.interpolate_support_dims(hypercorr_mix43, hypercorr_sqz2.size()[-4:-2]) + hypercorr_mix432 = hypercorr_mix43 + hypercorr_sqz2 + hypercorr_mix432 = self.encoder_layer3to2(hypercorr_mix432) + + bsz, ch, ha, wa, hb, wb = hypercorr_mix432.size() + hypercorr_encoded = hypercorr_mix432.view(bsz, ch, ha, wa, -1).mean(dim=-1) + + # Decode the encoded 4D-tensor + hypercorr_decoded = self.decoder1(hypercorr_encoded) + upsample_size = (hypercorr_decoded.size(-1) * 2,) * 2 + hypercorr_decoded = F.interpolate(hypercorr_decoded, upsample_size, mode='bilinear', align_corners=True) + logit_mask = self.decoder2(hypercorr_decoded) + + return logit_mask diff --git a/submodules/lang_seg/fewshot_data/sbatch_run.sh b/submodules/lang_seg/fewshot_data/sbatch_run.sh new file mode 100644 index 0000000000000000000000000000000000000000..23bae49cd068e455aec648e9ff893a9bf56bbddc --- /dev/null +++ b/submodules/lang_seg/fewshot_data/sbatch_run.sh @@ -0,0 +1,81 @@ +#!/bin/bash +DATE=`date -d now` +EXP=hsnet +NGPU=4 +partition=g24 +JOB=fewshot_${EXP} +SAVE_ROOT="save/${EXP}" +SCRIPT_ROOT="sweep_scripts/${EXP}" +mkdir -p $SCRIPT_ROOT +NCPU=$((NGPU * 10)) +qos=normal # high normal low + +function print_append { + echo $@ >> $SCRIPT +} + +function slurm_append { + echo $@ >> $SLURM +} + +function print_setup { + SAVE="${SAVE_ROOT}/${JOB}" + SCRIPT="${SCRIPT_ROOT}/${JOB}.sh" + SLURM="${SCRIPT_ROOT}/${JOB}.slrm" + mkdir -p $SAVE + echo `date -d now` $SAVE >> 'submitted.txt' + echo "#!/bin/bash" > $SLURM + slurm_append "#SBATCH --job-name=job1111_${JOB}" + slurm_append "#SBATCH --output=${SAVE}/stdout.txt" + slurm_append "#SBATCH --error=${SAVE}/stderr.txt" + slurm_append "#SBATCH --open-mode=append" + slurm_append "#SBATCH --signal=B:USR1@120" + + slurm_append "#SBATCH -p ${partition}" + slurm_append "#SBATCH --gres=gpu:${NGPU}" + slurm_append "#SBATCH -c ${NCPU}" + slurm_append "#SBATCH -t 02-00" + # slurm_append "#SBATCH -t 01-00" + # slurm_append "#SBATCH -t 00-06" + slurm_append "#SBATCH --qos=${qos}" + slurm_append "srun sh ${SCRIPT}" + + echo "#!/bin/bash" > $SCRIPT + print_append "trap_handler () {" + print_append "echo \"Caught signal: \" \$1" + print_append "# SIGTERM must be bypassed" + print_append "if [ "$1" = "TERM" ]; then" + print_append "echo \"bypass sigterm\"" + print_append "else" + print_append "# Submit a new job to the queue" + print_append "echo \"Requeuing \" \$SLURM_JOB_ID" + print_append "scontrol requeue \$SLURM_JOB_ID" + print_append "fi" + print_append "}" + print_append "trap 'trap_handler USR1' USR1" + print_append "trap 'trap_handler TERM' TERM" + + print_append "{" + print_append "source activate pytorch" + print_append "conda activate pytorch" + print_append "export PATH=/home/boyili/programfiles/anaconda3/envs/pytorch/bin:/home/boyili/programfiles/anaconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin" + print_append "which python" + print_append "echo \$PATH" + print_append "export NCCL_DEBUG=INFO" + print_append "export PYTHONFAULTHANDLER=1" + + echo $JOB +} + +function print_after { + print_append "} & " + print_append "wait \$!" + print_append "sleep 610 &" + print_append "wait \$!" +} + +print_setup +print_append stdbuf -o0 -e0 \ + python train.py --log 'log_pascal' +print_after +sbatch $SLURM diff --git a/submodules/lang_seg/fewshot_data/test.py b/submodules/lang_seg/fewshot_data/test.py new file mode 100644 index 0000000000000000000000000000000000000000..d16d9931e7fb804ab7c70ad6cc53db936484b76b --- /dev/null +++ b/submodules/lang_seg/fewshot_data/test.py @@ -0,0 +1,94 @@ +r""" Hypercorrelation Squeeze testing code """ +import argparse + +import torch.nn.functional as F +import torch.nn as nn +import torch + +from fewshot_data.model.hsnet import HypercorrSqueezeNetwork +from fewshot_data.common.logger import Logger, AverageMeter +from fewshot_data.common.vis import Visualizer +from fewshot_data.common.evaluation import Evaluator +from fewshot_data.common import utils +from fewshot_data.data.dataset import FSSDataset + + +def test(model, dataloader, nshot): + r""" Test HSNet """ + + # Freeze randomness during testing for reproducibility + utils.fix_randseed(0) + average_meter = AverageMeter(dataloader.dataset) + + for idx, batch in enumerate(dataloader): + + # 1. Hypercorrelation Squeeze Networks forward pass + batch = utils.to_cuda(batch) + pred_mask = model.module.predict_mask_nshot(batch, nshot=nshot) + + assert pred_mask.size() == batch['query_mask'].size() + + # 2. Evaluate prediction + area_inter, area_union = Evaluator.classify_prediction(pred_mask.clone(), batch) + average_meter.update(area_inter, area_union, batch['class_id'], loss=None) + average_meter.write_process(idx, len(dataloader), epoch=-1, write_batch_idx=1) + + # Visualize predictions + if Visualizer.visualize: + Visualizer.visualize_prediction_batch(batch['support_imgs'], batch['support_masks'], + batch['query_img'], batch['query_mask'], + pred_mask, batch['class_id'], idx, + area_inter[1].float() / area_union[1].float()) + # Write evaluation results + average_meter.write_result('Test', 0) + miou, fb_iou = average_meter.compute_iou() + + return miou, fb_iou + + +if __name__ == '__main__': + + # Arguments parsing + parser = argparse.ArgumentParser(description='Hypercorrelation Squeeze Pytorch Implementation') + parser.add_argument('--datapath', type=str, default='fewshot_data/Datasets_HSN') + parser.add_argument('--benchmark', type=str, default='pascal', choices=['pascal', 'coco', 'fss']) + parser.add_argument('--logpath', type=str, default='') + parser.add_argument('--bsz', type=int, default=1) + parser.add_argument('--nworker', type=int, default=0) + parser.add_argument('--load', type=str, default='') + parser.add_argument('--fold', type=int, default=0, choices=[0, 1, 2, 3]) + parser.add_argument('--nshot', type=int, default=1) + parser.add_argument('--backbone', type=str, default='resnet101', choices=['vgg16', 'resnet50', 'resnet101']) + parser.add_argument('--visualize', action='store_true') + parser.add_argument('--use_original_imgsize', action='store_true') + args = parser.parse_args() + Logger.initialize(args, training=False) + + # Model initialization + model = HypercorrSqueezeNetwork(args.backbone, args.use_original_imgsize) + model.eval() + Logger.log_params(model) + + # Device setup + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + Logger.info('# available GPUs: %d' % torch.cuda.device_count()) + model = nn.DataParallel(model) + model.to(device) + + # Load trained model + if args.load == '': raise Exception('Pretrained model not specified.') + model.load_state_dict(torch.load(args.load)) + + # Helper classes (for testing) initialization + Evaluator.initialize() + Visualizer.initialize(args.visualize) + + # Dataset initialization + FSSDataset.initialize(img_size=400, datapath=args.datapath, use_original_imgsize=args.use_original_imgsize) + dataloader_test = FSSDataset.build_dataloader(args.benchmark, args.bsz, args.nworker, args.fold, 'test', args.nshot) + + # Test HSNet + with torch.no_grad(): + test_miou, test_fb_iou = test(model, dataloader_test, args.nshot) + Logger.info('Fold %d mIoU: %5.2f \t FB-IoU: %5.2f' % (args.fold, test_miou.item(), test_fb_iou.item())) + Logger.info('==================== Finished Testing ====================') diff --git a/submodules/lang_seg/fewshot_data/train.py b/submodules/lang_seg/fewshot_data/train.py new file mode 100644 index 0000000000000000000000000000000000000000..aca972e27c3259f2000755250a6e6db8032aed74 --- /dev/null +++ b/submodules/lang_seg/fewshot_data/train.py @@ -0,0 +1,103 @@ +r""" Hypercorrelation Squeeze training (validation) code """ +import argparse + +import torch.optim as optim +import torch.nn as nn +import torch + +from fewshot_data.model.hsnet import HypercorrSqueezeNetwork +from fewshot_data.common.logger import Logger, AverageMeter +from fewshot_data.common.evaluation import Evaluator +from fewshot_data.common import utils +from fewshot_data.data.dataset import FSSDataset + + +def train(epoch, model, dataloader, optimizer, training): + r""" Train HSNet """ + + # Force randomness during training / freeze randomness during testing + utils.fix_randseed(None) if training else utils.fix_randseed(0) + model.module.train_mode() if training else model.module.eval() + average_meter = AverageMeter(dataloader.dataset) + + for idx, batch in enumerate(dataloader): + # 1. Hypercorrelation Squeeze Networks forward pass + batch = utils.to_cuda(batch) + logit_mask = model(batch['query_img'], batch['support_imgs'].squeeze(1), batch['support_masks'].squeeze(1)) + pred_mask = logit_mask.argmax(dim=1) + + # 2. Compute loss & update model parameters + loss = model.module.compute_objective(logit_mask, batch['query_mask']) + if training: + optimizer.zero_grad() + loss.backward() + optimizer.step() + + # 3. Evaluate prediction + area_inter, area_union = Evaluator.classify_prediction(pred_mask, batch) + average_meter.update(area_inter, area_union, batch['class_id'], loss.detach().clone()) + average_meter.write_process(idx, len(dataloader), epoch, write_batch_idx=50) + + # Write evaluation results + average_meter.write_result('Training' if training else 'Validation', epoch) + avg_loss = utils.mean(average_meter.loss_buf) + miou, fb_iou = average_meter.compute_iou() + + return avg_loss, miou, fb_iou + + +if __name__ == '__main__': + + # Arguments parsing + parser = argparse.ArgumentParser(description='Hypercorrelation Squeeze Pytorch Implementation') + parser.add_argument('--datapath', type=str, default='fewshot_data/Datasets_HSN') + parser.add_argument('--benchmark', type=str, default='pascal', choices=['pascal', 'coco', 'fss']) + parser.add_argument('--logpath', type=str, default='') + parser.add_argument('--bsz', type=int, default=20) + parser.add_argument('--lr', type=float, default=1e-3) + parser.add_argument('--niter', type=int, default=2000) + parser.add_argument('--nworker', type=int, default=8) + parser.add_argument('--fold', type=int, default=0, choices=[0, 1, 2, 3]) + parser.add_argument('--backbone', type=str, default='resnet101', choices=['vgg16', 'resnet50', 'resnet101']) + args = parser.parse_args() + Logger.initialize(args, training=True) + + # Model initialization + model = HypercorrSqueezeNetwork(args.backbone, False) + Logger.log_params(model) + + # Device setup + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + Logger.info('# available GPUs: %d' % torch.cuda.device_count()) + model = nn.DataParallel(model) + model.to(device) + + # Helper classes (for training) initialization + optimizer = optim.Adam([{"params": model.parameters(), "lr": args.lr}]) + Evaluator.initialize() + + # Dataset initialization + FSSDataset.initialize(img_size=400, datapath=args.datapath, use_original_imgsize=False) + dataloader_trn = FSSDataset.build_dataloader(args.benchmark, args.bsz, args.nworker, args.fold, 'trn') + dataloader_val = FSSDataset.build_dataloader(args.benchmark, args.bsz, args.nworker, args.fold, 'val') + + # Train HSNet + best_val_miou = float('-inf') + best_val_loss = float('inf') + for epoch in range(args.niter): + + trn_loss, trn_miou, trn_fb_iou = train(epoch, model, dataloader_trn, optimizer, training=True) + with torch.no_grad(): + val_loss, val_miou, val_fb_iou = train(epoch, model, dataloader_val, optimizer, training=False) + + # Save the best model + if val_miou > best_val_miou: + best_val_miou = val_miou + Logger.save_model_miou(model, epoch, val_miou) + + Logger.tbd_writer.add_scalars('fewshot_data/data/loss', {'trn_loss': trn_loss, 'val_loss': val_loss}, epoch) + Logger.tbd_writer.add_scalars('fewshot_data/data/miou', {'trn_miou': trn_miou, 'val_miou': val_miou}, epoch) + Logger.tbd_writer.add_scalars('fewshot_data/data/fb_iou', {'trn_fb_iou': trn_fb_iou, 'val_fb_iou': val_fb_iou}, epoch) + Logger.tbd_writer.flush() + Logger.tbd_writer.close() + Logger.info('==================== Finished Training ====================') diff --git a/submodules/lang_seg/inputs/cat1.jpeg b/submodules/lang_seg/inputs/cat1.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..ee1cc7c25d85eb656aaf30746a05d6e715c9066e Binary files /dev/null and b/submodules/lang_seg/inputs/cat1.jpeg differ diff --git a/submodules/lang_seg/label_files/ade20k_objectInfo150.txt b/submodules/lang_seg/label_files/ade20k_objectInfo150.txt new file mode 100644 index 0000000000000000000000000000000000000000..333653c5e171cbde5d462086b1c9e5c8d21a97c6 --- /dev/null +++ b/submodules/lang_seg/label_files/ade20k_objectInfo150.txt @@ -0,0 +1,151 @@ +Idx,Ratio,Train,Val,Stuff,Name +1,0.1576,11664,1172,1,wall +2,0.1072,6046,612,1,building;edifice +3,0.0878,8265,796,1,sky +4,0.0621,9336,917,1,floor;flooring +5,0.0480,6678,641,0,tree +6,0.0450,6604,643,1,ceiling +7,0.0398,4023,408,1,road;route +8,0.0231,1906,199,0,bed +9,0.0198,4688,460,0,windowpane;window +10,0.0183,2423,225,1,grass +11,0.0181,2874,294,0,cabinet +12,0.0166,3068,310,1,sidewalk;pavement +13,0.0160,5075,526,0,person;individual;someone;somebody;mortal;soul +14,0.0151,1804,190,1,earth;ground +15,0.0118,6666,796,0,door;double;door +16,0.0110,4269,411,0,table +17,0.0109,1691,160,1,mountain;mount +18,0.0104,3999,441,0,plant;flora;plant;life +19,0.0104,2149,217,0,curtain;drape;drapery;mantle;pall +20,0.0103,3261,318,0,chair +21,0.0098,3164,306,0,car;auto;automobile;machine;motorcar +22,0.0074,709,75,1,water +23,0.0067,3296,315,0,painting;picture +24,0.0065,1191,106,0,sofa;couch;lounge +25,0.0061,1516,162,0,shelf +26,0.0060,667,69,1,house +27,0.0053,651,57,1,sea +28,0.0052,1847,224,0,mirror +29,0.0046,1158,128,1,rug;carpet;carpeting +30,0.0044,480,44,1,field +31,0.0044,1172,98,0,armchair +32,0.0044,1292,184,0,seat +33,0.0033,1386,138,0,fence;fencing +34,0.0031,698,61,0,desk +35,0.0030,781,73,0,rock;stone +36,0.0027,380,43,0,wardrobe;closet;press +37,0.0026,3089,302,0,lamp +38,0.0024,404,37,0,bathtub;bathing;tub;bath;tub +39,0.0024,804,99,0,railing;rail +40,0.0023,1453,153,0,cushion +41,0.0023,411,37,0,base;pedestal;stand +42,0.0022,1440,162,0,box +43,0.0022,800,77,0,column;pillar +44,0.0020,2650,298,0,signboard;sign +45,0.0019,549,46,0,chest;of;drawers;chest;bureau;dresser +46,0.0019,367,36,0,counter +47,0.0018,311,30,1,sand +48,0.0018,1181,122,0,sink +49,0.0018,287,23,1,skyscraper +50,0.0018,468,38,0,fireplace;hearth;open;fireplace +51,0.0018,402,43,0,refrigerator;icebox +52,0.0018,130,12,1,grandstand;covered;stand +53,0.0018,561,64,1,path +54,0.0017,880,102,0,stairs;steps +55,0.0017,86,12,1,runway +56,0.0017,172,11,0,case;display;case;showcase;vitrine +57,0.0017,198,18,0,pool;table;billiard;table;snooker;table +58,0.0017,930,109,0,pillow +59,0.0015,139,18,0,screen;door;screen +60,0.0015,564,52,1,stairway;staircase +61,0.0015,320,26,1,river +62,0.0015,261,29,1,bridge;span +63,0.0014,275,22,0,bookcase +64,0.0014,335,60,0,blind;screen +65,0.0014,792,75,0,coffee;table;cocktail;table +66,0.0014,395,49,0,toilet;can;commode;crapper;pot;potty;stool;throne +67,0.0014,1309,138,0,flower +68,0.0013,1112,113,0,book +69,0.0013,266,27,1,hill +70,0.0013,659,66,0,bench +71,0.0012,331,31,0,countertop +72,0.0012,531,56,0,stove;kitchen;stove;range;kitchen;range;cooking;stove +73,0.0012,369,36,0,palm;palm;tree +74,0.0012,144,9,0,kitchen;island +75,0.0011,265,29,0,computer;computing;machine;computing;device;data;processor;electronic;computer;information;processing;system +76,0.0010,324,33,0,swivel;chair +77,0.0009,304,27,0,boat +78,0.0009,170,20,0,bar +79,0.0009,68,6,0,arcade;machine +80,0.0009,65,8,1,hovel;hut;hutch;shack;shanty +81,0.0009,248,25,0,bus;autobus;coach;charabanc;double-decker;jitney;motorbus;motorcoach;omnibus;passenger;vehicle +82,0.0008,492,49,0,towel +83,0.0008,2510,269,0,light;light;source +84,0.0008,440,39,0,truck;motortruck +85,0.0008,147,18,1,tower +86,0.0008,583,56,0,chandelier;pendant;pendent +87,0.0007,533,61,0,awning;sunshade;sunblind +88,0.0007,1989,239,0,streetlight;street;lamp +89,0.0007,71,5,0,booth;cubicle;stall;kiosk +90,0.0007,618,53,0,television;television;receiver;television;set;tv;tv;set;idiot;box;boob;tube;telly;goggle;box +91,0.0007,135,12,0,airplane;aeroplane;plane +92,0.0007,83,5,1,dirt;track +93,0.0007,178,17,0,apparel;wearing;apparel;dress;clothes +94,0.0006,1003,104,0,pole +95,0.0006,182,12,1,land;ground;soil +96,0.0006,452,50,0,bannister;banister;balustrade;balusters;handrail +97,0.0006,42,6,1,escalator;moving;staircase;moving;stairway +98,0.0006,307,31,0,ottoman;pouf;pouffe;puff;hassock +99,0.0006,965,114,0,bottle +100,0.0006,117,13,0,buffet;counter;sideboard +101,0.0006,354,35,0,poster;posting;placard;notice;bill;card +102,0.0006,108,9,1,stage +103,0.0006,557,55,0,van +104,0.0006,52,4,0,ship +105,0.0005,99,5,0,fountain +106,0.0005,57,4,1,conveyer;belt;conveyor;belt;conveyer;conveyor;transporter +107,0.0005,292,31,0,canopy +108,0.0005,77,9,0,washer;automatic;washer;washing;machine +109,0.0005,340,38,0,plaything;toy +110,0.0005,66,3,1,swimming;pool;swimming;bath;natatorium +111,0.0005,465,49,0,stool +112,0.0005,50,4,0,barrel;cask +113,0.0005,622,75,0,basket;handbasket +114,0.0005,80,9,1,waterfall;falls +115,0.0005,59,3,0,tent;collapsible;shelter +116,0.0005,531,72,0,bag +117,0.0005,282,30,0,minibike;motorbike +118,0.0005,73,7,0,cradle +119,0.0005,435,44,0,oven +120,0.0005,136,25,0,ball +121,0.0005,116,24,0,food;solid;food +122,0.0004,266,31,0,step;stair +123,0.0004,58,12,0,tank;storage;tank +124,0.0004,418,83,0,trade;name;brand;name;brand;marque +125,0.0004,319,43,0,microwave;microwave;oven +126,0.0004,1193,139,0,pot;flowerpot +127,0.0004,97,23,0,animal;animate;being;beast;brute;creature;fauna +128,0.0004,347,36,0,bicycle;bike;wheel;cycle +129,0.0004,52,5,1,lake +130,0.0004,246,22,0,dishwasher;dish;washer;dishwashing;machine +131,0.0004,108,13,0,screen;silver;screen;projection;screen +132,0.0004,201,30,0,blanket;cover +133,0.0004,285,21,0,sculpture +134,0.0004,268,27,0,hood;exhaust;hood +135,0.0003,1020,108,0,sconce +136,0.0003,1282,122,0,vase +137,0.0003,528,65,0,traffic;light;traffic;signal;stoplight +138,0.0003,453,57,0,tray +139,0.0003,671,100,0,ashcan;trash;can;garbage;can;wastebin;ash;bin;ash-bin;ashbin;dustbin;trash;barrel;trash;bin +140,0.0003,397,44,0,fan +141,0.0003,92,8,1,pier;wharf;wharfage;dock +142,0.0003,228,18,0,crt;screen +143,0.0003,570,59,0,plate +144,0.0003,217,22,0,monitor;monitoring;device +145,0.0003,206,19,0,bulletin;board;notice;board +146,0.0003,130,14,0,shower +147,0.0003,178,28,0,radiator +148,0.0002,504,57,0,glass;drinking;glass +149,0.0002,775,96,0,clock +150,0.0002,421,56,0,flag \ No newline at end of file diff --git a/submodules/lang_seg/label_files/fewshot_coco.txt b/submodules/lang_seg/label_files/fewshot_coco.txt new file mode 100644 index 0000000000000000000000000000000000000000..4455a6ef7412d1618f3effc6e145ca9d6dd5986a --- /dev/null +++ b/submodules/lang_seg/label_files/fewshot_coco.txt @@ -0,0 +1,80 @@ +person +bicycle +car +motorbike +aeroplane +bus +train +truck +boat +trafficlight +firehydrant +stopsign +parkingmeter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sportsball +kite +baseballbat +baseballglove +skateboard +surfboard +tennisracket +bottle +wineglass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hotdog +pizza +donut +cake +chair +sofa +pottedplant +bed +diningtable +toilet +tvmonitor +laptop +mouse +remote +keyboard +cellphone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddybear +hairdrier +toothbrush \ No newline at end of file diff --git a/submodules/lang_seg/label_files/fewshot_fss.txt b/submodules/lang_seg/label_files/fewshot_fss.txt new file mode 100644 index 0000000000000000000000000000000000000000..dff6d70465ab049e1aaf8034183149df9da762bd --- /dev/null +++ b/submodules/lang_seg/label_files/fewshot_fss.txt @@ -0,0 +1,1000 @@ +ac_wall +acorn +adhensive_tape +adidas_logo1 +adidas_logo2 +afghan_hound +african_elephant +african_grey +agama +air_strip +aircraft_carrier +airedale +airship +almond +ambulance +american_staffordshire +anemone_fish +angora +apple +armour +ashtray +assult_rifle +aubergine +avocado +baboon +backpack +bagel +balance_weight +bald_eagle +ballpoint +banana +band-aid +banded_gecko +barometer +baseball_bat +baseball_player +basketball +bassoon +bathtub +battery +beacon +beaker +beam_bridge +bear +beaver +bedlington_terrier +bee_house +besom +birdhouse +bison +black_grouse +black_stork +black_swan +blossom_card +boa_constrictor +bolotie +bomb +border_terrier +boston_bull +bottle_cap +bouzouki +box_turtle +bra +bracelet +bradypod +brain_coral +brambling +brasscica +brick +brick_card +brick_tea +briefcase +brown_bear +brush_pen +buckingham_palace +buckler +bullet_train +bushtit +butterfly +cableways +cactus_ball +cairn +camel +can_opener +candle +cannon +canoe +capuchin +car_mirror +car_wheel +carbonara +carousel +carp +carrot +carton +cassette +cauliflower +celery +cello +chainsaw +chalk +cheese_burger +chess_bishop +chest +chickadee_bird +chicken_wings +chicory +chihuahua +children_slide +chinese_date +chopsticks +christmas_stocking +cleaver +cn_tower +cocacola +cocktail_shaker +coffin +coho +collar +comb +computer_mouse +conch +convertible +cornet +cosmetic_brush +cottontail +coucal +cougar +cowboy_hat +coyote +crane +crash_helmet +cream +cristo_redentor +croissant +cucumber +cumquat +dandie_dinmont +dart +dhole +diamond +diaper +digital_watch +dingo +dinosaur +donkey +dough +dragonfly +drake +drumstick +dugong +dumbbell +dutch_oven +earplug +eft_newt +egg_tart +eggnog +egret +egyptian_cat +electric_fan +electronic_toothbrush +eletrical_switch +envelope +esport_chair +espresso +excavator +face_powder +feeder +ferret +fig +file_cabinet +fire_engine +flatworm +flowerpot +flute +flying_disc +fork +forklift +fountain +frog +fur_coat +garbage_can +garbage_truck +garfish +garlic +gas_pump +gazelle +gecko +german_pointer +giant_panda +gliding_lizard +globe +golden_retriever +goldfish +golf_ball +golfcart +goose +gorilla +gourd +grasshopper +great_wall +green_mamba +grey_fox +grey_whale +guacamole +guinea_pig +gypsy_moth +hamster +handshower +hard_disk +hare +hartebeest +harvester +hawthorn +head_cabbage +hen_of_the_woods +hock +hook +hornbill +hornet +housefinch +howler_monkey +hummingbird +hyena +ibex +igloo +indian_cobra +indian_elephant +jacamar +jackfruit +jacko_lantern +jellyfish +jinrikisha +jordan_logo +joystick +kangaroo +kappa_logo +keyboard +killer_whale +kinguin +kitchen_knife +kite +koala +kremlin +ladder +ladle +lady_slipper +ladybug +ladyfinger +lampshade +langur +lark +lawn_mower +leatherback_turtle +leeks +leopard +lesser_panda +lhasa_apso +lifeboat +light_tube +lionfish +litchi +llama +loafer +lobster +lorikeet +lynx +macaque +mailbox +manx +maotai_bottle +maraca +mario +marmot +marshmallow +mcdonald_uncle +measuring_cup +medical_kit +meerkat +melon_seed +memory_stick +microphone +microwave +military_vest +miniskirt +mink +modem +mongoose +monitor +mooli +mooncake +motarboard +motor_scooter +mount_fuji +mountain_tent +mouse +mouthpiece +mud_turtle +mule +muscle_car +mushroom +nail_scissor +neck_brace +necklace +nematode +night_snake +obelisk +ocicat +oil_filter +okra +one-armed_bandit +orange +ostrich +otter +owl +paddle +paint_brush +panpipe +panther +papaya +paper_crane +paper_towel +parallel_bars +park_bench +patas +peacock +pen +pencil_box +pencil_sharpener2 +pepitas +perfume +persian_cat +persimmon +petri_dish +pickup +pill_bottle +pineapple +pingpong_racket +pinwheel +pistachio +plate +poker +pokermon_ball +polar_bear +polecat +police_car +pomelo +pool_table +potato +potted_plant +prairie_chicken +prayer_rug +printer +proboscis +psp +ptarmigan +pubg_lvl3helmet +pufferfish +puma_logo +punching_bag +quad_drone +quill_pen +raccoon +radiator +radio +radio_telescope +raft +rain_barrel +recreational_vehicle +red_bayberry +red_breasted_merganser +red_wolf +redheart +remote_control +rhinoceros +ringlet_butterfly +rock_beauty +rocket +roller_coaster +roller_skate +rosehip +ruddy_turnstone +ruffed_grouse +running_shoe +saluki +sandwich +sandwich_cookies +sarong +scabbard +scorpion +screwdriver +scroll_brush +seal +seatbelt +shakuhachi +shift_gear +shih-tzu +shopping_cart +shotgun +shower_cap +sidewinder +single_log +skua +skull +skunk +sleeping_bag +sloth_bear +snail +snake +snowball +snowmobile +soccer_ball +solar_dish +sombrero +space_heater +space_shuttle +spade +spark_plug +sparrow +spatula +speaker +sponge +spoon +spoonbill +sports_car +spotted_salamander +spring_scroll +squirrel +staffordshire +stapler +starfish +statue_liberty +steak +steam_locomotive +stinkhorn +stole +stool +stop_sign +stove +strainer +streetcar +studio_couch +stupa +submarine +sulphur_crested +sundial +sunglasses +sunscreen +surfboard +swab +swimming_glasses +swimming_trunk +table_lamp +tank +taxi +teapot +tebby_cat +teddy +telescope +tennis_racket +terrapin_turtle +thatch +thimble +throne +tiger +tile_roof +titi_monkey +tobacco_pipe +tofu +toilet_plunger +toilet_tissue +tokyo_tower +tomb +toothbrush +toothpaste +torii +tractor +traffic_light +trailer_truck +trench_coat +tresher +triceratops +trilobite +trimaran +trolleybus +turtle +usb +vending_machine +vestment +victor_icon +vine_snake +violin +wafer +waffle +waffle_iron +wall_clock +wallaby +wallet +walnut +wardrobe +warthog +wash_basin +washer +water_bike +water_polo +water_snake +watermelon +whale +white_shark +wild_boar +window_shade +witch_hat +wok +wombat +wreck +wrench +yorkshire_terrier +yurt +zebra +zucchini +ab_wheel +abacus +ac_ground +african_crocodile +airliner +albatross +apron +arabian_camel +arctic_fox +armadillo +artichoke +baby +badger +balance_beam +balloon +banjo +barbell +baseball +basset +beagle +bee +bee_eater +beer_bottle +beer_glass +bell +bighorn_sheep +bittern +blenheim_spaniel +bluetick +bolete +bowtie +briard +broccoli +bulbul_bird +cabbage +cactus +calculator +camomile +carambola +cardoon +ceiling_fan +cheese +cheetah +cherry +chimpanzee +cigar +cigarette +coconut +coffeepot +colubus +common_newt +consomme +conversion_plug +conveyor +corn +cornmeal +cpu +crab +cradle +crayon +crepe +cricketball +crocodile +croquet_ball +cup +cushion +daisy +digital_clock +dishwasher +dowitcher +drum +eel +english_setter +equestrian_helmet +espresso_maker +fire_balloon +fire_hydrant +fire_screen +flat-coated_retriever +folding_chair +fox_squirrel +french_ball +frying_pan +gas_tank +giant_schnauzer +gibbon +ginger +gyromitra +hair_drier +hami_melon +hammer +hammerhead_shark +handcuff +handkerchief +har_gow +harp +hatchet +hippo +hotdog +ice_lolly +iceberg +icecream +impala +ipod +ironing_board +jay_bird +kazoo +kit_fox +kobe_logo +kwanyin +lacewing +laptop +lemon +lettuce +lion +lipstick +loggerhead_turtle +loguat +louvre_pyramid +lycaenid_butterfly +macaw +mango +mashed_potato +matchstick +mcdonald_sign +microsd +monarch_butterfly +monocycle +mortar +nagoya_castle +narcissus +nike_logo +olive +orang +oscilloscope +ox +panda +parachute +parking_meter +partridge +pay_phone +peanut +pear +piano_keyboard +pickelhaube +pig +pillow +pinecone +pingpong_ball +plaice +platypus +pomegranate +power_drill +pretzel +projector +pubg_airdrop +pubg_lvl3backpack +pumpkin +rabbit +raven +razor +red_fox +redshank +refrigerator +relay_stick +revolver +rock_snake +rose +saltshaker +sandal +sandbar +saxophone +scarerow +school_bus +schooner +scissors +sea_cucumber +sea_urchin +sewing_machine +shovel +shuriken +siamese_cat +skateboard +ski_mask +smoothing_iron +sniper_rifle +soap_dispenser +sock +soup_bowl +spider +spider_monkey +squirrel_monkey +stopwatch +stretcher +strongbox +sturgeon +suitcase +sungnyemun +sweatshirt +swim_ring +syringe +tape_player +taro +television +thor's_hammer +three-toed_sloth +thrush +tiger_cat +tiger_shark +timber_wolf +toilet_brush +toilet_seat +tomato +totem_pole +toucan +tow_truck +tray +triumphal_arch +tulip +turnstile +typewriter +umbrella +upright_piano +vacuum +vase +volleyball +vulture +water_ouzel +water_tower +weasel +whiptail +whistle +white_stork +windsor_tie +wine_bottle +wolf +woodpecker +yawl +yoga_pad +yonex_icon +abe's_flyingfish +accordion +american_alligator +american_chamelon +andean_condor +anise +apple_icon +arch_bridge +arrow +astronaut +australian_terrier +bamboo_dragonfly +bamboo_slip +banana_boat +barber_shaver +bat +bath_ball +beet_root +bell_pepper +big_ben +black_bear +bloodhound +boxing_gloves +breast_pump +broom +bucket +bulb +burj_al +bus +bustard +cabbage_butterfly +cablestayed_bridge +cantilever_bridge +canton_tower +captain_america_shield +carriage +cathedrale_paris +cd +chalk_brush +chandelier +charge_battery +chess_king +chess_knight +chess_queen +chicken +chicken_leg +chiffon_cake +chinese_knot +church +cicada +clam +clearwing_flyingfish +cloud +coffee_mug +coin +combination_lock +condor +cricket +crt_screen +cuckoo +curlew +dart_target +delta_wing +diver +doormat +doublebus +doughnut +downy_pitch +drilling_platform +dwarf_beans +eagle +earphone1 +earphone2 +echidna +egg +electronic_stove +english_foxhound +f1_racing +fan +feather_clothes +fennel_bulb +ferrari911 +fish +fish_eagle +flamingo +fly +flying_frog +flying_geckos +flying_snakes +flying_squirrel +fox +french_fries +ganeva_chair +glider_flyingfish +goblet +golden_plover +goldfinch +groenendael +guitar +gym_ball +haddock +hair_razor +hang_glider +harmonica +hawk +helicopter +hotel_slipper +hover_board +iguana +indri +ipad +iphone +iron_man +jet_aircraft +kart +key +knife +kunai +lapwing +leaf_egg +leaf_fan +leafhopper +leather_shoes +leggings +lemur_catta +letter_opener +little_blue_heron +lotus +magpie_bird +manatee +marimba +may_bug +meatloaf +microscope +minicooper +missile +mite_predator +mitten +moist_proof_pad +monkey +moon +motorbike +net_surface_shoes +nintendo_3ds +nintendo_gba +nintendo_sp +nintendo_switch +nintendo_wiiu +ocarina +oiltank_car +onion +oriole +osprey +oyster +paper_plane +parthenon +pencil_sharpener1 +peregine_falcon +pheasant +phonograph +photocopier +pidan +pizza +plastic_bag +poached_egg +polo_shirt +porcupine +potato_chips +pspgo +pteropus +pumpkin_pie +pyramid +pyramid_cube +pyraminx +quail +quail_egg +rally_car +reel +reflex_camera +rice_cooker +rocking_chair +rubber_eraser +rubick_cube +rugby_ball +ruler +samarra_mosque +santa_sledge +screw +seagull +sealion +shower_curtain +shumai +siamang +sled +snow_leopard +snowman +snowplow +soap +soymilk_machine +speedboat +spiderman +spinach +stealth_aircraft +steering_wheel +stingray +stone_lion +stonechat +stork +strawberry +sulphur_butterfly +sushi +swan +sydney_opera_house +taj_mahal +tiltrotor +toaster +tower_pisa +transport_helicopter +tredmill +truss_bridge +tunnel +twin_tower +vacuum_cup +villa_savoye +vinyl +wagtail +wandering_albatross +warehouse_tray +warplane +wasp +water_buffalo +water_heater +wheelchair +whippet +white_wolf +wig +windmill +window_screen +wooden_boat +wooden_spoon \ No newline at end of file diff --git a/submodules/lang_seg/label_files/fewshot_pascal.txt b/submodules/lang_seg/label_files/fewshot_pascal.txt new file mode 100644 index 0000000000000000000000000000000000000000..1168c39990e4604bb76326833eb7814ed275fcec --- /dev/null +++ b/submodules/lang_seg/label_files/fewshot_pascal.txt @@ -0,0 +1,20 @@ +aeroplane +bicycle +bird +boat +bottle +bus +car +cat +chair +cow +diningtable +dog +horse +motorbike +person +pottedplant +sheep +sofa +train +tvmonitor \ No newline at end of file diff --git a/submodules/lang_seg/lseg_app.py b/submodules/lang_seg/lseg_app.py new file mode 100644 index 0000000000000000000000000000000000000000..f146205064ac2f96fb6daf5a516848150850988a --- /dev/null +++ b/submodules/lang_seg/lseg_app.py @@ -0,0 +1,386 @@ +from collections import namedtuple +import altair as alt +import math +import pandas as pd +import streamlit as st +st.set_page_config(layout="wide") + +from PIL import Image + +import os +import torch + +import os +import argparse +import numpy as np +from tqdm import tqdm +from collections import OrderedDict + +import torch +import torch.nn.functional as F +from torch.utils import data +import torchvision.transforms as transform +from torch.nn.parallel.scatter_gather import gather + +from additional_utils.models import LSeg_MultiEvalModule +from modules.lseg_module import LSegModule + +import cv2 +import math +import types +import functools +import torchvision.transforms as torch_transforms +import copy +import itertools +from PIL import Image +import matplotlib.pyplot as plt +import clip +from encoding.models.sseg import BaseNet +import matplotlib as mpl +import matplotlib.colors as mplc +import matplotlib.figure as mplfigure +import matplotlib.patches as mpatches +from matplotlib.backends.backend_agg import FigureCanvasAgg +from data import get_dataset +import torchvision.transforms as transforms + + +def get_new_pallete(num_cls): + n = num_cls + pallete = [0]*(n*3) + for j in range(0,n): + lab = j + pallete[j*3+0] = 0 + pallete[j*3+1] = 0 + pallete[j*3+2] = 0 + i = 0 + while (lab > 0): + pallete[j*3+0] |= (((lab >> 0) & 1) << (7-i)) + pallete[j*3+1] |= (((lab >> 1) & 1) << (7-i)) + pallete[j*3+2] |= (((lab >> 2) & 1) << (7-i)) + i = i + 1 + lab >>= 3 + return pallete + +def get_new_mask_pallete(npimg, new_palette, out_label_flag=False, labels=None): + """Get image color pallete for visualizing masks""" + # put colormap + out_img = Image.fromarray(npimg.squeeze().astype('uint8')) + out_img.putpalette(new_palette) + + if out_label_flag: + assert labels is not None + u_index = np.unique(npimg) + patches = [] + for i, index in enumerate(u_index): + label = labels[index] + cur_color = [new_palette[index * 3] / 255.0, new_palette[index * 3 + 1] / 255.0, new_palette[index * 3 + 2] / 255.0] + red_patch = mpatches.Patch(color=cur_color, label=label) + patches.append(red_patch) + return out_img, patches + +@st.cache(allow_output_mutation=True) +def load_model(): + class Options: + def __init__(self): + parser = argparse.ArgumentParser(description="PyTorch Segmentation") + # model and dataset + parser.add_argument( + "--model", type=str, default="encnet", help="model name (default: encnet)" + ) + parser.add_argument( + "--backbone", + type=str, + default="clip_vitl16_384", + help="backbone name (default: resnet50)", + ) + parser.add_argument( + "--dataset", + type=str, + default="ade20k", + help="dataset name (default: pascal12)", + ) + parser.add_argument( + "--workers", type=int, default=16, metavar="N", help="dataloader threads" + ) + parser.add_argument( + "--base-size", type=int, default=520, help="base image size" + ) + parser.add_argument( + "--crop-size", type=int, default=480, help="crop image size" + ) + parser.add_argument( + "--train-split", + type=str, + default="train", + help="dataset train split (default: train)", + ) + parser.add_argument( + "--aux", action="store_true", default=False, help="Auxilary Loss" + ) + parser.add_argument( + "--se-loss", + action="store_true", + default=False, + help="Semantic Encoding Loss SE-loss", + ) + parser.add_argument( + "--se-weight", type=float, default=0.2, help="SE-loss weight (default: 0.2)" + ) + parser.add_argument( + "--batch-size", + type=int, + default=16, + metavar="N", + help="input batch size for \ + training (default: auto)", + ) + parser.add_argument( + "--test-batch-size", + type=int, + default=16, + metavar="N", + help="input batch size for \ + testing (default: same as batch size)", + ) + # cuda, seed and logging + parser.add_argument( + "--no-cuda", + action="store_true", + default=False, + help="disables CUDA training", + ) + parser.add_argument( + "--seed", type=int, default=1, metavar="S", help="random seed (default: 1)" + ) + # checking point + parser.add_argument( + "--weights", type=str, default='', help="checkpoint to test" + ) + # evaluation option + parser.add_argument( + "--eval", action="store_true", default=False, help="evaluating mIoU" + ) + parser.add_argument( + "--export", + type=str, + default=None, + help="put the path to resuming file if needed", + ) + parser.add_argument( + "--acc-bn", + action="store_true", + default=False, + help="Re-accumulate BN statistics", + ) + parser.add_argument( + "--test-val", + action="store_true", + default=False, + help="generate masks on val set", + ) + parser.add_argument( + "--no-val", + action="store_true", + default=False, + help="skip validation during training", + ) + + parser.add_argument( + "--module", + default='lseg', + help="select model definition", + ) + + # test option + parser.add_argument( + "--data-path", type=str, default='../datasets/', help="path to test image folder" + ) + + parser.add_argument( + "--no-scaleinv", + dest="scale_inv", + default=True, + action="store_false", + help="turn off scaleinv layers", + ) + + parser.add_argument( + "--widehead", default=False, action="store_true", help="wider output head" + ) + + parser.add_argument( + "--widehead_hr", + default=False, + action="store_true", + help="wider output head", + ) + parser.add_argument( + "--ignore_index", + type=int, + default=-1, + help="numeric value of ignore label in gt", + ) + + parser.add_argument( + "--label_src", + type=str, + default="default", + help="how to get the labels", + ) + + parser.add_argument( + "--arch_option", + type=int, + default=0, + help="which kind of architecture to be used", + ) + + parser.add_argument( + "--block_depth", + type=int, + default=0, + help="how many blocks should be used", + ) + + parser.add_argument( + "--activation", + choices=['lrelu', 'tanh'], + default="lrelu", + help="use which activation to activate the block", + ) + + self.parser = parser + + def parse(self): + args = self.parser.parse_args(args=[]) + args.cuda = not args.no_cuda and torch.cuda.is_available() + print(args) + return args + + args = Options().parse() + + torch.manual_seed(args.seed) + args.test_batch_size = 1 + alpha=0.5 + + args.scale_inv = False + args.widehead = True + args.dataset = 'ade20k' + args.backbone = 'clip_vitl16_384' + args.weights = 'checkpoints/demo_e200.ckpt' + args.ignore_index = 255 + + module = LSegModule.load_from_checkpoint( + checkpoint_path=args.weights, + data_path=args.data_path, + dataset=args.dataset, + backbone=args.backbone, + aux=args.aux, + num_features=256, + aux_weight=0, + se_loss=False, + se_weight=0, + base_lr=0, + batch_size=1, + max_epochs=0, + ignore_index=args.ignore_index, + dropout=0.0, + scale_inv=args.scale_inv, + augment=False, + no_batchnorm=False, + widehead=args.widehead, + widehead_hr=args.widehead_hr, + map_locatin="cpu", + arch_option=0, + block_depth=0, + activation='lrelu', + ) + + input_transform = module.val_transform + + # dataloader + loader_kwargs = ( + {"num_workers": args.workers, "pin_memory": True} if args.cuda else {} + ) + + # model + if isinstance(module.net, BaseNet): + model = module.net + else: + model = module + + model = model.eval() + model = model.cpu() + scales = ( + [0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.25] + if args.dataset == "citys" + else [0.5, 0.75, 1.0, 1.25, 1.5, 1.75] + ) + + model.mean = [0.5, 0.5, 0.5] + model.std = [0.5, 0.5, 0.5] + evaluator = LSeg_MultiEvalModule( + model, scales=scales, flip=True + ).cuda() + evaluator.eval() + + transform = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]), + transforms.Resize([360,480]), + ] +) + + return evaluator, transform + +""" +# LSeg Demo +""" +lseg_model, lseg_transform = load_model() +uploaded_file = st.file_uploader("Choose an image...") +input_labels = st.text_input("Input labels", value="dog, grass, other") +st.write("The labels are", input_labels) + +if uploaded_file is not None: + image = Image.open(uploaded_file) + pimage = lseg_transform(np.array(image)).unsqueeze(0) + + labels = [] + for label in input_labels.split(","): + labels.append(label.strip()) + + with torch.no_grad(): + outputs = lseg_model.parallel_forward(pimage, labels) + + predicts = [ + torch.max(output, 1)[1].cpu().numpy() + for output in outputs + ] + + image = pimage[0].permute(1,2,0) + image = image * 0.5 + 0.5 + image = Image.fromarray(np.uint8(255*image)).convert("RGBA") + + pred = predicts[0] + new_palette = get_new_pallete(len(labels)) + mask, patches = get_new_mask_pallete(pred, new_palette, out_label_flag=True, labels=labels) + seg = mask.convert("RGBA") + + fig = plt.figure() + plt.subplot(121) + plt.imshow(image) + plt.axis('off') + + plt.subplot(122) + plt.imshow(seg) + plt.legend(handles=patches, loc='upper right', bbox_to_anchor=(1.3, 1), prop={'size': 5}) + plt.axis('off') + + plt.tight_layout() + + #st.image([image,seg], width=700, caption=["Input image", "Segmentation"]) + st.pyplot(fig) + + diff --git a/submodules/lang_seg/lseg_demo.ipynb b/submodules/lang_seg/lseg_demo.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..e9d2a06641810ae0e273c84107ca9937172dd3f9 --- /dev/null +++ b/submodules/lang_seg/lseg_demo.ipynb @@ -0,0 +1,1479 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "38d0ddcb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "#os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", + "import torch\n", + "torch.cuda.device_count()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "0b6abf5e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Namespace(model='encnet', backbone='clip_vitl16_384', dataset='ade20k', workers=16, base_size=520, crop_size=480, train_split='train', aux=False, se_loss=False, se_weight=0.2, batch_size=16, test_batch_size=16, no_cuda=False, seed=1, weights='', eval=False, export=None, acc_bn=False, test_val=False, no_val=False, module='lseg', data_path='../datasets/', scale_inv=True, widehead=False, widehead_hr=False, ignore_index=-1, label_src='default', arch_option=0, block_depth=0, activation='lrelu', cuda=True)\n", + "** Use norm [0.5, 0.5, 0.5], [0.5, 0.5, 0.5] as the mean and std **\n", + "{'base_size': 520, 'crop_size': 480}\n", + "train\n", + "BaseDataset: base_size 520, crop_size 480\n", + "len(img_paths): 20210\n", + "val\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/rranftl/anaconda3/envs/lseg_releases/lib/python3.9/site-packages/deprecate/deprecation.py:115: LightningDeprecationWarning: The `Accuracy` was deprecated since v1.3.0 in favor of `torchmetrics.classification.accuracy.Accuracy`. It will be removed in v1.5.0.\n", + " stream(template_mgs % msg_args)\n" + ] + } + ], + "source": [ + "import os\n", + "import argparse\n", + "import numpy as np\n", + "from tqdm import tqdm\n", + "from collections import OrderedDict\n", + "\n", + "import torch\n", + "import torch.nn.functional as F\n", + "from torch.utils import data\n", + "import torchvision.transforms as transform\n", + "from torch.nn.parallel.scatter_gather import gather\n", + "\n", + "import encoding.utils as utils\n", + "from encoding.nn import SegmentationLosses, SyncBatchNorm\n", + "from encoding.parallel import DataParallelModel, DataParallelCriterion\n", + "from encoding.datasets import test_batchify_fn \n", + "from encoding.models.sseg import BaseNet\n", + "from additional_utils.models import LSeg_MultiEvalModule\n", + "from modules.lseg_module import LSegModule\n", + "\n", + "import math\n", + "import types\n", + "import functools\n", + "import torchvision.transforms as torch_transforms\n", + "import copy\n", + "import itertools\n", + "from PIL import Image\n", + "import matplotlib.pyplot as plt\n", + "import clip\n", + "import matplotlib as mpl\n", + "import matplotlib.colors as mplc\n", + "import matplotlib.figure as mplfigure\n", + "import matplotlib.patches as mpatches\n", + "from matplotlib.backends.backend_agg import FigureCanvasAgg\n", + "from data import get_dataset\n", + "import torchvision.transforms as transforms\n", + "\n", + "class Options:\n", + " def __init__(self):\n", + " parser = argparse.ArgumentParser(description=\"PyTorch Segmentation\")\n", + " # model and dataset\n", + " parser.add_argument(\n", + " \"--model\", type=str, default=\"encnet\", help=\"model name (default: encnet)\"\n", + " )\n", + " parser.add_argument(\n", + " \"--backbone\",\n", + " type=str,\n", + " default=\"clip_vitl16_384\",\n", + " help=\"backbone name (default: resnet50)\",\n", + " )\n", + " parser.add_argument(\n", + " \"--dataset\",\n", + " type=str,\n", + " default=\"ade20k\",\n", + " help=\"dataset name (default: pascal12)\",\n", + " )\n", + " parser.add_argument(\n", + " \"--workers\", type=int, default=16, metavar=\"N\", help=\"dataloader threads\"\n", + " )\n", + " parser.add_argument(\n", + " \"--base-size\", type=int, default=520, help=\"base image size\"\n", + " )\n", + " parser.add_argument(\n", + " \"--crop-size\", type=int, default=480, help=\"crop image size\"\n", + " )\n", + " parser.add_argument(\n", + " \"--train-split\",\n", + " type=str,\n", + " default=\"train\",\n", + " help=\"dataset train split (default: train)\",\n", + " )\n", + " parser.add_argument(\n", + " \"--aux\", action=\"store_true\", default=False, help=\"Auxilary Loss\"\n", + " )\n", + " parser.add_argument(\n", + " \"--se-loss\",\n", + " action=\"store_true\",\n", + " default=False,\n", + " help=\"Semantic Encoding Loss SE-loss\",\n", + " )\n", + " parser.add_argument(\n", + " \"--se-weight\", type=float, default=0.2, help=\"SE-loss weight (default: 0.2)\"\n", + " )\n", + " parser.add_argument(\n", + " \"--batch-size\",\n", + " type=int,\n", + " default=16,\n", + " metavar=\"N\",\n", + " help=\"input batch size for \\\n", + " training (default: auto)\",\n", + " )\n", + " parser.add_argument(\n", + " \"--test-batch-size\",\n", + " type=int,\n", + " default=16,\n", + " metavar=\"N\",\n", + " help=\"input batch size for \\\n", + " testing (default: same as batch size)\",\n", + " )\n", + " # cuda, seed and logging\n", + " parser.add_argument(\n", + " \"--no-cuda\",\n", + " action=\"store_true\",\n", + " default=False,\n", + " help=\"disables CUDA training\",\n", + " )\n", + " parser.add_argument(\n", + " \"--seed\", type=int, default=1, metavar=\"S\", help=\"random seed (default: 1)\"\n", + " )\n", + " # checking point\n", + " parser.add_argument(\n", + " \"--weights\", type=str, default='', help=\"checkpoint to test\"\n", + " )\n", + " # evaluation option\n", + " parser.add_argument(\n", + " \"--eval\", action=\"store_true\", default=False, help=\"evaluating mIoU\"\n", + " )\n", + " parser.add_argument(\n", + " \"--export\",\n", + " type=str,\n", + " default=None,\n", + " help=\"put the path to resuming file if needed\",\n", + " )\n", + " parser.add_argument(\n", + " \"--acc-bn\",\n", + " action=\"store_true\",\n", + " default=False,\n", + " help=\"Re-accumulate BN statistics\",\n", + " )\n", + " parser.add_argument(\n", + " \"--test-val\",\n", + " action=\"store_true\",\n", + " default=False,\n", + " help=\"generate masks on val set\",\n", + " )\n", + " parser.add_argument(\n", + " \"--no-val\",\n", + " action=\"store_true\",\n", + " default=False,\n", + " help=\"skip validation during training\",\n", + " )\n", + "\n", + " parser.add_argument(\n", + " \"--module\",\n", + " default='lseg',\n", + " help=\"select model definition\",\n", + " )\n", + "\n", + " # test option\n", + " parser.add_argument(\n", + " \"--data-path\", type=str, default='../datasets/', help=\"path to test image folder\"\n", + " )\n", + "\n", + " parser.add_argument(\n", + " \"--no-scaleinv\",\n", + " dest=\"scale_inv\",\n", + " default=True,\n", + " action=\"store_false\",\n", + " help=\"turn off scaleinv layers\",\n", + " )\n", + "\n", + " parser.add_argument(\n", + " \"--widehead\", default=False, action=\"store_true\", help=\"wider output head\"\n", + " )\n", + "\n", + " parser.add_argument(\n", + " \"--widehead_hr\",\n", + " default=False,\n", + " action=\"store_true\",\n", + " help=\"wider output head\",\n", + " )\n", + " parser.add_argument(\n", + " \"--ignore_index\",\n", + " type=int,\n", + " default=-1,\n", + " help=\"numeric value of ignore label in gt\",\n", + " )\n", + " \n", + " parser.add_argument(\n", + " \"--label_src\",\n", + " type=str,\n", + " default=\"default\",\n", + " help=\"how to get the labels\",\n", + " )\n", + " \n", + " parser.add_argument(\n", + " \"--arch_option\",\n", + " type=int,\n", + " default=0,\n", + " help=\"which kind of architecture to be used\",\n", + " )\n", + "\n", + " parser.add_argument(\n", + " \"--block_depth\",\n", + " type=int,\n", + " default=0,\n", + " help=\"how many blocks should be used\",\n", + " )\n", + "\n", + " parser.add_argument(\n", + " \"--activation\",\n", + " choices=['lrelu', 'tanh'],\n", + " default=\"lrelu\",\n", + " help=\"use which activation to activate the block\",\n", + " )\n", + "\n", + " self.parser = parser\n", + "\n", + " def parse(self):\n", + " args = self.parser.parse_args(args=[]) \n", + " args.cuda = not args.no_cuda and torch.cuda.is_available()\n", + " print(args)\n", + " return args\n", + " \n", + "\n", + "def get_new_pallete(num_cls):\n", + " n = num_cls\n", + " pallete = [0]*(n*3)\n", + " for j in range(0,n):\n", + " lab = j\n", + " pallete[j*3+0] = 0\n", + " pallete[j*3+1] = 0\n", + " pallete[j*3+2] = 0\n", + " i = 0\n", + " while (lab > 0):\n", + " pallete[j*3+0] |= (((lab >> 0) & 1) << (7-i))\n", + " pallete[j*3+1] |= (((lab >> 1) & 1) << (7-i))\n", + " pallete[j*3+2] |= (((lab >> 2) & 1) << (7-i))\n", + " i = i + 1\n", + " lab >>= 3\n", + " return pallete\n", + "\n", + "def get_new_mask_pallete(npimg, new_palette, out_label_flag=False, labels=None):\n", + " \"\"\"Get image color pallete for visualizing masks\"\"\"\n", + " # put colormap\n", + " out_img = Image.fromarray(npimg.squeeze().astype('uint8'))\n", + " out_img.putpalette(new_palette)\n", + "\n", + " if out_label_flag:\n", + " assert labels is not None\n", + " u_index = np.unique(npimg)\n", + " patches = []\n", + " for i, index in enumerate(u_index):\n", + " label = labels[index]\n", + " cur_color = [new_palette[index * 3] / 255.0, new_palette[index * 3 + 1] / 255.0, new_palette[index * 3 + 2] / 255.0]\n", + " red_patch = mpatches.Patch(color=cur_color, label=label)\n", + " patches.append(red_patch)\n", + " return out_img, patches\n", + "\n", + "args = Options().parse()\n", + "\n", + "torch.manual_seed(args.seed)\n", + "args.test_batch_size = 1 \n", + "alpha=0.5\n", + " \n", + "args.scale_inv = False\n", + "args.widehead = True\n", + "args.dataset = 'ade20k'\n", + "args.backbone = 'clip_vitl16_384'\n", + "args.weights = 'checkpoints/demo_e200.ckpt'\n", + "args.ignore_index = 255\n", + "\n", + "module = LSegModule.load_from_checkpoint(\n", + " checkpoint_path=args.weights,\n", + " data_path=args.data_path,\n", + " dataset=args.dataset,\n", + " backbone=args.backbone,\n", + " aux=args.aux,\n", + " num_features=256,\n", + " aux_weight=0,\n", + " se_loss=False,\n", + " se_weight=0,\n", + " base_lr=0,\n", + " batch_size=1,\n", + " max_epochs=0,\n", + " ignore_index=args.ignore_index,\n", + " dropout=0.0,\n", + " scale_inv=args.scale_inv,\n", + " augment=False,\n", + " no_batchnorm=False,\n", + " widehead=args.widehead,\n", + " widehead_hr=args.widehead_hr,\n", + " map_locatin=\"cpu\",\n", + " arch_option=0,\n", + " block_depth=0,\n", + " activation='lrelu',\n", + ")\n", + "\n", + "input_transform = module.val_transform\n", + "\n", + "# dataloader\n", + "loader_kwargs = (\n", + " {\"num_workers\": args.workers, \"pin_memory\": True} if args.cuda else {}\n", + ")\n", + "\n", + "# model\n", + "if isinstance(module.net, BaseNet):\n", + " model = module.net\n", + "else:\n", + " model = module\n", + " \n", + "model = model.eval()\n", + "model = model.cpu()\n", + "scales = (\n", + " [0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.25]\n", + " if args.dataset == \"citys\"\n", + " else [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]\n", + ") \n", + "\n", + "model.mean = [0.5, 0.5, 0.5]\n", + "model.std = [0.5, 0.5, 0.5]" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e9c0e0bf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "MultiEvalModule: base_size 520, crop_size 480\n" + ] + }, + { + "data": { + "text/plain": [ + "LSeg_MultiEvalModule(\n", + " (module): LSegModule(\n", + " (train_accuracy): Accuracy()\n", + " (val_accuracy): Accuracy()\n", + " (net): LSegNet(\n", + " (clip_pretrained): CLIP(\n", + " (visual): VisionTransformer(\n", + " (conv1): Conv2d(3, 768, kernel_size=(32, 32), stride=(32, 32), bias=False)\n", + " (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (transformer): Transformer(\n", + " (resblocks): Sequential(\n", + " (0): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (1): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (2): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (3): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (4): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (5): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (6): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (7): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (8): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (9): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (10): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (11): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " )\n", + " )\n", + " (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (transformer): Transformer(\n", + " (resblocks): Sequential(\n", + " (0): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (1): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (2): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (3): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (4): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (5): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (6): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (7): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (8): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (9): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (10): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (11): ResidualAttentionBlock(\n", + " (attn): MultiheadAttention(\n", + " (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n", + " )\n", + " (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): Sequential(\n", + " (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n", + " (gelu): QuickGELU()\n", + " (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n", + " )\n", + " (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " )\n", + " )\n", + " (token_embedding): Embedding(49408, 512)\n", + " (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (pretrained): Module(\n", + " (model): VisionTransformer(\n", + " (patch_embed): PatchEmbed(\n", + " (proj): Conv2d(3, 1024, kernel_size=(16, 16), stride=(16, 16))\n", + " (norm): Identity()\n", + " )\n", + " (pos_drop): Dropout(p=0.0, inplace=False)\n", + " (blocks): Sequential(\n", + " (0): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (1): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (2): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (3): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (4): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (5): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (6): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (7): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (8): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (9): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (10): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (11): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (12): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (13): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (14): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (15): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (16): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (17): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (18): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (19): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (20): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (21): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (22): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " (23): Block(\n", + " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (attn): Attention(\n", + " (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (attn_drop): Dropout(p=0.0, inplace=False)\n", + " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (proj_drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (drop_path): Identity()\n", + " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (mlp): Mlp(\n", + " (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (act): GELU()\n", + " (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n", + " (drop): Dropout(p=0.0, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", + " (pre_logits): Identity()\n", + " (head): Linear(in_features=1024, out_features=1000, bias=True)\n", + " )\n", + " (act_postprocess1): Sequential(\n", + " (0): ProjectReadout(\n", + " (project): Sequential(\n", + " (0): Linear(in_features=2048, out_features=1024, bias=True)\n", + " (1): GELU()\n", + " )\n", + " )\n", + " (1): Transpose()\n", + " (2): Unflatten(dim=2, unflattened_size=torch.Size([24, 24]))\n", + " (3): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n", + " (4): ConvTranspose2d(256, 256, kernel_size=(4, 4), stride=(4, 4))\n", + " )\n", + " (act_postprocess2): Sequential(\n", + " (0): ProjectReadout(\n", + " (project): Sequential(\n", + " (0): Linear(in_features=2048, out_features=1024, bias=True)\n", + " (1): GELU()\n", + " )\n", + " )\n", + " (1): Transpose()\n", + " (2): Unflatten(dim=2, unflattened_size=torch.Size([24, 24]))\n", + " (3): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1))\n", + " (4): ConvTranspose2d(512, 512, kernel_size=(2, 2), stride=(2, 2))\n", + " )\n", + " (act_postprocess3): Sequential(\n", + " (0): ProjectReadout(\n", + " (project): Sequential(\n", + " (0): Linear(in_features=2048, out_features=1024, bias=True)\n", + " (1): GELU()\n", + " )\n", + " )\n", + " (1): Transpose()\n", + " (2): Unflatten(dim=2, unflattened_size=torch.Size([24, 24]))\n", + " (3): Conv2d(1024, 1024, kernel_size=(1, 1), stride=(1, 1))\n", + " )\n", + " (act_postprocess4): Sequential(\n", + " (0): ProjectReadout(\n", + " (project): Sequential(\n", + " (0): Linear(in_features=2048, out_features=1024, bias=True)\n", + " (1): GELU()\n", + " )\n", + " )\n", + " (1): Transpose()\n", + " (2): Unflatten(dim=2, unflattened_size=torch.Size([24, 24]))\n", + " (3): Conv2d(1024, 1024, kernel_size=(1, 1), stride=(1, 1))\n", + " (4): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n", + " )\n", + " )\n", + " (scratch): Module(\n", + " (layer1_rn): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (layer2_rn): Conv2d(512, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (layer3_rn): Conv2d(1024, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (layer4_rn): Conv2d(1024, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (refinenet1): FeatureFusionBlock_custom(\n", + " (out_conv): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))\n", + " (resConfUnit1): ResidualConvUnit_custom(\n", + " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (activation): ReLU()\n", + " (skip_add): FloatFunctional(\n", + " (activation_post_process): Identity()\n", + " )\n", + " )\n", + " (resConfUnit2): ResidualConvUnit_custom(\n", + " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (activation): ReLU()\n", + " (skip_add): FloatFunctional(\n", + " (activation_post_process): Identity()\n", + " )\n", + " )\n", + " (skip_add): FloatFunctional(\n", + " (activation_post_process): Identity()\n", + " )\n", + " )\n", + " (refinenet2): FeatureFusionBlock_custom(\n", + " (out_conv): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))\n", + " (resConfUnit1): ResidualConvUnit_custom(\n", + " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (activation): ReLU()\n", + " (skip_add): FloatFunctional(\n", + " (activation_post_process): Identity()\n", + " )\n", + " )\n", + " (resConfUnit2): ResidualConvUnit_custom(\n", + " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (activation): ReLU()\n", + " (skip_add): FloatFunctional(\n", + " (activation_post_process): Identity()\n", + " )\n", + " )\n", + " (skip_add): FloatFunctional(\n", + " (activation_post_process): Identity()\n", + " )\n", + " )\n", + " (refinenet3): FeatureFusionBlock_custom(\n", + " (out_conv): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))\n", + " (resConfUnit1): ResidualConvUnit_custom(\n", + " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (activation): ReLU()\n", + " (skip_add): FloatFunctional(\n", + " (activation_post_process): Identity()\n", + " )\n", + " )\n", + " (resConfUnit2): ResidualConvUnit_custom(\n", + " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (activation): ReLU()\n", + " (skip_add): FloatFunctional(\n", + " (activation_post_process): Identity()\n", + " )\n", + " )\n", + " (skip_add): FloatFunctional(\n", + " (activation_post_process): Identity()\n", + " )\n", + " )\n", + " (refinenet4): FeatureFusionBlock_custom(\n", + " (out_conv): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))\n", + " (resConfUnit1): ResidualConvUnit_custom(\n", + " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (activation): ReLU()\n", + " (skip_add): FloatFunctional(\n", + " (activation_post_process): Identity()\n", + " )\n", + " )\n", + " (resConfUnit2): ResidualConvUnit_custom(\n", + " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (activation): ReLU()\n", + " (skip_add): FloatFunctional(\n", + " (activation_post_process): Identity()\n", + " )\n", + " )\n", + " (skip_add): FloatFunctional(\n", + " (activation_post_process): Identity()\n", + " )\n", + " )\n", + " (head1): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1))\n", + " (output_conv): Sequential(\n", + " (0): Interpolate()\n", + " )\n", + " )\n", + " )\n", + " (criterion): SegmentationLosses(\n", + " (bceloss): BCELoss()\n", + " )\n", + " )\n", + ")" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "evaluator = LSeg_MultiEvalModule(\n", + " model, scales=scales, flip=True\n", + ").cuda()\n", + "evaluator.eval()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "bc383c25", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAD8CAYAAAARze3ZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAEAAElEQVR4nOz92a9t23beh/1aL8YYs1hr733Ke25JiqQqqhZNUZGsyJGsGPKDYgcQkkCAHgIIech79Jwn/wMBEiFxkJckDgwoji3DiZFIiJMotgpbBWVRpHhJ3voUe+9VzDnHGL331vLQ+phrX5r3ko58oSNgD2CfdVYxqzH6aL21r33f18TMeHu8Pd4eb4+3xw8+wj/vN/D2eHu8Pd4en/fjbaB8e7w93h5vj9/keBso3x5vj7fH2+M3Od4GyrfH2+Pt8fb4TY63gfLt8fZ4e7w9fpPjbaB8e7w93h5vj9/k+JEFShH510TkF0Tkl0TkL/+oXuft8fZ4e7w9ftSH/Ch4lCISgX8C/KvAN4G/BfwPzewf/Tf+Ym+Pt8fb4+3xIz5+VBnlzwK/ZGa/bGYr8H8E/tyP6LXeHm+Pt8fb40d6pB/R834J+MYb338T+CM/6I9DTpbGoX8nT19+fbIrT7+2/v/i//n+P3vje9ueykDMMAxseyrDzN54uP2GL/tfPezXv6S/qFl/rL+GmaHm35v56/U3fX28iIAIgiAi9G+R0L/vzy1v/tt+hr+GmaFqtKaoKaagpv45t3P2fW/YT4CZEUQI4fvfU3/LqIGZYtcz4u9fvu/JxC/G9nh5+pvtOgiC9fOxnZfrubGn15MAQfpzBd54TvoF96/fVwRdL/Cv/4xPz71d/+97wBt//+Y1u77f71sFQgh+TUIQJAoxhuv16iezP4tdHyNBIMjTZxAwNVprfq2q0WrD2vefe3vzPW4fzt58HevXwN74zP799f3I998HT0/hP5Tta/9c/s/Pv4Sn318fbv7eTUG3r83Qppja9TX8cd9/9q5v742zHWIk5kCIgRjC9z1iq3LNjLYq2uzpHH7fR3rjmvVrEQKIPJ2f6/q/niK/Fw3xz+PvBomB88v5UzN7/9e/dfjRBcrf9BCRvwT8JYA4DLz/e3+nBxLxcBL6VwBCDyYxQBRIRgsGUYhB/GdvnAwRQYInywFBVbGmhFbRtiIKQRvBFMwQrfiaNqIpFQFRRBXrazNIjwE9ePTPcL042+urNg8uJqxLYV2VVitNW7+X1J+7v9kYIzEmRAIpJYYhknIkJSHn3P8lhiGTcmLIAylGUkrEENBaWdfC6XTm8bxwulxY5sqyrJgpwxCRKBANxLAAFgyTRgiQgpCHSEx+Lg3QVqkVllWpWkAaIQR/DBFtgpmAGqYRDy2RGBshCkP09yaAWIRmqAq1GGWtlKKUVSkFtPk5iQmGITGMARkgT4kYQYKf61YDaMQ0YgrXC2NbUHlaAyJy3TxUt/vD/IYLEEL0uBv7hgIY6uHJwKpCpb+WYcGYpkQaEuMuMx4Th5uJaTcwjImcIyEEVJWYInE/MAwjeczsjxN5yijGsi7M88zpdOL+s3vuXj3y8XdfcX69wOofpalBE9BEKw01gaagARqgirVKCr4YQwiEEEg5IKKECBIhRCGIgQWQgCDULUAQkRgIyUj7yLCL5F0kT8L+MDAMQh4SZgFrSlOjFqUuxnxS5nNlPTXqxZgfV+qsaFFa077R+52r+JpT6ec9+u9ijOzf2XN875bjix3Hw0QIgVorqoo2f67T6cLLb95RH1u/p/1eMzFMlCDJg2pPNFIwxskYh0iMgRQbASWkhARDQsOs0Rqsq1KbYZZgEPJuz9/83/7Cr/6gePWjCpTfAr7yxvdf7j+7Hmb2V4C/AjAcD29s6lvW1Xcm82DHGzvgNZCKOHgg1ncy6c/t/9kSjSB4JmBCawHT6lkNhvTdSzAChokvJRFDooH6hcF4uimfPoMnkoi/HzNEwvV3MSZy6u+j+N8pyzWz3G7mLY/AFDOlaiNrX6hb6kxAVUArDAFM0f6+tO/yqg1tSq2V1pQQxL96YgOxn7e+8cQU+vsPfpP2bE019gULQs+c+uf3t9uzZzOMillE+jnIQyRlI4phzaiLYVVoVaklUlZjXRq1GK0J1jeLEAwJRgg9ywXEBFEwCWDRz0f/rELETDCqXzuR63l/89r4B/czGKMQQvAsLwHJb2YxMPHgadpoVWlVkJ4taYV5LcilcXlYSPeRx8PCzYsjh+PEtGtMu5FpPzIdJ8abHcebA8M0kgcPsA1jXmbyZYAUaKY0acw6k/eZOivz44KeG0bwNY1g5WlTRfzu8GzWrpVADIKIbzYpi6/doAR8zdMzegRqU/++Vy0B/xpTYshCSgGJhtEIPTK1VrHaaKtR1karPSszvVY5vpZ9Q4WtIvKM1zc8z8ZjENI0MOwy425gmAaGMSPAMCZMPUiua2OdC3nI2OD5YYgQop8Dk0AMEaNv/iqoGbVXSaRIiBEoIE+VQAiCaPD3peZrf2hI1h8a0H5UgfJvAT8lIj+OB8j/AfA/+oF/LSAxYa357uAxA0J4SuWtIRK86O2ZnO8sWyn4VIqG4DeUAYSKoASUmIQxJqwYpVRaqYiJBxJ/Gyggoj1JEYztibagak+FmeC79RvljIh4FiNKTNvJT6hFWl0RE8wCItEXtEQgIkBrionQpKFmYILaG2WgBZIIzQyLSooRMygr1CY9a6vU4juzPzfE4EGCHAhJIPTSW4yUkn+m0MtsC9cyPohhpGsAx3rmdC3rA2YBaEhQhnEk7ys5J8RAi5KI1Aq1wLKu1FWpKx4kzW92EYgpEBJe1vZNDfPr3apgFntGHhAJaK86gyQMc8ihb45vVmg5ecpvmJdl0ctMDQ3SBhEoEj3zadWgdoimGaGBFNAFmKHWSn2E5TyzXhbOzw8cbkaevQu75zt2z3ccb58xHXZMh4EwQIyJpooMQFSqLqxr5rKM7G93hLSyro1xn1juK+fHwnpasTUQW0TNt1gwYvZzExHfXARiUt9kUvSgJ4JEiCKeiRJ8YTfPqNSMEAeSxA4zQAyBnCIeOr0qUlEk9PJ4yIRaMSqlNrQK1gxrQPMb1jNJ9aSmb8YhWQ+awcv6HMiHkZsXz3nnxTOOtxNpjNf7TE0ppRETrOeFcR8BJZheAyMh+HsT6eeGnlBAM2U13/CQQARCUKL0DbJ/jbtIMEMIWEpI+OGh8EcSKM2sisj/FPi/AhH4t83s53/YYyQIYl7qbLXudiP57994fvHMbdtRPXuQN2Ccp8wOPIsK4jtcCsKQRyCxLiv1UqitXbGfiAcCwcsWs75b8oTOqdkVN9syQk9UQk+Gg5cI6pVPTJVBGqsoNMHUL5j0TFiuWMr2Cp6ttVaRINQaEVtBhUBAUiNopNWAmVFro6k+ZbG5YSh5CExDJo2QkmDZs5ImzbPpvhFZABP1N9x8YYfogUrt6f36yaffTNbPjWcM+0NkOlbSzhCpWE0EEVoFFaG1nkmugrUOEYpneDF68A4b7seWNfbr2jcWM3/fDsFdr8b34VQAra+HlD1wWD+39ECJKSEKtmUYAha9rIsjtBZAFenZpK5KjAGLsBageVZyfqxc2h2L7okH4dZuIEfCLhKmQJoSYfBbLJlRau2fWYhDYNgH8uqVtmVFkxFyJh8T51eR06czurS+ihshBkL0bVVMiaGX131dhwDRC2ukZ9QhAmo0/F4KBK8SYsccmyIqaFOCJcT8LwT1za5vQDF4tplyICdYpTlOIa3jyb8OQNxK5Rg8kyOQQySNmWm3YxwnxnFiGEfGKUHw+7aUQgiVapU8Rsbd0DdNemD0JGZ7yaZbuHDoQ9QwC6yl0hr+nhFiFdL2niT554ni2XN4SnZ+0PEjwyjN7D8E/sPf2l/LtUzqeOs14Ihs2FPoTYBwLZfVvGzqr3ctAUTCNesL5pE6BkgSiWIE8QxoPyY0wryaZ5fqjZDtOX2XvCaU17JuO6dm0ktBkKRof89+IwcP6KExxIAGIdRMbSNWvaQL0gMOel0IIUQIeoUNWmvXndqDsiA2kOwJvzPzmynnRIwHdoeJWgvgCyWPvsArFQtKVWg0xypRh7Hw5g0CwUJvvBhazUtgtoaM47AEQaISY2M6BA7PlWFfsahE3VGlUUwoNC5VuCxGXQ1rXuZ7Vdczn9jLxw2O2Bo3yPVa+K32FAR5Y0PxaxKu34cUSSkSYs88rsvMruc7IP36KhaNkBRLhlKRDNk6flocElAxCEoeIlYDzGCqGA0RRSNohIpnYoqyWmWU7JhbKV7F1EqzRtFKCwtpUiJKTAbZCIMSRkFzRCUzU+FsBCKBQIxKDIaoEiQTo2dFUejBs28YW3Zt2zlSP+/XZg09IfAbbi2FeYEhJCyI44k9PeidUEKEFAMpCzYYUgOaAmXxm0aCPKFI5oE2EAgSiCmSYiSPmXGYGHImpkgeBsd4oz82pUQphbnN5DwwTpmAUKtn1OF6H4YrfGZA0+oQWejZiSYatcNEAczPoGwQiwQkQcoRJGC/Cf/nn1sz583jmlH1dP3aPZanlPoJJFbf79SbOT01AbadJWJAbY0YAikEohgpGEkgx9AzloqYN1ZyEoIkaqm9Y9Y7eqrft+hgS156QNetix6oxYixZyo90osEhjEQBiXvBlIODiQvlXWp1NoQC2jrGEvzxWXWMx3omaV6uSRgzJjADiH0Rk2Iwm4cgAx4lhaiLyQJRm2Fag0IVF0IQX0nFsc5LfZ7QQWSYK1hyTNGWvBggGJREDXHwoYAMRKGxHBTScfGcEhE1WvJ3WpERKmteVaK4Z20jrFFvNxOct0QtWeqwfDsIPSFsN0SMXk3Hr8ZQ5Ae4QNmjRgDIUV/7uAba48GvXPbcV9RghR/Zo8yqBUsmDcNMUISLPYNOwhtEmIx6sXPJU2Z9gPHZzumw4QJ1FZY5oUUEtaM9dJoasyXhXI5MS8PXC4XlvmRVk/EpOTBsKDkQdDBKFWJaWVajdgCS2q0WUkWvXKhb6jWCDERTQFveFjrWaT09Snmm7LhsIr5ug5Zicnx061aaMEI2Yipl+TavLmnCirUpfWMU5/Ol/XSmh50xYOxJ36ClV4ZZCGM2Uv4aOjGCIiROA4EFDEhkxARajPG3cpU9xhnmJe+UQc86McrHKfacVc63u6ZCkIkaPRMuvcyYoyk5OcoRm+gWofQftjxuQiUWyFl4qmG2YacPMVBMdBroiFXtogQemfSGxOqoHgZqVVpUUjJAd5rqcKWifl/WmveANGGmDlekxOWjForpp5VqW1NmOvbvuKYZoI2OnbkZVxIxjAl0pgZD4ndIXgJJzCfG+dz9XJ0UVozWlHq0hx+kOBBE+tdXmhUigi5NWpsDJKIwbvmaRBSjL3sD71B4jf8Zb0wlwtWK2q+CAkBU4XgOK8D7ds5FqSZY5TBfJWEQBTvfA9DZpgGCI0P3j2S8mtkB2kHk0Q+vbtjHF8Qdcd893qLamyMhg2qiMkzlyCOKyO6tbn6Oe54tQga/CYQb1X7RhdCz8QB1DPTFK4ZE1YJcfCX107R6lmlNiVJwkpDrWHSkBwgGhYNSdaRY0NpyBChCS00sgVqh1ryIRAPjr2ZGKUoy6VAOyMhUEplXirz+cJ6eUBZqW2hydKx4kbKhuSImtKSYasRzbAbD3BNFKISl+bEZzOHAsOGqjd445xtkFXT5rmzglhgo2gZQlNBJBHHkbyPxEmYxkjOgRQj2tTvxSY9I24sS2W+KMtilLU8VTjCtevvt6ffv55o9Mx8LWh2PDykSE6OC6cYSdEZDobQWiWSyKWQU2QcMjoOBFX/PKoI8cpo2KopPy3+mqZ+T4YIod+3qoZIIsTEMEZSDqSUEfEqouo/n2bOf71D8MiiT6XtBgPC1j7Z/nDDTcK1HITOibpiVzh0pUZDaSJUHGdR8y6pNQVraGu01tDaMHXKTM/sGXIm50iplXUp1/J+y2AV68ETwNMyVSWBZ11jII2wOyamY2baJ6YpkzIsC1wujfOpcnlYWYuyzhUJlXVVrPprhE4DMVVvnnQAu/UGiyCEThcacn7iX0rwz6fqoH/slWfz86SmVFl7oyYiIT5l7VGIUbDacYvUszKRjjFlwhBIKXBzGPnCsx/nxYtnfPf0a6gE1BKv7869wI1IKYh5uywlT3di9pI7dLpO79n5zS/RyyVPCzuc4V1LXw84HGBbUO3d8o45elNee8f7TbhEUQVqQ1uhifhlS0bIwfHZrFhSJCkSII2C5d48qvXKE0tRkRTJxwyDQTSaNpZlQRAul4WyFObHmctpZpnPYIU4GIQFjQULK1UqYQhoPxeSAjkERn9rtAg1dLipKVRx6lXvam+4bcCzxW2j2ZqTtXqGEUVoqjT1gjqkQAyJlDK7Q2Y8ZMacCLGBeAXj1YQQQqJJA3NGAB2eDBJoGE3LNQiDB2iJod8PRq3eTCmlMNjk1KTYg6l5AKRzObU3rfziKkGUFIUW/f2LdHhMBFNFVaml0HoDsrUGGlD1BqamRjLH461DFykNTreLnpUqDnf9sOPzESj7ERxhvnLbtgaNbhV5/950KwN9AVjPvMAxGYI/KKhgWqiK3/yknm347uuE2dZ3IfVMxSDGDSSFPAwM08i0q8znlWVdrnyxK24pybvZop1HaWCRnEfiaAyTsRthnBL7fWKYIrujsJuVcVpIIXA+NQIZbEYp/pkR58cBVj0IK94USDHR2koJAVomqWcksQdyRUEqGjygegPMMzFtSrFCZUFDJUlGLRGC40HSy3FNHcoQx0NjFMbjQMxwGAKH3cB+F9jvBn7sg6/x3vycb37yDW5vDrSH73BHpCwQGIHqzQjxaxZiIAQlpOBlpBN18HaE0JnQHgKCeCAn0NQ79D1hBOs0FE/vrzerZ6/+WDNorVJL8WDYPHh4BxUs47BH8gAT1+hBaOy4YYxEBY2NSKIBOTv/N49CTGBWqG1lno1lLdTFWE4ry2lmPV1o64rFShoraQBigWGlxdZLexhkJI+JtV0YQkVIvgaawUWhCDYbwQJiiraKVcMk+t+K9ewJCOrZVDNac8BKCVQTZBCGKRJGh24EIREZQkRSRmIjm7GuC0iltkYeAnUIDCnAqjSgbSTVjp/bBub3rNau9y0QAkGgqMNbRiBE72I3GgK04vdPa43SKnNbaFqAglEBZ2Io2rvtijTPsqu2Doepb/At0IoiY0CGRJBEitl5y706coqhV2HyRmb6Gx2fi0B5rcw2bC/0stO0f4gNfeilqIgnn2qo9CBpV0ql77K9PDG8tDYBqY2WQKR69toq2qrzxLSBai9XNzWFL+IhZ+JuYpoGzpeBy7ywrP0x9M74FqijE4Pr2liXwv44ITERh8gwZYZdZncYkJi40USeLpidUbnQTBmI5CF2elAPVOYNlFYbtSq1KUsp0MvAZto7ug0LvgC9LGuoQamVUlfm9cxSVpa20ChYbMQIWxDzfoifZYmBOASs9K5wUCT7z4YhYMH46odf4IPxgOSEXAJ/9vf+eZbyKX/n5/82+f7Ar54+5RyNGio1gzWHVyR6ANu63N7RdbhBN4J0hxDykB33xWGVwEYJ6efca3NfGaaohSemhEGrfhO31ZAWexaq28f0zLOpI9/GlWPpuJURxSEUggcoG70DHiUh2c+JZL8+82VhXYxSzsznyuVxZb1foDbEGiEWhqNcgwYqSHBcOspATgPDlIlWKGvrSjIhFYMjUJTVFJ0DoQnaOu3JDLV2LYENpVZvyrlyRmgCDcUkMu125F0i56FDV4paozSnFMWt4ROCU5uaY5UxZdIu0lpltIiuUDtOuXXHPdsPjolKw5I4lp3EewpRWdtKaStzWQgr1BAZhgz4hqbqmaE1uyZNdOod5k1czya3/Bli8nI8tHBtiTshpYF5cy8PLuAIG9vDpPOy61PD7wccn4tA6aGyZxThCmf5b1Sd4gFsLDmxJ+zBo6d5A+ia/veypDda6KXqXApFnFMJjVYWrK2oVQJCCoEwwIb5CEOXdLkqaDcOjNPItOxYlpXLsrCujbYqZVsk0gmvJlzOK7vjwMG8uzsMgTQ5J20cDgzDnt2uEeQlak4GHqfEkEYA1lqp1VURrRRqDdhcKa2wtgVKQCV59lmMMDQsZlJ0upOaUdU4Lyfm9cJ5WVjbStXipSWh00icy4lseFPyoEWnmjhZz5tECFOIHI8JSuP54QW/7yd/hneG53w4vsP+3Z/gxz78Wb78n/+/+Xf/g/8Nw7sfUh4+I1tmrqs3djoh/RrQthtcIsFZ8YQYGKZMHhKI0RTf+KqvCfBrfaVT9XKM3sCQTi/TRqfA9EzUXCVkbJiWB3BC9OxbDI1Gko0ipb0E7A0CA0sQxZDkUjwLSqNSF2i1Ml8WLg+V0+sL7aJeSmfvFmtRf73sW3+QgSlN7HbPOB52xCxURk5yz6yFtazEQWEn2FkIK+jqjS3FOvbqn815ht6AbNVoJqAeWLRVDKMKxLUwkTvlxwPMulbHjCX641vzDaZTFEJMEAopR3TA+aY5IHOgt5EBD7S+fsyz1RRIQySNiZB9Mzbp792U0ooT+1e9bpx+XTyB0f7ZRDxwq+q1t+DXVzsrwxHslBJGw1rfSqVRtRKiME0jwxAYc3IFVQjUVlACrf0LUHp7f6X6LrDhD9fOMf0sPS1uzPFH6UFSNu5l7w5jm+Spqy4QpwX0po20QqOitWCteFZhkKKXfq7giH4yY3QMLTiNIEY47B0QznOiVqWUlbk2yuoXl15KWIXH+4X9MXGoGdXmAHpM7PKemEbyPvD+uwl0JMV7aq3euZVIrY1SC8tSOD8Kdq6E0DB6GUlfFE72YUEYaiKNgiS/UbQoS11Ya6FopWrt5zUQU3BIQpyeEaMQJBBUeoblO3jcyhNAVBjTyCiROEyM4zM+mj7iWbzh5a98m4/nX2R3+5w/+NN/mi995ef4//4nf5Wh/Md8/bMzj23P43xhaSeCOIlaeolN3LrRnnE6hy6RUsRMvfRteAHWpJfcsjW0vZQOvhZ0w6Dl2ovFYkCtPTUC1fp190YdDawFNClkfz3e8IyR3qmXvhnHLhaIvevadEXXRrnA48NCvS/wWJ0sHxqhlzyhKK2CqJNJh3Tkxc27PHv+nHHKpCFhEU7jPS/jp5T1nnaZkSTMqeHQhNOQvNINHiiCU5ViFOKQ0dyoa/PPtPq1bF3AsJwrl2ElDZkhDk43Cl59SdgI/V6FGUKziNYNourYYQDJ4vh1cNmtmqt5LLj6K2WnpcXBg2UYA3lIV6wZ80BfizcwQ3O5saoyrxeKFlrnNOu2+W1qsSAEcZG8mSLNEycT1+FrV5sRIjlHcs4Mw8B+PzDmyKYuj0EQVbTT6X7Q8bkIlPAUAENwkLZHSDZlDmzoRz/J2jML6Z28jjOGXloR6Mx7rvI7Y1Pc9Iy8q2RqaUjfUGarBFxrvTVvvCPcS/JgTkYOxjAkYjTnaKZKy4FShLI2amtoM5a58OrliTxE8pgYdzvCNIBkYshITOx2gXdeBMZxx+Pl8coNbK1RSybMM2tz3bihaGvUWlhqI9aF1EBWJayNvFOSRmLuosyi1OplSBP1kjpGQnT1w6ZO8U556HLPAMWJ4g66e8BQrVRrWFPe231AnANfOXyNu29+xjdf/iKPrz/B1hVLkf3P/y1+6vf9Sf6N//7/hBfvvc+/89f+DzycG8MofHx3vuKS242lYWvKRNIQmPYT4zR42d0atW4Yq13Xi1nnAvbGgjXnhratv4Y5qVw6L7TL69zEoRNmt3JOe5nXrKunupzTjNgzJgmhry0XMBB6c6E3FbV1xde8UpfiVB3NWGu0aIQUaFUoCzAI02Fiv3vG7f4dbg8vuLm9IQ0ZE2VKByBSl0hdPmNZK+EIujRsjUjxspFgpBARGikK03FEpoiIl7KXsyJ3jbJUtODNsRgo2ih1ZdTEkJL3qLrYorVNnrqpXlzQ0KqTuM18RWS8yZJ3qVPYgC7uCCEQciCNrrgKyelszijw+6dqgVpp5iyTTf64NWU23bf1hquL1QVXZ9G5wZ4oaKte/ougXV0mBMZh55r8ITPkzDTtmFK6xoxaK0ULvgX/4OPzESitA+y9/PaA6Ds4vQO+ZRAWu6zjKWT20ttNLGI/ifEKVdi1nI8EkNRLZPECu/PMXCdt1Isy26XzrIScI9ECwZLz8jDvkDV9I3y7fMuVPxHGiNRKq+4Sc3msvH45M0yZm32E40CrQotOxdjtHYwPORNGWJa5f+bsCxQj5bMHNjGaNdZqtFrIOH4YRMipIUMjxkqIYJZoyYghIS0h6tJOV95k55MN24ZkXa/eNynxEtY/omfcMQ5oW2mt8GJ3y5fHL3JYE9/8lV+gtJUUEofdDWaN9VXjH/yNv8bvfHjNn/5TfwFT4d/+d//X7HYjw5zRnjVv/EkzBzwkCHkQdrtETILzLkEkUIo7GTVrPIXvraHWSRP986kvEadt/YaLLlybcbphzV39k6o40b5ZV61AEO0NiK0RJR1LW6ka0BJoK9TZOY9txSscrYBRFy+NSzTnMBYhMXB7eIebm3d49vwdDvsD4zhRbQU8U10vyrpU0AvBClJm5rVDQ6WrXlQJKTJMkelmIB4S4zCgYozFKO8Il4eZ06n2658YpgTRqVFbGChtoZXAeu2l+eYQQvSmaZf2NvFmKpirqsZIaIJJ4qpKC0JMQhpcNmsBiEJIqV8zpbYKpWfzV7jNm5aYU4ckGK1Z38z0yjKxvpatk4BDEJK5VNRUnZyfEiknduPIbhzJKZNjYhyHvrF5tz1Z2ywBfuDx+QiUgLTqix3rLGToKxlhwyPNu30dfzR6Bildx3vVhHItJWFTwEDQBia0FLEGNTVq9VuudDuxVpR5FkyWp0yqwTgEQnYqDj0jFVWs04q0tR5w5VqieHnruNf5tHB6mDjdFualkbNRaYQpkePA4bBDYoLkZVDV2mVZcJlnYtQrdtS00RSGMTJMIFEZp8zuIIyHCIMhoqgGYhZaibQQfTMyDzqSE3nwjq3E1Gk2esXtJDo9BXP2QDNfpMewJ2nCHhq//as/zquvf4/1cmYYJoJFlrWwn3aEALUGvv6P/g673ZE//af+Aq8f7viP/j9/lY9LZhCjBZeXamto89I+RWG3T26skfse2IAUnWt6tZGjO0w5rt08AvQH9KYfcg2GbPxZAzfX8HVhVDdXwbNuqwIFWIWWFRkiFjftc3++5h1Sbxa4m1Fr6lhyrWjrpPlgiLnJhTahrq7sabWhTchhZL+/4fb5c9559z1ujkdvCrVIo1Fb4Xi48HhZwBKSGsd4y2u75yxn2r34e4+NPI2Mt5l0iAz7kTRGYkrsiJRV2T3LHFdjuSwexM2FEGqFphFRobXVOY+dsB4lEGNGnORFMd+ktsyNFCB5VUDyx4RO8wkxEFLodDDP7E0CoQfTK5HJNslq51xqo3YfBiuVcC2n6ZLY5tdKvamp2htasW/4Gmi20nCWRhoCu2ng5nDgZr/37Dk4nzPGcGUsrMsPj0+fj0Bp5oAeIN0swUyxrqIBvGxGPcHoBGRVvdJ9oigxOi/fqXpK2hQfXR8aew6o6jhX1kQpSgsQg1FXqETvWLfKQ529y1yMMjby6HzFTWpo6mWB1QbariYBrVtTAb08c3D68XHm7vWJ3fTANBw6oRuQyJAjTGOn81TO8xlCcPsqaxQtFFvdXirAtBeO+wFiIU8D0z4xHMx5enn7nNHNJ4LfqBuMsJVFrgvPzmWMkY0k3FpzY46qDshrJEugaSGIcTPe8MXbL1NfXSjnB4ZhAhrjOFJbpamScyLnHcvlxLf+4d9ld7zh3/zX/wJ3D9/lG3/3r9FC8isqfi2sKqjTsdIQSEkc1xNxvqO4zrfWwmatVXvmAU5v3FQi2nlzIXrX13cHuGLedPilP3bDxEMvoVtxBx3JniXGENC4Ekyv+njwjVK1dXzTmx4miiQhZHEjDW8DowGqVbRFWAVtkRhGUhjZTTccD7fc3BwobSWUeOUHjsPEbpp6y1+ZEAaZeB3ueV0fKHcrSSDkQJ4i4z6Td5mYA8NuR4gJRZlKYZlXlgusc8WqYCRCMixUtIsTNociV8dUWmtEi72i8x6zdAhLkhGHAC04FhztClHFlJz61auUq96+N2FqLUgxmvkG7lxau7I72loRM0JVd3TS1nHnLlTAaFqhY5GG82yjlyVe4QVjmgZujgeOxz273cA45n4NwaxSW3D/h/TDU8rPR6DE0FJ7ae0ZnnZ5YOi7vf+ZuUWYNVAjOgpHjDCk7gIUnQcZugtN7FmAROkaZgUC1gKtelnagpBioMTGEgohBtrSWFZF7xdqUYZhde/BIRGj+M3QO6seGN1gtDbHeKrW7lbjTG8JxnyZeXX3SEheouZnA4u5iQRWIYvrlIcBW8+0urCuhVIu3rW0SojG8XZgHDKCkobIuDfyDuKAqwyonvkGB/C9+enYmoUAKRJjJseu4smpu8d4Cte0oGWlLKWLmLw0CRYZAgxD4+Z4w3yZXccrmWGcCGKMIQGK9Oz05niL5oGP/+nf5p0B/uyf/Yv8vW/+F/zy6RUrijRoXR2RENIkhIzTZpyXhIhLUsvanCPX8UkJgmm4Vhwi0HAJ3uFw4HhzoNbKw8Mj83z2jc3brmwEI5FNtNBltASiRg+cCtKNdU23DbxnQE3drkutw52GWXPMcAykmNCcKOeVWtx5SquxLI1xCOymIykPjPs9027vQR3I40DR2lUj0cnROcO4wwK0oFibuXnnQGhw0jM6r8jG301e6o6HAXJg3I1IANWRfF5JU2FYC60Y2i3cVFx9FKKSU2Lz5lSVfq56oMtGlEwcdmj161GHzizoJgkxdjhlY6hIQCR1M4u+Ec3Ns9Ombt6bxSsmvO/QSqUsK2YNUYe5Wt0EFrD5ubo5VkMkdlwZLPRM0YycIzc3ew7HPbv9zptl3SjFGQxCmwtPBsw/+Ph8BMprs4Xrjemr3klBjqX3k9/pQkE8VU8owTZqgBFzIyUc34jiVYRv9m6E4AUlNKMU13q2HClr8Z0qQRuUlqMb1xZFW6FmpSwNCUs3PXU/x9CNABzQd8cf3fDWHFx5Kt6nbE05n8+eGQwDYx4ZVKltRjKkMSLJgXGrlWW+sC6FUlYM70WMU2A/3ZBTcAB/VNLYSFnR4DhTlM3aC0roxN9OdbGOzSbMgVwcD44xO6l76/4HYcwR7ca9IsJEJEnk/f1zBoz5dGKQwZ9b3OczpuD2FQIxRpZ1YZ8jtB2f/pd/j6/83J/m3/gzf5H/5b//v6CFBa1CVNdU5xiZcmRM0bmO5jhiU78OtZhnbp0S5LHfSzv/kcMth+OO589veX57Q06JeV54ffeal69fM1/WLmDY1F7BaSetdbqRB1JRwUrz7C8ZmjJI8SaPYxIdX3U4xxtixrjPbq+mI+si1P3K+WFhOVdq8fJ1SEKeMvv9gWn0QNbUzTLojYhlWThfTjStTEMkhR21QItKJpIlIXPDznCx5sKKOdIKpJtESJnpuGMYhitmm6eRPC8sc2GZC1o2ZVoPhKH2+yQ6mfzN+1OEmBM5JdCEFmU3hSuGqd29qrXu9Wreza6V7oje3Ywc5HTnclzNFC2g0RkF2ipalbq6v4BouypwwN5Q7+DZe/Sg4ZZ/Dr9o097h3rE/joz7kTxGTwiymxZba/5Pu3Ve+xek6+3dNXNLKMNvPDPHhQBIV7pWsNZpJJsTTECkEZORkjClDXvTJ64edP+ZfpM1kBBRFUozN7WNTpnRVbGspGVlXaC17pcYhdoa67LSKhDEDTWiICFh5ioB7RrqSCJEuzalRAxdV+az8a2Pv8vtu8/5IBlzU6wGck3kKVOXC2VdmC8XSjEvm81xmuGY2U2RISVSBOkBOyZYtHmDqlf9Gz8xdMrHtUlghWpG1ESS0LFYV3FIL+1zTt4QogBOg5liYGDgi+98iX0ZKWVGa2Mc91077t1nSZmmlVorQYTz5UIIkcNuz8tf/M/5E7//X+Xv/KP/F3/9l/4mGkckrqQQmXJmGnJ3EgpgkVb982x80itX1n5dChCc67ebRm6fHTje7B23HTI3h4njcWLcjXz7u9/lcp75PhVPZ0f4Neq9wqbO2VzDdS0h0c3HVUk5dBWWB92QPMAPaeJmPDCmGwIjy7zy+OrMZ5/e8/r+zLQPjLeJvM88e3bLNO3BhFoapRRKNc7nC4+Pj5xODyzLA1ghp9HvhyqkcSAdE/GdgK0naCv10rC10i6VWhqjRUKHNzb3nmEy0hgZpkK6rCxzoZbiTAUTIPWsz7yzHxy1iimQwoS7rHuiQhocu5TctZcuo/QkJnRu5sI8r5zPq8uJu7FvkOiyx9rvXwlkHbCtw10brO5Onq6aYu1r2XmtanYlmVv3dnDLvMY+DUzTyLNnR3b7RJ7cxCUNyaGz0JnU5kG99s3ihx2fk0BpoA2hG8EGcZMCc+napsm5xswrh646cZnGRvxP0ZUUMZqbhso222RrzfSeehByhlq9vJIYaAliBU2BWio5QBoTeZjIKfcLEZiXwuW0cDrNXE4riQhSHMyW6FlG8GaC8y/9jcek1KrYooQh8ur+M54dRn9ei37xmqJ1Yb6cmRcv+1XV8cQxMebENLqJQE5ueS/RrcJkDaAJofXdHdrqjQ2SEjW6UqV6c2aVAgipRMSiA+bRW2UhBnIWYpowK0SMm8Oe9+Q9Php/G/W+uENOd3Cy2ggIay0cDrfM9w/koD6BIiWWtSF5Zrx/Tf32P+XP/cv/Pf7h1/8B31XHpnNMDEmIwelWblrsZXmpTr7X2IgZt/fqDT1tbpKANUJWjs8mnt8e2Y2uXR6GzDC4xr6p8vj4yDr7OAp3KjTvrEfp5Grf2IzUtdU4BqkdBxMjZZBuaus0FBgkk/OOw/EZt4cj+3wkhx2ByPL+ymfvvOKz+1fM64n984kX777L/tlzpv3gjbfaKMtCqZWHuxOvX97x8tVLzsudn5vBM90Upb8lQ1LmeNyhy4HXa0HXxvnuQthlZLxgA4y7vZuO0JCYvDscIoOLNVFdITgOKL2zHGP0wCACGsiScMPOBDGQSNc5N2KJa0TtJazHWWWcRoYhM00j5/PMZe4CDXUTkt5bpZnRpDgNbF1dT14hSiWnJ/bJMGZS8nVqUQH3YXClTr26BO2GHcf9DYfDntubW8ZxZBgHly8mJ8NjQlk7xoyT3X/Y8TkJlH0XB2/1dw6X0wZCxxM2ZxkgOK4idHeb3snEuslSsC6y90UvyHVUAB2P2vznQg7u+dDEVQvRnzsEiHnPkEdydmpBSokYMxBYLgsvX7/m1Wd3XB5mtBWEhJr7NEow2lJdIhg2xCYQspttDDlwWR4pdmE/7BxQl8LSYKkrZfV/rhzZOuhOxI4pkAYHwI3uzh2gqBuiGQHVQG2Oywbt1JZtITc3Iqj1gg1KGDOOQ6bOFXSnFYcpLsQcmFLE2sruJvN82FPLp8SUsZD85tqA+iCcH07kNFHXk/M2Efa7RKkr9w/G4eXH/Pjv/P38sd/xR/n3/t5fJ1h2elOM7vAUIqZ9wzRQCiFpt+7viyBY19x3LbwKVStNK8MwME4T4xD6TJtMjIF3o3GabzlfTjzqpat28EAbHNbpTLTtgjnnsgW3oOtGzapKSplNBSMxMoXMtN/z4vlznt++4OZwyzTsoAiX08zuOHJ8mHg8PyKj8Pz5c8ZpRKXRWmFdxDOwy5n7+wfuX7/kfLrncXkgSmS/M8ZhuhLqq4CKe16uBo0ACuXU0E8u7unZ8cFBMmnIfSMXd5vKLuOtLWAkykrnIrcuEfUNYNPKe3DxNYIkzxx9gE3nxFqnTNlVPVOrb4IpBY43O+IQuX88syzL1SVfq2OgVRfq2s1nmhPLhwQyuDadFHrja2K/PzAMIyYr1SpLnbFWeo9CGOPAOO447A9M045xGkhD9gZT8OaSaqVqoajLKUtZf2h8+twESsAbNPJE9FU6oVjkSrdxOb1nnK16mRgbV7fm1hpJBXvDDUS2ZkZX/Tjoz/VGCF2JE4IPU7Jo1OTOKsM4kqKz+nPO5DQQSCxrZdy7O/Prl/c8vjo7383cMsxhAVcahYZbbhlMQ2LaTewPIzEql/UVu6MvBBNlLeqmrj1b0jcaCa6YsK6f9caCy/uMZa0sa6GJ/7xVJ96KuBkIFqld9xw6ybCoGw+IDf785kSQEIO/prqKpIoxpkwWoS6Ry+OZ2JzkG6XbVYXu4GSVVmeOh+fUafKOZZl5fLxnmEbOj2e+/e2v89s++JA/9yf+Tf7TX/jbvFpPxOTd9+h1bpeBQlQ6HNAYslBEXYEjTvFw4nAjEigVTqeZpa7cjjcMQ2QatyxCGQbhxYsb1rpC/JT14iWn6ub9KZ210EByNxLxRgKrErN7aJL7WApxqlXASdXDlNjtJ54/e4ebF7eM00hskcvpwrCLpEPkUPakHLnZH5iA3ECqUsrM5fzI4+MDDw933D98xuPljkt5RNW4XGYO+1vGNPosIvM13qInva60TLQKp1czJtEpQ7sJzQmroNkrBd9cexNlSGhfN611j9R+jyTBecEBt+ewLt20hso2QmTjPdvT/dXvow0HDjESaOxiwmQkJjjNRqkF+jqvzZVEXZROsuzqGTUiAyntGYc9+92e4+7IOE7E5LZ/RsGsXCmDMRgpRXa7HcOQyLvkHOTQDTXWQtVCbYXWVlqr/6IESrnSDwwX+DuHOBClouJcNPeeczWNqmecSsRCg9SwaFDdCEECtOYd8ba5EXVW/+bJ5+mDY5dibuRJFDQqKbtmdRpHdtOBcRgZp5EUHZupxchj6tPeRnK85/HhgXlZvMyP6RqIrUWaCNY7eykG9vsd001mFXM35xgAZS0rZXXyvHfRcSljWQnRp/TVWnuWXSEU1CrLulLWihLdU9J8GJmokMi9N+Y2XaaJEN0af62Oufr7dKcfV1a40mFIGWJmno2bsOOD6Uscy8CjGDn4Dt1aI3bynRskBM7zidoaQxTymCirsq4+6MxUufvmP+ULf+CP8q/9oT/Dv/+f/Z/RXXL6SZ8iKOIVcTFFUnOT5WZEekDr+l/vSD99cz5feHl3x7vvvdPHYDjWqn2aYsjG/mbgHbvhdF6cPmSg9KFiHRNWhNrPlaq6cXaHfFAgOrbmegbnykZ83MHu+cjtewemnLEGcXTDXZISS+R2nNg15YUM6LlRLs4usPnCcjpRLzOX+9ecl0cWPVNqIa4zpVUOu2dEElqEuq6s2oNViBRtaFFUAg+vFyw/Mh72SAIJFSwyTINb0iW3XNsmm1owNPhGILY5qm8D3HwQGpIwUafidblxa7XThlo3EIlXvDdsyifBzVrUmKaJGBMhRi7LzOVycd5pFWwVaK33x6qLPHIkDiOH8cjN4ZbdtLu6AAmRnEbGMROjUqVQbQGrbu4xBPKU3JU9eBO3tea0p1poZUWrm4+kTVb5A47PSaDsOBFc3atdn/VkfSRuNd3Z+Rv1gDecxbsxQMIHrwWuz7nNKnbDUtdhSx91uZG6RcS7xSH2cbVG7tjINA3c3B4Zxx05ZYTAWpSQHduz1h1YpJCWSG1XhbHjOJ0DaM1vwhQn9rsjh73bsLl3pFLqyrKsLLP27E/RCuuystaFpIEQEjWoA95h9U6s+pTAqoHWSbdm3fdRfHqhloau4do1tuD4U6C7sXR8aqnVKVo1Qo40TQy2J1ri9vA+u/yCejc7t7C07mmYugbYr8s0jVStHG5uuJweKZeF3TiQYqKsM/N55f6Tz9j903/Mn/wj/wp/+x//P3idAi0659SurBtFYoPQnJcoEFufY9ONS7aO83aeMePVwx2f3r3i5pDRVn2oFm5cEaZALsLOMmEXuToP9RuotuYBoEBZG+va3KAhVrYBa2oeOF3O4Y/fiNDDkEljJE8Du3GkdpfwtgRuLpEv8y7P28D7acczG1hLoyVhmVc+rZF/dL7j4/vX3F9OFC1OvsZoFDSeCDGRZYQKRXz8yMpKE6WYE7C3aZ2nTx+5O+5J4w21FVLLYJCGeOX3gpJzdMOVzc2pmduQkdgG1Km5xLO0BakFswVVn4Wk5iYiMUgf9ZB8NEXc+LwdC+xECxkixuDjkYvyuFxoa8VqxYq5Zyqu+gk5gAVyGokyEmxAcHunmDJDTC4cELy51H0ZYgjdJ8DpZNqH6zxNKlXvypfmJin6L0SghBA8MPQkD8yp/iJAHzRl1m0KukHlNjWwlG7dFQMhtm7nv/kWNjdccP08Sl/avYngRgddD9xf24JndxZ8/OcwJfbHHYfDrZeZJiyrm0vUUlmWHWvtZOEwuxEp/TnEgea1FRx2c6pHkokx7chTJGdACrpWvzmXyjo36rLQVmG9zFRcMhFrV6BQILhGtbXgkkak+/JFtLqCJWi/nxuUZescusKho7fXhldMfmOEkBEZCGEgxInYBm5vDyx1IIdbTB+gqndicVpIygmzelUS5WHCVIjjnoRSlgurLQzZPQjXMvPZN3+Rr371v8vP/vTP8Z/8k/+UsnOD3C1D9ZuwwTZRsLmaZyNFO2vFYQJF+80haCt89+V32B2E5zdjd/lvXkKOwk5GZDCG1ogh93nm1jXgsTsAFdbSWFcvyy6rN/m4DtXyd3C1/oLr4Ctrzgu2NCAWuCmJdy8HdnXkA3Ykg/XVTB6UIYTOFDhwlMBdfs0vX36Z54y8KoVVXa2m6vxYMIZ4QIvPINI1UNrKaoVLU6wkRAWphtbKp9/4jLxLjLeR1CkEWly9hTRMKtIbaE0CSPcvRV3csDlL4WNUTEJ3umpU6w5VtXJZu1XZ2eez76bEbvSkoqoLMmJ02pJVV6YJmTVPrFEpWpxlUBx7thaR0NBgpDFzGgpDroRufUd3s0fF3ZjoVYApw5DYxrNoa1jtnFzccrEshcv5wnxeWJbSteXXLslveHxuAuWmkng6ulNK51iKuJWW/3HYMnq/2MV5WC2vXq5WI6xd2REFkuNanWfaaZvbInePPmsKXW0DdEdkpw3lMZJzcrOGnROBw5JYayHPI2kc3OB3HN19vHPyQgio4FlszaQUmMYDtIy1jNYBaRmfCBspy8ldWlZjvVTPJGd6VuHO0560eefONlNWFZDh6oLdmqAV6lqhVAexCQ5JpK5WyfI0TAwf6hSi45MxZoZxzzjuGeLEqAOJG965eQfmSjNjLSChYKFhfTaJkIkhorbQWnNCes4stRBipNSVOi88v31GGBOlGPOv/DJ/5Hf+IX7+O3+fu6zMpdJCpbWZKosP9fJ2LKj7LNYmrNVhiY1CmbMwTI4V5jRh1nj58BkhPmMYR+hO2XlI7HYTIQlDz6RTjATxpo9nH77W1qV6t3aZmcvA2hbmdkLCClauGLE3FZXWKm2ttKUwXy5IUZ6dEx+8Suxegp0Lj3VhvZwd/hFBmzvuv/PiBR9+5Uv8RLjwS5/c8K3H2eGS1ne6TpNZ6uql45qoS6MuiaW4zFY7idyNYQzTyPm+cv/ZI++MR0IMzMvCapFYnTPaNLgna2+8BPFSXHrXzGd+b4PN3BYODdRmLEthWVZYumFJa1QtzGVFQ0LSnt1+zxSye8GWRuuTKJNkzJQcA2MKFImsy4ouuOMRhRKEdsqozsR2h2jCbqNXCNZHggyR2Fz6qzSHzqobQjdTylopnSa34aHzeWE+rSxzZVmNsqwdL/3Bxz9ToBSRXwEeelSrZvYzIvIO8O8APwb8CvDnzezVb/H5nH3fsconxYS8wZvr4JWIBwZzr8KmzkULQcnBPRpVlGDB1SkpXDMQfy06V8+eFoY+eUp6leXabnc0WsFWJGR3SI6BuHEYc7zOqYkxeoYQ+jiFGNxod42MeeCwO5BjdorODC2H7pEYqOfAevG5OVqMsvj8laaKJFeDFFzemVJ0x3dLdM9hrJddtbpbdFvBqpfpQ/YhZmvHlAI4YyAIIeTr+Q89M48hMaU9Q9qRJFH73CC3enXDAS1uJKzJEFtc1ytGzgMhJiRGDsc9a82YGblGdF19iFu3PXv96bf48u//af7gj/0B/uav/l0uGKvV3pVU3wia0Valzo06Q5kFrdErb1zdMeToeFY2QjRSCMynCw8pMtRKzkLOAZVGzpndtMNSoDbnCYBLOt2IwRs01mA8XZjWmXnJrGXh8RxY9eQcXOu8IXH+r2ljLStLqTy7axznwhdeK8PdPY9tdbMN9dsuxeCWfyqoNj75+GOqwpe/+BV+/PlX+db5U1Ycb/R9onOHlW6CW51dEaNLLaVe3c2dJYFnhk24XM6UMhI1+SZZKkkB3bwn1ecoWadobJ2q6EYtwzgyZnfgCZI801djXyeWdWW+zJzPFx/5QMcxY0BDQ2Jjt9+BGmWptLWPk8AbUa6ig5QT45hYijd2thhQ6wJz5eG+Z5LWqHVPKSPLWMhDcNrQKEiojmf3aUfgeGtTpZTSKXO+gS+deL+uhbYU6vqjJ5z/K2b26Rvf/2Xg/25m/5aI/OX+/f/st/JEb2pvPUBu5bigug3L2v72yRoDjFY8MNWgxKS0vtN7QiIo1ke/9MAbXPomvTm06YddYeNd0NzNPZ1bV1AKVVcM1xmrNjZWm1uG+fhLrZ5Nhu7uHNNT134aJy9Za6AVoc0CVVjKynxqzCejXArLpVHnhpa1N4WcY+qzfryxZeKLW1V9Xrb1TFI9s9DVaCpYqWijl9uO2WYJvrh65/Y6mMn3HtSEEEfG4UAYMkn9hlrPFzjPVG1E9WFaqkISfLIewelZdG02Ss47zpcZXdwirpUVi3AYB07nO8qnZ372d/7L/P1f+3lOZfHStPjNW1alzA2dlTIr5QKtCFZ9flDIkZACmqTPzTFIBTPv9N7d3zMuC7tdZBgTpbq8cRgGx7jy4OqMRq8gAnkY3IILY9zvmI4T6zpyvjwSg3BehdpOXMoZJzsbowSiuNNUernwTp35cR2wWiiq7BmpWcjDSFlcrhlCAx1oZWFdZ16/fMXzZx+gqlyWk091tE3z3sH7pmhQfOiQ82MlqmdUEeqGn/Z7KItLe5dlIe0EqV5WPRWr0ZP1+mQtFzq0FaKQhsy4H5mGiZyGJ0I+SgwJ1QNlLZznC3cPd8zr4hStmJnGyZt3UgnjwJQzbXZ5YlsLPslq4+26S7mErsjBx8mG5NMw1QoPp3uaFU7nkWkYGafMOEWmw8Swi+RByKOglmi1XrNICVBKcQVOqSzzwrrMrKtTktq8ekPthxw/itL7zwF/sv///w74G/wWAyX8BiW4bB1xw2g9i9guVVcESKekV/MMrnQT4J46BTOCOrUkRreBiupyvhCTu9NIdBdtcUAvBMNkQMQvnIZM00aZF1pw66t1vXApF9Y2s7QzVSu2CfzDNqPbgeZhGCjFaRhxyASSm23MFcuRy9y4v5+ZT15yz/PsjYTWjWY3qpEIBM84idFtvEw8I12b43i1k80rbmVl0KrTJ0IwJHup6Z1cd0+PwY2CTSoSK2FdOK8ndtMNU5gYUyIuYJeCxES5zLSYKLhx8TQkIPkkQxJFjbou3L1+zWG3d6fuHLF1ZbefMBIWEvvDc15/9zu8/7t/Oz/z1d/L//OX/w6nZWtsKeVSaBdjXZW2mvt9lj7+tjtpN2s+rbAJoTl3sAahmiHrSm0FsRHRgaCZmlY0ORl+08HX4rZybrjiEx4l9Nk4KSEysnEF8xI4n30kwtouZJkY8w27uONmFn5KJn4yHKE1zmUhhgzBOaJaG/tp5zK71NBa0eRcw4fLhXM9sYs7hpwI5wvWKk3wsQ4mII1FIVgBMypGNXf4CcGbJWaeVYbBGEYfpSHmstjQAmSB4JXPxg0FJdjmMO/3yTgMHZLxSaIpxZ5ohO7O5ZaFIyNjS0zPEsuyUkrxBiqCFTARUvDmjCAg7svZzCvAGLbhcErIRhLHmt2F3LnODUWt8nB6xB7uiUkZ9yPHmyOHtmNqmXFMDCWQl+Dc2eRJigh9rrpSFh8nXOfeE5hn2mys8492Zo4B/zfx9tn/ysz+CvChmX2n//67wIe/taexrWn6xs+uRv/dC9L//+pYGTaskZ4Zbs5Ahlb1DGcNpCiEBkSnFcVtWFWnergFlAc4J2VbnzljnRuo1LYSSkO1EcLKvKycLycu8yPnyz2X5cy6rqBOGZL+EUT8uWN0EHxZLqjduiZ2ragWch0dr5sr8+ylaWvWVTnduKGpO0dLJ9sHczlXN71oamgxrOjV8ks7txSA6O4sKQhDdCy2rQoBSix90JN3N93oInCSR3bJ9dxjhFgzoflI0dJB8pScRnVazoxWGHTCQiPlgePtC25uodZCnvYoK5aEuRYeLyes3mCl8MGH76H3n/Bzv+vn+OXvfJtv3/8C86LM50adhba6/ncD3MM2TTN0alANaHeEaVU7F9QzYMxde+IKKQyMQ0QLPqNafSbSXFaW0lBRrE3Q3J7LSZMBNlJ7p80cdjcMKdOo2MU4hiPj/oZ3ZeRn0/t8dXiXZJFHXQgEUsgYzqjIIRNDIoph1RNDbyQpKbmJ9G06kjRi4q9vqripe/Bs0bb7ZOv8b1ptp/KogZhzefMukadIHPBpH6F3huWJLaDqYyJ8sJwHQM/uPIiadf6hNPLoxO1oXpVpX1NRJ2KOjLU4aV29o2x9CmRSIcfMMGTOGLbMaGxIUKZDxGIiDoLkwDpXUnajmpT7dbXeYa/WEwZ/T8ty6cbAPUYYjruaMlhmU/Vp7X6z1acQVPWm6eVc0Lmy/ogzyj9uZt8SkQ+A/1hE/vGbvzQz60H0v3KIyF8C/hLgQvXgEd1Vx08ziB1T7D1O27AXoGN1PURuL9gvUqQ2I5RGaYZ1HMSqEVNwUnZo1wFXqGONbEqLvmuaGaU1lrIgi2AMpOT60Mt5Zl7PzMuJZX2klAulVGLQriPPXbuuVwK9oCzrzOl8wqY9VBf1x3RBcfLrshYvTzs9qDU3qfUmet8ieqbc8KmR1h1sWnU/TTPrVKR2zdC1+/NFcaUMfc6ImWua6+Kvy7oQmvqsHDtzb68I1ZDpyJfie0SD0qrTUpaZdWk8e77n2c2R+1ev0VKYdkdII7UYh5sjU4qkPDBfTgjG6f4lwoPLJMcdl7ky3j/yzld/gv/W7/gZvvnpSz55+DW0CjlMSIq0OpPES3bEJw6KbHN/3KZN8bEFVKeB1FqhrVgrLEnIQ+Yw7ZAYqUv1LivCZamcLoXzfOJwHJl2ow+CmyZijOxwSpApxL4h55DZ726YYuYmTrwTdvyx29/B144fuYmJVqdz5QE1GMbJ4fDgnFbTRhpS5ypG0EjFnfa/+M6XGH51pCwPxJi9GhDvPRvSB3f1REICkgJERUOfcdPL6xgDISt5ykiix3zpngcAPrPbjXJ7OS7+N8ITrWethdQCkiLRogfG5HS9IM7PDcGpOskiaivaAtoSWhzGpTjlCBVGHVjWzFAz2gqhCbsxM+w9oLeSEZo7/fR+gWpwPvHq0uZgIypG05XaAmrO1Nh6GlfTE+tVkyq1OIOhNsfttQi0RFuLe3T+kOOfKVCa2bf6149F5K8CPwt8T0Q+MrPviMhHwMc/4LF/BfgrAONhMKCb8HonWnt8fSrFn4Lk5gcIvVK+8uhaNyBtIL4AW/S5JRuZvDWXKVroGIyZW6GxuTaLd8rV6RO1LVwWwWKl2UAMzhlcloVSVppWxypjxWLro0HN3aDxeTPW33irgSaF+4dPgWdugybaVTyN8+JTElspWHVOn/cKuqON+Gdval2547u1l0/tmkE2bR2i6sqVjtWa+cCwakpC0KB4Veivvy7VYbBq1Obl7rqsnOYz6fhljl/YQSysncaVY2JdC3ev74lEvviVr/Ldb/0a9/cvGfeNmGbW9ZHbFx+wO94SU2S5XNgfbqg5si4zy1IcCzwZ8o2v87t+6g/y0c//A/7O13+RF9MNs1QUYT+MtCiE0qhNqKGDqbiZM+ZjL8raKFT3rVQPPIZSysLpYeTZ7XNX1jQ3OKmlUlbl/DDz7Y8/Jmbj+fNbxiky7gd2hz23hz0hRlopNGtUa0RT9jIyTnve0YE/9uFP8eP5HebzitUVIxFDxoIw5smZEiJv+C6a+0jS8edk5CVzOT9y8+6XwBIBYSDScp92KK1T5ISi2mmc5k2drSKKvnmoKSFn4jRgGWJ2t2/n7Lpl3laJYIEUgtvG4e8pCL1ZCcZKWX2w3MqCBCWmPSmNHlS7qbNROg8zESyhFSxob4y6Y7zW6jZwu+5dWhutFsBpfVOIPo4jRGSDG9TQsrl79a62uQdEjNqdxBSRxjCOHV4L1wo1kP08W59jX5W6Or2qLVDWQFl/RDxKETkAwcwe+v//GeB/DvxfgL8I/Fv967/3W3q+6z/ru6dRpRPUrk2XNzPKrXPdnQX7DosIqF7dQDYvZVXttlyGJmghEPskx2gNM9ci0KV4AwBuDrrWBVl80FeObiNfS/EAKcY4ZVqb0LpQZwPtpGUFq06AVczJ4dYo1Xg43bmlG+40pNpY1pmyLu7u3Lp9l4U+JRAf3kREzfGd1swHVUnstvh9UYog0mu0ranVv1dVmrl7tdujtf45u0NPNWpwzuU4VOpQqFMj5S9wGwYKTgBuTVxK2kvD8+MD3/lW4f33v8D3PvkYE7pby8D9wwMvX73m5niglYUUoVYl5YHH85ln45H7hxPTuGOvyh/5XX+YX/iVX+J+nZnPTm6XCHmYIC6E2ohNuoVXc/5mNVqtGEJKA2gjR4FUCcGRtKWsrGvFBscvlSeiuLbG5eFErTPlPLObBiTBdJi4vHjO8ebGBQ74pnITIlPc8U6Y+OPPv8aXw/u08+pWUxJJw+QcxBAgZFLsTk3Bm121OLYe+7mP0Y0mLuvCEDMpZHZpRxVjaWufQV27E79rmoM4fheSEaeGJmdHWI/A400mThCzuN1a59S13rgj+PTBIMmljeKeCLHjlzGkzjMFW421Veb5wlQyOWWGvCfF6GEoGBp8/IRao8wrar55V/OOfKv16Z8Vd2YfuzqoOS/Zbdw8eGvzZEDB+Zfm7AUNitRETAMxubSUjmFi2XsD/X1YrZ3LCWqRZtGbkos3CudZOZ8a5YdX3v9MGeWHwF/t2V0C/vdm9h+JyN8C/k8i8j8GfhX487+lZ+tduisvzby170R0oVuBe9gTX9xIL7sEJ5VLo4UnfiTa56VsmYc6ptKaj4uoVck50CKMo5CjkEYvz3Of12Ldq7AUkKLUbWZPp1DkHBEZHTLSwMVWL39VabWg0T0Gka6x7YG4lItnjebqioBRqruDu5rAA5KoZ5M0D4J9/oDvsuoOKOJaTqcZYZCcwE+wp3Kq8xEVJQEWsrvGiCK44qlUJzA3CkEqNhsMgq7w7tcO7CRxrn1WgEnPaEH6TPBalY+/95IX736Zy3whpAlCYsoJmYSyrgwxcpnPjEPissyIFc7nE+/c3jKfL+ir1/xLv/cP8Y1v/DL/wX/2N3i2P3I3n91pPYDGRCguIZXoi70WpS6uHJnGHfuD052GCZZ19tk7NqCslHphrUd35TbFWkVNrsYe2ozL/YnlcUYipPszZfZRtCkHWjBGhDEd+N0ffMhPHz7gxbjnMt+zViPHkSHm3k0O5Og4mQMePg5WtTgrokNFKSVqm0kxMFpCWuN2f8teFopWpCaqGU0cf9QghKBXxxwPgM6wICqa3IdgOCbCqEjy8SUShZATrdfZrfhY1yS5G+pYh6MS0zCQh3ylcbW2sswrJhGtgSEUhlhJcWRICcvu5UrwMSaRxGILYo2mDjtZ9/r0yGO+PqNvGCmE3uxxKzctDpFVq34vqc+nNzXHoHvGGGNgHJ3jHKOLGZpWz3R75VnVPU1LrbQVympoDdTqlDnDHL74Icf/34HSzH4Z+P2/wc8/A/7Uf81nA7SPpuxDw9TVAqbdHZo+ZVq6q3VPLTWYd5Z7Zrk1deg8LLRjmL1xI5283GKFJtQWGIZAHvoo2t2e3d5tzKo+0ZNq6Z3Bplce5eb1mPpCbH1s7nqpjhd2VYW16pirOKYaomeEhk81fIIZutdfl5JpeyKFSwNpPeCaQotOJ+3zl8WcjhOCIS08BUciQRToTACUKg0kuHmvKCUo0gIhNCR4c6QVKKFiZaZVdcrOfGJdl+4BWN3lveOvhEQadqQ0sqxnDjdHkMQ47hEx5nXheHPrri+PkbKeGIeJVuH8uDCFmcO44+H+JTcffcS/9Af/GD//S3+fj5eFIu6A3aSQihHSyGMstHnxhK1TbZ5PB2SITDvhcHNA4pmio5N8W3Mys1Xm6s5BtNAlsa4cam2hdvPcYEaowrwUdL1DlsC4n2Bs7MaJ3/Pl38Ufev5VRlPOpzNWmg/h6g0TQiClyVkVvRIKAUQLkXatisx8oxnHHVqLGyK3lUszQkyMCTRVpDqNSdrs61DdMcjdfnzUh7prBfkQ2b8YGG4i4+AzYtag5K75FuusiGZESYwhkbqIIadEjJFxNxKyO5M3+hhjGX39kljnxvm0EPNIiJHINizON2rVSq3eX4iW6B0YTEKX2BotFhgaURWt5q5BCE0gmjqPVsWZDktBi49uyd1JS/qwPRF37E+p/1zcgi10fXfo/NKgPeHomv6yGqUVQo7kN+C83+j4fChzOkXMRHr57SXCVT1z7WpvVIa+wIIHQe0Zoz9mkyt1A9Lt85u98cWuP3b2kSApEAZX4EzjyDR58b0siweH/l7cpip3YrJc32MeMsPk81NEYaVCc06jl7ytm3OIS9BjQ6V1R2kvhTd9s7tCbL6MHrAdi4t+wxk4T/HJKNgUx1/Ud0fP8gwJxW/Q4GNhnW/h3VFi9X/4zRrN5yErvqisFZZm1LWSK6ROoVoX9w7Vpoiknj031nVFJJHznjEfkJR5PJ18UmFduX/9GSkPfPjhB3zz5fc4TJOPHtDKw/nEtJtIY+byve/y5S9+hf/2b/t9/PVf+XnyNF0lfMPODXO/8fo1427gElbu65ndNLLbB9597zn35VNevJhI+YZiIxe98HBesAyNE2tL1CVCcWJyQymt06fEqxczOEwHHs4nymmFYcYuC8Mh8sd/7+/n5774NcYKy1wQnIYSQ3IOrrnsL6XcM8h2tfei/5Oe6oeYiOKWgMTo0NCQqDvhmJ5R1wWtTsfRFdZWuh2cXZMBz6wEjYJlY8iBw83IuPdxsXEIHd+LVxu9ppUUAzlFEpFdyozD0CcVTn3SohO00U5sFx/oFYJn4su5ktLspXoK3vSJgmQYxsGrFOcHIU0797dbt/XP0NqTH6SZUJtnfq7HNqe7rcXlmhWkuZeqJOdxumO5ezykKH2wYLw2Y82c6+uPdyaIttw5o6u/n2yM+/GHhqjPR6CEa2Mb6dZq1162/9cRse27rYmz/c626rp3hTvlgl6C97C4dcKuWad51zAN2W3iY3K5Xw+C45AYhsi8RKftVMdRgoTugOIlmwhEC+SU0KzU7DQO6c7LW5DcDASQzu1M4sB6f4cpdBs12+hJ/nsJjitZz7C3+40GoU9J1NYpHp0GtHX+YvKRDBbAEoTcfIxFFJDqJToRUYOgaIhvyAadCXCzP7KLE9aMcdwxnx56V70QQ8CHO0bGaccwjBCNz159yu2zF260mkdYhdP5EX18YL2cee/D9/j6L/0it7d7pjFRWuW0zBzKnuHugduvJb76Ez/BR5/+Gh80z4SHaceA8b27V7QDWDPWvfJL99/g+bvPuF/uOUzGfrfjZjRePH+HVWY+fVwwExYJaFXWOqNroq2pd4KdISD9umrzEakxZJIlIsJOhZ1V/uSP/yH+5Fd+mmgrl7q6bZ30ABQSIQxYSAx90NoQxTdVrGNn0g0H+jUy7ZplFxCknNEp8/6LD5hYeTWfYA2sNWIISy2srdGkesDt5TzBg6WMsMuZOKhPDR3xIWDZR1hUXBI7xEgOkSlldmFkl119k+KAkajaaFZBQs9cixvuqrE2JRtusrtm6lJJow8xw+iqtcBu2hEsYTZDUm8SUq/iBrrpcVO8yqvGsqxOb1u7cm32Oeht9Q17yhNkf/6Ug4+/DV2f3hMMT5L6tM7mIynWpVEX5XJeOZ+NsritXIiC5IhMPzw8fW4CpfUs6Ur16Ya7pu3ayAnSA6J4gPOZxD0gbhkk4UnNI53yg3lGteEjG4bZS5AQfIhTTqMbk+KeeuDA9jRMoMKlPSWo/m69xA0CQ4rUCCUakuvVfNTEurfm1e+GzYDBqu+uEhT6MLTQMVbzeawgPiBNg1HFd0ZDulytj3Gwzu9MwJghF89Uk885kazu9h7pYyscG0Ji75r3xWqRoMlVOF3ilmJgN47sj0fPfmIg54F5vXRnaSUPe8b9gTzdELJr4Z89OxBCZBwGzvOJ47QjxcSrT77H+Xzm7u6Bm5sbPvn4O+z2Azf7CVPlUhbS2WjnB7700Vf5mfc/4jvnC8OQefdwIKrxCwTeOdwyzxdO1jisgWm/42XZEYPxwTtfZk4n3r95nzm8ZuURbXCWC+WSWEtmLTPWKqsZsTpzYTdFzAYulxmpPsoix8Qoxpefv8fP/Lbfxs/91E8jy8ylLZyXlTGPoOra5RCxHBnGyYe1RbddQ+huT9BNsAh9Lebsw7xUm/9tHqk5Mh5fsISV52ng7mQEEywLl9QoeLOQ5iyLVnwOuZnPXYrDgMYAIfXqzE0nxCrNts01MsbEFBJTFKJ5UDFpSHLj6laNqoW1uXa/lOr+oq1heSRHoS6JlvasqZImQX1cOEQh5UQkEyT0yqShIRHE/UOlz3bRKp2tYZukHylKW1xv7xrw6AEcZTf1UctRybGRgld62yBCDxHuE9FqRfsYirU09wcQwWIj7FzZJSEQhx8t4fy/kcN7I1tH+6ks3owDNiutjUfppbZ2eyW7ZpoeTLdMrGdF8sZ4UfEM0P/fPQxbM+ZL4Xiz6ygnfac3TLv2WaRruQNCZAgDkdDdZmAbnuRDzq69I/88wXz0qxUPTG+mzjh5O7CJ9jsWFL3UMgtol3H6QggeOzedIdblln00Rg5IbqQciIMhST2LHMVJeKGbFEfDohu0KuK9sRqhZqzgDaRmhATTbsf7zz4imY9SiH1crwSw6prvPIzkcccw7ZAU2I07rBnnx4feCKg8lMrN4cjtzQ2ffHbm9Wef8pWvfZmH84MPSksD87pyqA0doF7O7G9u+bEP32f8+CW7w4FjSv6533ufh9L47P4OGxNfHA6QBu7bSkqN6fbALBe+evwyr+cJ2RtTPXHSO5pmWj7y6QLrWoi1oRi7YeAZmSqBT9ZCGiP7NHA7TPzYe7f8d37PH+b3fPXHseXiDvRVSX2oluB+szENpDyxDdoak+PJIURyirRaCNFLTlXzJk4tPnbVnJBucSDv9+Tjnp1EaJW87GjRiKERJbprvTlUQ2uO1Zt1e7FIHCIhic8GBy9JUEIz1tbIJkhyHN75U4biJXIziC26A1do7nHg+h+arp2G1ni8LBSrnNeF07Jy0AO7tiONgslATm6rN+0nUkpkyZieCO2CrY2Y3LharNDqjI+wdfaJtxbcnJvo6xAL/vmykAdh2sVrkuEVoNJ0JViiVfMpnhrcOq8+GcVoMbQ5fSxEN0oZpkze/fBQ+LkIlACbvtttDWUrljupHHonpgcg6XFm+5183xegQ5Ybj/ApSG5B0zFHD1iX88r5dGG/S6glVB2L22a3bJhH7IO4gvggLReyRseYTAnqBgYbLLA1CkyUFrwrG0S6k7jjNIq5vX/zTC/gXDEEolnv6gut06YI3qS5+i9uGFWKXRvbJ9MlJU1OFQnJXM0TgewGxyH0zaIrHtwxPPl0vNKIMfPuO7fs8g1fePcLpJhBIiG5NMys8zkX90ysxTONm+fPmG1GRNgddo7ppsD3vvc97u/vUWvElFiXC3enE1/40lcplxP7yW3aYk7sjweCCiHvOOxueXd/dgegutLUeDZmnu0PvH84UMWoN88Z88TL0yPDbiJOmYflkffjO5ymAdbKzeF9Fj3DsCLDnn/4MDMdDpxOZ4Y0cMyjZ/jDxH9xWalT4kvTDV86PuNf/6N/gi/f3kC7cGqFGiAMmbasno2kiWl/JA+jl+DWq6HWIES0rN71bduoBe0Qirr8NvamUh8xUWJifzxiFrFSmZZCK8oilRCSm5i0Ap11sE2UC53lQFBC6jOYttcqhqi6e48JKytFMsOUvMQ2waqhVslDxjCfJd8WlnqhtpWqM2WttNJYW+Pu9JqYMuPwKe+2L/BcnzEdBqdriZEkMg5GzhkbjGlsroLxqXU9LemVnrq80xTq2ijVec+xj4d2/wSHkkLyrnZICQmKok7c734IATfI8YpRWVd3CloXddu8Wgh9mGBIkCcYp/hD49PnIlB+f79lK6P9Z2pbBtUDqHRDiDcCqoi3/yV6YyN0+eGVOkTPTnuHNmz+g3QorgkP92eGMRKjn/jaMrbDA2AvoSRnNqNXNxJQes/VQeIOC6QQSTGiAZbq9CbVrUwPpJx812ytu67rE/Yozg2MKfbM2bt7huOWUdzEQGpHW8WxRTeD8CxScnD6RYacQi/hK5LFh9IHd1WC5i7XMhLCQOuuQAnjvXdv+cL7HzKGHVMPwNbHuMboHXkRo1wWqikPLz/js+/6LOXb5y84Pn9OytmbIjc37G9v+PS7n1DWhVpmghkvP33J7/kDf4BvfP2O93d73rl9z7mOdaHNDwQqKSUOu70rrIBSleeHG8bdztVJzYgyUOrMh8+esaqxG0dMnxNjpnLgRnZ8++5j7GZPK8on84U//NFXEISX4SXvHm/ZjTvm2qhE7l/dkabMF/fP+Lmf/IN8eJyQtnJZF5bFGw1i7Xqjx2F0Ckxt7HbZA2RThhR748XnnAdxnqqxYcANgqtj6PjjPiR0mMhrYpTEOgw+symvpLwQY5dABtdkV+v8YHA8WANiXXDRIRrZOL3Fuq+BcLLqenHcZzMHx1FNVy7LiWaFuS4sdWFuBTUXQejqo2abKktT2iVwyQtNfEpoCLekEMnJXaXWuF411zlHUky9EehZdSk+ZEzUsKru5tMa1ryLHqJzT3PuY19ipxFF6bOperXX3bu2Zm3rhrxl9QzY5xuZy0SbQ2vDLjJOShoiKecfGqM+F4FyK5uhN1yuHeqndk7Yft4r6zfLc66/Z1N2XnFAFdcuI3TStzdGHNvbmj/G5dz47OMTtRpLaeyPidqUOmWG7POsU4po65rTroQR8fG6rVa0tWtpjyVCCOSo3Z/S8ciUMjm5nZfzw5TWVtSquwBt8k08y5MgxCwQGq16uqpNCNqxT22ONyaB3AnHvVxxmgpYbJ2zhgfV4AGvWaIVI5B95EH1cQYfvv8uH33wLjeHI4fhhvd277AbErF6hixRGHYj62qsstKqEcJAHiJDTizLwuV732UphWfPXtBq470PPySmyOlhoZbCMp94993nfPvXfpmf/t2/i7vvfYMpBbQa88Mj6zShpweaVvLugNTK/f3K+Xx2j8PRpYFpzGCBsgQvNR/u2Q2pd5BhGEb2duQ5SkqJtTR+5VvfQt59zuvHR776hT2jwv54wwK8Xlbso68Qk/BTX/zt/OQHH1LKPXX1jUI7HSvnjBVhHPcd+/Yb1gxSHFCtRElIgloWn/euldYaIW26etwUo4M+PiwOLEPaZWKbr11dyUIeIvtpotmC2UppvqmrBA92GrHi1Ydo8428aDfOcJw8mJATQOPC4kyJGknBIFychG9etBdbKLX0QLOVrd1ToXrQ1QZrKTzIiZzuyL1zPgyNdV4RII+ZJD4xNOXoZhomlNUNKlqnt23nEXNhiE+69FHJnazhm8E2fbQPAfTqSK4V1hOM500e1dCbTEqc3N2/YaQs5Mkxdx8a+IOPz0WghDeoQLwRLMU7m1GcrmoWNqiWzUGIbg3lhD4cqDWcCrP9/s12OT1zvUrQO/8Q4Xyp1E/PlFZoNjo3sVWmMTKmgdRL/9aUWp2oHPqQpS0Y0ufURPHRBSFI3wV9tnKKiSE7ZhNDoJoyl5llXajVhbFqrseVrhTC3OdPpKLRcdBEuHYLVRVEkU2rHtWdWEJ0CpLwpil3N75wF+kYBrR62YMKx8OO9959xjsv3uG423EYjhzyjlILy2Lspj1NG89T5uHuNaoVaW5akFIg58AwjcRhIEY3tLi/P/HJp/+In/jJn6CsK1YHjrdHxmhMGJ9+8h1uxsRnn36P4+0LWl05v37NMj9we9yT8wAh0sw4zzM3tYJE7l5/xjBMrKtvMufzIyJwSZE0ZPb7HWtdKOvCLiWev/sOa1GiOs3mJg8MQ6SeZ8ZpwmLm+UG4CSNlaXz07F3iCLUGquvxCOZd7FJWhnhgviwQG7t9dH7iWpBobuJcW58e6htc7WqxUiqlVFcP5eyaZoRh2DHt99zZgqGkmHxo2T5hMmA6sNYRtQM5wlkStp5pwUvn1sdxmCiteHlqq3owFoduckh9bTaKVmxpWEik6LJbC/5PRT1gqrcgtRn0WVVaXApME/crkMB6Xngtr9jvJnbTjnheEBWsNupaCBK7feDcGQa92dIHAgZz13035fAxJu7zIMTuah47du/BUa7B0eNGlzua0/tT9OcLIqSoSBYfLGPCNESaOs3J/WZ9suoPOz43gfLpAz8dnkniYPOmPrjqF0PvZ2xdYm+ceN/c88rQbdgaT/zFzaOudb7lxmczvBE+zwW5a6SoJALWCrUEdFTG7Lw7X+gFkUzAsywPct1QAw9EKeJEVvHO55AyISVyjGTLTyaxqc+1MTBza36nBfWg3zuX3n9ShiE4l1K9NC+rf8aw0Ylkw38S9HnNEGjWvO3aB0mZNQS36BJzFdI4JW6OE89uJqZpYhgHCI3TPLNe4KMPbnmWn/Hqs1fsdntODyceHu5YlxnTShQ4PLuFGJh2Oz74wpd4dvuM8+mRl59+yvsffYHL6YFpl4itcjtGTsvJ5XrrwqtPP2PaZ9blTCuNHCPpOHA+XzifL8Rhx1qVu9OJWpWH+09Zyuq+oSHQaiHFyKgDMQuvXt4TgvDixQtvBNaV9959wbwsHPc75vXCQykcxpE8HdD7MzsNHI57bm+nPiHQgOjBR6FZJeXBtSYSScPg66pTXrYpiXkYHC9rbqgcQqaszsuttRKi49Yw9SCQ2D+75a58SmkrSiUkGMcAmrE2UdpE0wXVRM4DQ24wBooZWoIPsqPzNNWt5xzYbkiXJrrCy4PUJs6QCpaaU8ay68cRI+Y+mVTpCqbOZ27dkacK0JBmLKZ8/N2PiSHx/Lnr7qfJ59Y7lQ3qvDLPF9fim3W8u6/R0KW1ybX4oXvBBvyxrliLbKY5+O3fM3Ev50PYAubW92hAdYlzFLefi+IQgIivmw6d/bDjcxMot4xya7pcM8ae8W2jWq8lt1qfhbL9FX5yeia6YZeB3j3mKR1/s7zfGkR0xxQM1ktjPhXOoWAaqBpp7UId+m5ca99pu/Fpc/dp21LXjkUSIFqDlBBg6BzNIY9kspdv5pI0MQ/iSylgtZ+HDVvtC7+v6hACeQS1rvix1Am8Tug1cHxmCc7diw0RRdsAK1gyJK2AYlZJROi+fRoqaQpMx+wEcLddYl1W9vlAzplpGEkSeXg88fh4YS2N8+nOTYajcPfJBcmJdbf3saI5MY4Tx2HP8TCxu83ktVLmR051JrSK1kY63FAvK+el0MqZYHB379f8k08+5fHxxIv3PmRpAg8XTCrDfqLcrTQTTo9n9ruJtTQIFXu4cDqdePf99xARHu7uuLt/5NnxltBpTw93r9jFgSCBWhbqciFgfOVLX2TKibvz694UcFhFouO4RZXdOBEle6MvBKRBkEiMI8kM1pXi1jeoKuu6+Lz2Uvoa5Do2xKdXQh0zn1xecy4PSDAfo6pOcSPBMApDE5oF5rXz1UJEpF2bGmmjyqm6gXMDibHzNV2iSx8yR4Am7jgv4mUpgnMcRR1D781DNe2yVSdz16aoRifVN6NdCnf1zv0ISuP25pYyTc7gAKS5McayzKzrTOskfwkbXcoNbVLq42VFey8iXDNFZ720PnfQfL66babTDkfFPjCwqWfFysZf3uhA8n0ZqZl1EccPPj4XgVKgR/Y3sEovPLeWyzV7NLqKoR9P8W6D1js5e4sYge4QveEXG/7XF8RGKVIX36M+V+N8XxlTIUrAbKWWhFanedRafbRrqD7FsfWhXTE+gadCpxU5n5JO4ckpMQ2ZITptAozQAiGa278htOrnwicCro7FdLmWmXWNdiNlnx+T1Kgl+mCuWgg10jSA6ZVuJKHrwKNjt/FKqlJEKkEyDBFSoYWFNATS1MeXzolRBj5670NSFFKO1CoMeeT22Qse7u9BEnnKrKUwDsmzqZR4/emnHJ7fEiOcC7z8uPFjv/unePz4E5IoRYzz4z2lLHzhyz/G5eFjWnObshfHvVvSqXK6XCitcvfwijyMHI/vMq8rKWa/KaJbmiFu7nBZVw43N6QUSZJZl8Z3P/4eQmBMmePNkcfTicv5wn5/4HS+oAKH/Y4Pv/glDjc3PD48ekWjTgFTgTzsqFWZBnfHEfGsCMvOMY2ZOAyOmFQ3yajFeZraGsviEMswZFDp/qfBTYynHbaf+OxXPuPT5WNyypS69tnTjbVWz7AEcs7sdzvKBVa7+DTBFW8k4dlTU3UaWBBiv+JCRK276xhIemosiW0ZVq/Aeqlt6tmjNfG5NRahMzy2QWSOzyasBR7vHvs9pdS698aT794uElhnlnWmWrlSjpqGa//BzS3CtfR29yvr3GbndxrekLDq0twtfrTmxiiururVnEcFr95C6JimN6/YfGfZGDS/8fG5CJSwNVXkWkpLT7X7Or0OFvMmTf//7W96Ge5VtbeORYPbP8lm9fsGBmri82LeDJo9G/WxmsJyrjymmaCRtkLIFc2QxuQ3Y9uwR+9Yh6CIullH22SLPJX3npGkvlv2WcPJJWxCQDKuiDHjoo1atA8P83ev6i4pIo4lOcAf3JC4VkQNUcfJbPEFsWUBJo5vSvKZ5SQHz1Nyfa+KT0f0GSwrSz2h4rzPdakc6jMmG0kJLvMDeoLL+cJyblwuF8pyoawrGiIxZfK4Z397ZK0zu2nk5jjxzvMbavG5Q9/6J7/A1373T/Lx1+/Iy0LNGURYy8I47Tg9XAgpslRF0sD96cRSCiJwvsw8G0fWWnnx7AXr44WGS1v3hz37lHj1+hXHmxseT49MuwOv7l4zDiMvP3vJF7/0RQ43N5zPj9zd3bHWyuTsFESCz6K2wjKf/aZrfSqgeQNnXVdCGBD1axckEVMmxV4dICRAQiJNbuAhAqWAJgXOnZUBtVYOux2d30VOmWbCd7/7PT5bX7IbR59JbpXSimOK5uqY0LXVu3GP7gN1fkAvFautm2ZE6MyEKD5ONiDuLWq+1jQ6Xc2hJ+tD7nzdmXTUTt0M2apA2/iboVvb9QAl+BpmM8/2wWP3DycfGSIB1zd5dVSqbxaldF5mZ6JIdCqZYORuo7b1b+lZo4jPuW/4FEbDsFLInd/bmnbhk10Dujd1eq4EXQMuncGySVy+H/b79cfnIlAarpYTvGS17m/nnWq8LO4fGL4/iwxvlNN+It2bUKzbjFWB2OdXtz7TOxjxWtnLNSAb7iCDeoC5PC5QEsPoVlbLAGN3F4pJsBjQ3qlEvJscJHRn7Y0u5IF8G7HaNlghgiRv8Hg26gF2LUopRisLVo22OseOZCQzcsZNLnoX39U2QGjQAlIch6m0fn5SH3GBlzg5EDK+mVgjZENDoEplLzCmgaVeeCiP5BaQy8DUEh89+wLLcuHu9WvODzOmxuW8sFQF89nZ61pcR27GeJz42te+ymm+cLrcM2VlGjOmjcvjI7/6j/9LvvjRR3zj5SskQl0q3/7lr/OFL/0EeTySZOayFm5vbpkvFx7u70kpkqcbct65XVoIVHFj2mU1YkzklFxq15t/FjKn+fHqzH28uWFZVy6XlcvlwrysDPOFPHac0Yz54TVprNQmXE5nSoMYM6W1LmhQuoMwpuqNGQkMEn0+eA2knpGlceczeB4fKHVlmiZKWSilkvpQui2T2h/3PMwz//RXv85DOHOc9gxDQoPSpEGsPVFoHhDMP2fKif3hAPOFtXpnnRBQGimoa7pj6jQaetDw6qrUejUWkR5Yr3QQ6079dF11GGnRpbjWmy4KmHkzis4JHcaRJIm2NC7t3Nki1t3wXWShbfWNaAuSmBuKdAehaE710yZPHfHeiCzNaVU0p9JZcNegEOJVZolJ90HoDZ7wJDzZpgaEXno3g6tq7wccn4tA+dTDsSccMmw0Aa5YHTwFSf9/ewp04gvz6ojenXqE4N036Q7LfUjTU4nvG4t2rqZjMB7o1nOFpaGjEAahTpHatol+QkrRO2edhuMUNcdDe656fS1B0LVQYkSHkW0wu+/4rpQYUmIcsg9mXwrV8EmHa/PhUa3B5JpeQus7rjsKBdzoleY40nV3F38nrfsPps6581EYXb4ZutojhM6FK9T1zHmJ7NbAFCZ2u8GtvqoPCHu8P3F5fKSpsM4rBA92ghEPI9MALz/7Di8++IDnX/gK3/jVXyJoYcwjQ46cv/US1oX98yOvPv4e0YTz6ZFa3mddL+T9SIxwuVw4X06cLg9Mw540utQs5cyyePe4VWW/myjrjKUD0zQyThNNhcvpRIiRu4cT4+FIw1VF96cHLpeLyybnGQNubvcM48g6n9F15f7+sc9VGil1gZiRmJGQMCKtLaBOWg4aWUWdZiLNJ2RilEsloAxjptmA+5GuPnkwJZ/7Y8Jx2pFvjnz74+/wa1//FnOq7KeBYRpIu0ALBcnKMGRM3NS5FHfosZquKjHv8SlWimOMORIFBhpDjEjo61KFxXzCp5tuWKfXdeFE7FxjMwKJKNHxvubjwDyHdIceNZcaB/GseMw7hpzZnLq026OVpsQYfdBXKz2bhBjEvbLVMdEcw7UKtF5NWQ9um0ZcPX13Q2U8CxZ9SqQ82QLwmVViDjtscuhtzjviQfDXN5J//fG5CJQON3T8rXdiPN3eiOfyRsP7+1HXa+C0vhO23sGWLnd0t15Cn8ToZrUC5o4u10DW8UrtQdK7l27QK00ILdJUaK2gY8A00apeXUvcBcOzDX/9Ldt9ajSpVeqyUJJPpJM8uK09YAQ3QU3d0SVFijhJqJVGWwwpFdPAMEQs+mbgnE5vBCndu1N7QmB0w4qGg7XRGzndF1AkO4lXcElmFJpIJwKfWOvAM77Is+EZy+XEcTwwTRPW4OHugceHOz8PbQPT/d98mWmlcby54f7Vp6zzyLvvvcfLj7/D6fzImiLWVr7+9V/iqz/2NUJM6FoYdwOfvvwe7/3/qPuzWOvWNb8P+j1vN8aYzWq+Zu/97X2aXadcVe4oOw62kVAagRQJ42DlJsANBCJ8QwRCCBFyA8hCygWNMBGR7Cg4uSCBG4QNRkFxoyDbIY7LdpXLrnLVOXXO3me3X7e6OecY420eLp53rr3Ldp1ysCydmkdb5/vWt9Zca805xvu+z/P8/7//k/esAd+tm+tsaLGHw8zmAuZ1YZgm7m7v8VSW45FxHIg+cDgeO5cQTvNsgvtiHajL62cspTKkZKdshNoKtQa0GSlKxVEqzMd7Ss3ElDitR5y3QDhEWddszv1abHiDUKqlZW42G9ANrjWa2AmtLguiSgwe5wKtQXCRMW1MkdEq45jwPvD29oY3n9/x0I6E5ImjZ9hF/CRIqoxTZNhYLlGrfQOUwBgSqxQDP2vPjxFlwCJhg1Oi78YEHK16SjXwRa09+piKL9aWkt4/DdYTwpiqNjmX8+QYDJumJg97dJ01DJqivcLrapTWoGbz0Oc+EBUxjoEGQTB3jFbrM9delkuvnvTMagWoWIaUk/7cdv+7DpP56kCldpCIjkjoUdKYxbEfmuhyox/1+PFYKPvP+JUlz+Q+vaNop8qvTaq/KrPPA6D2lfRB+8CmyePgp1eqhrDq65nv9PRzyJJlh3RhUTfnK7bQuCpoPg+DGlRbvEKAOPRBQp8aCmJCbzXQhfSivtZmYFTfjMzSp529X20LpjiiiwQp1oOiC+hLI+eKKw2p9BMulpgogVqlZ+toZxbyOAHka9xMFx1xVAMKxIAPaih931lvvf+UG6wlM86NTZx4/8k7zG/eMrcHpk2i1WpDEu85LdVSDp10KLENtUqpzPOCd5W3r77klAJTGjnllcPhnhQiqo0ffPR9PvzwJ5lLYX9xzbwq6iP3t2+IrpNdJOLc0E/wRvdZF5tClw47npeFcRg4nRZ8CqzzyrLMNOdNqyiROEyoFh4OJ05LodRC00IIjmWZLRKkT1gJnug9x/t71Hn8FHDSyHVlGCZwieVgva91tmHLMCbycaUcFkAZphHvheSNx9iq8nA40BqM0w4FlnnmyZOnTMOApMR3v/icL948GEIvZFx0jIeVca/4TaWUCB7LwXE2HHQyIq2x30Z08by9v2OtDYmYqiEEfKh9ECQEP9BEqOJNY+mwvncnGql0uU6XGgXpvAMS1ZkDp0mjnOOkRawHKSYGb73PKX1+YPxSc+LU2robp0Avl63lQY+V6AcasA2/2nT9rAwQuoYSHnOMVM7ayj40ehwF238iFjvtxGR7MUVwjnOH0kwivxUWyv6wfoG9qE6MmdjOvr4zFJSvSm0eZ+LWl3sUFQmPn6fV+jGPRGS1dl5TA2pYT9cWYC/yyLaU2nCt9d6m9Uxb8bjme/iXUqNSq8MHtaAo5yzhselXJ+A+XG5NodOha2k998ZgxVW19zjVpsrOBOv+TH3uJ99Wo+GhaF+b5Ncey2B/Fd/rbTzeKRqUEJQ4BHwUfDR3gzjBScO7SApWf5wnoGtpLKsne2szOK2kMbGeDrgGUTzjkAibDb6dWI/mAQ7eE6eE7xTtXAs1r0SnHA93tDLy7OlTXr363E7mrRIIvPzyJc+eP+Nwmnn3+XuUdWWtFSfe+p5qFJkYzT99eLg1+UrwaF2oFIIX1nXh9uGOi4s94hIlN4iwHfcgkMtKWWce7m+Y5zvKPHOx21kUwhjY7C/I6wpa8CFQcyF6qK4T0FXZXexxbuC0KmHYcmZXLsvCECcbMNSFkk8c7xa8CyxOSNHzcDhSqzBNe6ZpZFlPUJWL/YYmHjcM/PL3fpm6rPg4IE0t7bBCXRpEYzXOywHCHu88Q/BEhOIcQ3Bsxw3zurIeD9TcOByUaevYojgNFkkrDQkFh+IbRujqrgRV7QWqLT5BG8ENODV9Y1WrSlQbQQ25RrV43/qoRbSMG4uvsJNA66qSula0KmhAnBJ97GMg98hZRXsNJL5XW0LrUSa2kNJD0uyid30Yq016legeJUUgOPUsx4Xbo2302+34eL9+ZZb/LbRQPk6w+arf8vjnPozosJPHRUi+9gs+lu5f+zr7OI9l3NnTI63zHeVrb24HCPQOdR/IaPeaW9O3zg1twWJgmzdicgBtzrRfsb9ZX/teXxGNzH5Ys0mQWq604Lvf+xwTGwkuk3xEk/l419Gm4DnzCPeV5vqgqB+XfU/FC/QelC2kEoU4euLgiMkjvhOInBKiJw4WHG8ecXDBNqh1LZw4cZPveNjc24bRYzoMte+/grOuJ6r3lLwyhkSrK9RMyVDXBScr2hrrcuTly8xud8Hh8ECQym6wBWE9zaQ4Ms8HUtjg/YBEh+aViuJipDbl4XiLqsltNrtLWi04Ee7v7zgeZ0KwxVXVwt/SuAVx5Lrgm5W59zevyaeVmAYQj/OBOIxUVXwMnA4nlmUmxcC02zHnQkgDadyizeyqXgM+bXASUBzT1KBmalntfVZzSqW4oWnjcJhZ58Kw2RN8JK8r82k2731wrAp3Dzf89b/787jRkYL13yzGolh7pQRahWVdqc6GJC5BVOubOu8IIRCc5c6UbJCJ00NhH6LBUVwHPgfF07PNq9psKpwDuTpztVkWd5RAINDE4dX33rdZOSl9sFi1zxOMkVmb0bdKs0RRsxNbkJgTY6QGZ2Fntmn7R6NEe7y5OwjDy+OA9yz/+aqatDXj64YVRYxl0Ft4eV44PpzIayH6DW2yaA5aM8mR47eO4Nx9/TSHUae9Ch5Hc/LI0QO6SLT14zVf2wz6C/XYGDw/+km0C4b1vAT3mY/2SpkGUv0jqj63iqPLKVQxc6ujUaAYbaXGvgDR+3xA8z2y4ixB6nRrg/H2AUxzJj1Ze0yv75kqKgRJTElJEgjOsy6Z02wB7i0bx5LFsGo+QEg29HId+6Whn2hR3CD4pLioBnJ19nv4oAxDZEjOAp4iqDdNoCDknFklWKl8PHKx3VJLpZQV7yEmu8GdKDEG3GZC3IBSGZzFy44EVGBdVrwzRUFdKl8+3HN1/YTBXyK18eTqitOykltmjTAfMldPv8GS78nLgXGzAedZ5oXoA6/f3gITTQMX+y3rqZDbTNOKSLAcnWo5O9EHjscHamtMMXE6PvD65SvGcWAMW+K4Q4KnVNCcaevKOi9M40gpldIcDSEOw+P7tL24RCWwzIX5aIFlZSm0stoGTKbmE87BcsysOaMVtps9ToT5dCSvJ0peuXx+ZfIY7/nbv/SLvDrdkjYDgzrU92iP4FGxKX4rRpOv60xzwqkom7jF14ALnhSF6BOeYBALEU53ldlHZA+40qOZKyEIBSujm7PefgjBhnyqRLHFLJ2n2j2XRAER37kBpoGWZlVga93GqUJpQik93bOZ00e84NJZvtbvq9ZbVj3z5vGAIga6MJ+39cJKNeddU6H08t1mGOcSvtucWx9OeU8aYm/LjaQUGYahnya/Uoao/BbhUda+k52xaCjdtmiHu8co2y6zOYtPzwfH9rVx+Ncn/apf3ynEXAoiFmqEvfe984k2G/LwGL8J51Az6c+lrYcnef8YJVux01bFPMhnmdBZxNu0fVX296lra4XWAiWb77VVS+KD3FFSPHpQpykzHTP1tJLPYUjeTowuRkLQfiP16X90IDbjc8HjvFqvtp8+nffEwYASYQq4VPtF2sAtOCbaCoNMDDFxOMxQFa8rd3d39poopDhQxoJfZ1QzQ4oM44CTSskZrSc7XUrpk9GGuMhm3HA43BF2F1xcXnJ/OHFxeYX3gXEayD5TyoEYR5qfiOoZn17w5as3DBcT4zI/5hBJGkkoD3czAGteWeYH9pdXbHZ7tDbuHx4oesLvnvNwd0+rmc3mGSEmqhYudlc8nI6ICLla2T2fjoBjLitps7GSXGx493BQcAnvEy40YgWSZ6l0/ma2E1S209RSVsZpT1UotTCfTpAXwnZgM06gMDy55tNf+DlagETDd0spmB+5qi0UZ5IWUml4y9dxJwbz4yAUBudJLnLSTChCXhpvjxkJiUEUcYXBRfBqE+Z+KAgS8FUNOKGCdxEnEcUALjb76G6dSoey2KRZeh9TaKwl0wqAOc1qsUGMiJp/3bmvEfylV3v2e4l42/Ca4rwSfCX4gLdSqbfk5HEY12rjHNmsagcGofc7e9kvHob9xrLXfL+n1T3mGZmE6LcAZu3vfViLUc4t3S5RsPLU+hO9/9An1ucF4tcvil97vi76Pj/X2R74+DHOz2NPK48NYfuEpuefo5f0ZwK4gjqlFaOPm8i34b0+9hBtGGdvlkVIWO+xqYWRieswAGAYIk0tY9z5YE3yITBOG4ZxYYmFUmpn8YHEM5/PNo3zRfvVBSOP7A9rqvepZP+dLL1SLSrAC+KSvQ6lop1W5CSzzjNOC/l0pPZcz2XJoJkgle0UqD4gUkELzvuuEbSFo7XKkCaaLGbXdI4xbfDe83Ccub5+wsPxnouLS1KY8BJYS2YtrQ+pTL94/ewpIUbG3RHfKstyJPjnZIHTsjIMEY9Nu3eXFlH6+vYt891rtpuBZbINMMQRlUhII7urS+ZVgMi6LoAj54VaM7VBGnbULBwfbnr8QGKzf84Q9qbna42yrpweTpwOx14NVaQVtBTjIZaMj4Vx4/DJQxZCmhjGgWGcCJstjCM385Hryz0P8y3SeCxBne+tJ9oj3ESb2tTaCffLHU0crkWURhyU7RQ4ZQfFQRPmU+UQKwSP8wKLEgaHC0IIgkh4VGdY674PTB2UrgJx1doyNEdpjdIKpWtW0Z7M+ShTan0RMwmcpRH03nyHizh6FLOXxwn4+fwjYpNpH4JRf0zThlRzCkl3uxU5vzhmFdXewhL6pP5xqOMfy/OzrLCdubBqB6gf9fixWCjPC9LZJWOTtH5B6Fkr1D9X+PsWxq9Th+DXnyLNWtXL7D4AqmdmpcpjDvE5RkJVzJCvdgK0VbTP4boeUrS7EApkzG/rxdFzktBcIQJOu87RdrvzLmfkFOtX5lYppeKcEaxVPdoC8hhAxSOabRgtv6RosYn6OTbaw1lkVErtN5JdPCqhazDtgpDug8Uroaw4iaQUcN7Kv9YmVCrzfMerXNnnLfu0pbSIlHMfFUJQkkJ0Cb8JlOIxSnVmXU/91Nyn43FgM+5ourHy05krKPnAejry8uUXPHnyxHJNpspmsyc/3CE0luXEGD3alP3+grUql1fPuH/7GcvyQFlm8poZxpF1ORLGAfEJFxKtNF5/+QnTmBimJ2wv9szHO9K0YbO/ZBy3zHNhWRYcmRCE5XRE1SydLnjWWpEy4xAGvyPETW9NHNHmWY4zN29vqaW7r6rBcV0rBiFxyVL+hogLsJSVuBnYXuygNUJI+GniZp4pY+Dq+pIwC3k5subV7LG+o1YERO117qgBSiuc3InkEsFN+ARJhF1zzNmxnM4ll1Aq5KKE5s1p40w0fqbeW462DQHEdWhGq7i2ml602QHFsrrV6OG1dLGzGlH8HCmrdk/ghLWsoNVSE3vshao3/TKtpwU0apex2W3cW1IOYhSQ3LkFvaIsQHG4FromGYMai/VGtdtOvTNTh++/q5Ha+6rj6NEmA6rrj1yjftOFUkT+beAPA1+q6u/uH3sC/F+AD4HvA/+iqr4VW4n+98AfAo7Av6SqP/ebfQ/DUNkPb1599yjItt3K+g5nAem5nylf0z49lsb2hP3F/nuIRJ3G0wfR535xt2V1XNl5QVVQDSC1+7kxaVDvM9v3qgQ1YkuWBrmBeEpqX1sY7UJ3/eUOEnoDnY6+l8ckOm2O2E3+Z2OR9SUa0XvGcaDaNWO7fbXnqU3xYvImr74TaXpD3VWkmL8X6dk6YoJePzrWAoOeA8lswS3Y73FsK6+XtzwsB8bxktZWAo4QI8M4MYy2V+d8xOmIamHNK/MSWeaZUjLTNCJizM0YHMKCNiPxxHHHZromr1umaWSadvhhy1pnxugNiUZgWVd242RQEScM/pK7N58hTmjOcZzNG74uJ7RUnj97gbbMF19+QkR578W3uXz6jCF6Ljc7dtuhz+sa85IJTliXhfW0WsDXkChLo6zHxwGhC4nQlOP9gY0Gpk2wqbp4rp885/b+nuQDp5IpeaUUpfk+YcYqmHlduHxyZbEc4tnuJvbvXHOP8Ne+97f44vQ5DIFJEyk6/JqpXRomraCtdL1vp+wreNeoWji1hQufCKkPEwtcbB13TTgVCzAr6mmYU0tCP/mVRnTC2c8uIpTWkFIgW+kaeolrOVDOrvFWWOtq1YsTnLZ+yFDLtVfrK2rH8mufLDsXUDX9pnDWSNqJvvYDyzlqxIupM+JgCx2u4p3JkLR55lNDW3q8x723gVDrIX7eO0KIfS3o60eLNuxx4fHrWqXb237jxz/MifJPAf8G8O9+7WP/KvDnVPVfF5F/tf/9fwr8V4Cf6v/9QeDf7P//mz4ez4ANvlLP//pHX/v6v50nYzxO3ez1+IpCdF4jH0+av27oo7/+e/SntAX33LLuzXSabZodVKpnOK+Yicf+bIQZlx2l6zW9s/6g641qd57UtT69bnaz1tLIRSnrwpD08bQraovpOS3S+UAI1cqdrimTAiqO6kGKR0vXnPWOlUWAKuozDTHQr1hGehqEPDbmVBk1YB0NoWa7CQuVFUuQXNaF/Thi9BnPEBP0E6S40RBkqrS1PZY+m2lLrYXt7gLVxulwIqUNm80GrZUYIus6E7wn+IEX73+DU1XW0wOn+7fknBmGkVyEJoKLjkRgXRre28RUVUhxw/F05DjPbFJkmiZu7t5yODzw9Mk7XD95yv5izzrPTOPI2rylKCiMUTjd31BrxccRRJhPC63M5DLjGQhhQ/AD69pIw0RKiZor4Bmniflk6LB33n0X7u843d1ZTswQmeeZIJFlXolppOYVJ4HtuGWMHhkSv/LlJ/zcp7/Ml/Nr3OipRHwz6c+6VKqu5mdwJiQnOGqP8fBq5XAtleJWUvJEF3DVsa2eosr6ICb6dgZaKc3hMUeXYug0B/1AUAyGUc4ynmoqgyqUtVFLp3Fpo7ZsMbPOPQrPbYhii6Jd4zapFpGuDDZRSW4ZdY3oLCKafr95a+QTQmLwjmFypFEMWZiSVWQEWhMzerTYB0vn1plpK30IpGi21dY/bvlZ5ym7bTZNTeEx5F83/f37Hr/pQqmq/5GIfPj3fPiPAP9s//O/A/xFbKH8I8C/q7YC/cciciUiL1T1s9/s+/Ta+KuT4LkgVxOWNrUjuUofTPQJnJXLfXE8S2LOUgExp4jpL/3XPte+Xf/9gMd5ee9R9tOV59HW12hYWq2FFGk/viuCD9KRk3bBNoUQoHnBVYGkBJyRhNTKG2nn0DCbVJdcyFqp1eALTsyvntds6XfaJ3u9Aa7nDq46tIBm7RPRr8RV0v2fTdQmm73EMb+wMoyeOVUkVFBH6BgurQYTSC6yC5cMaSK4QGuN/XZHQCi5W9Ca9Vej9yzr8thw3263fYMaWObCxX7PZtzTmm0Ml7sLQgy8/+IDXr16yeF4y5ubV7z/rd/GjQrH+ztznWhlnPZIDLgQGOPUxduNq8v3AMcwem7vKzFt2F9dElMkBM8wbXn+4gN2F5eE6GnFU6PHZ7VecGu8efMF282G6yeX3N2dWE4HfMd7hTCS3IbSKofjPeO0QYkcj0fGaU9pK/lYmOfCuy+ecf38iuPDHSFGaqk83N5CK/i9vfc1r9y+Xbl+9xnpYs9+OxJC5OXxhtf5lhIqKThaHChlsQwYp7gYcO18mCic3RDSwRieRkIIVIKHYMUDQ3Ok3Aiz9laQuYVyVXwRS+REjF8ZbaFzeFotRtNXy6qhVmpWylJYs9rpszfzXRf0451ZbDESj6rpj+1A3ojBooG1E628t/smDZGYjEYvKj09NBCDY0yRYWjEVEgR4qD4brBozbSX2sKjRvocJx1cJISzThKqmF7Zd/lUSuGxhVdrI00R/cc0zHn3a4vf58C7/c8fAB9/7fN+2D/29y2UIvJHgT8K4JN7XLBsV/iqs3o2zZvFClv8sN3VtOg9y6X3HGx47mznFYAOlaD1MDrrRX416z53Qx5n270EpQdHKS70U6FTqoOl2nQ7+EAaBB+qmfnFejXr0mnn9Elcg9JLe+cC4HsfxU6awTky1SQ5eUaDfe9SM+u6ktcFrbY6P7p1umvCnhwontZ1lqZCcqYaaIZks9K7PZ6Gs8LxDs7INq2elDoktUWjvnvHkCam6YIpRobaUWHVBO9nnkMrjTWvFlwFDMPAMIyd8dhwu36TlJUnV9fgPGmcmKaJ5h1P3n3B65df8uXLl+BGxmns7p5CzTPNRVLPKxJRlnVhd/mM3XZD1WILV9pQc0GdkXbEeZ4+e5fdxSVpHO00Vd7wcLwBHVGvLOuRqydPmDYX3N68RbUwjAOqEakDpVYe5iNxCIzjBoc3qVOCeXZo19HGFLi43nPz9ob1YLzJ1jKtnCw7qTTWeUaaw0+R5XBgGrdM77/HfPMlX5y+oGrGYxNmqIiGvvDYReq9ibfFBVSz9QzF3ENSC2dXTfSeWAvqCz4KzlfEZzQENMAqimJyHlo1bLN20pXr/fQurbG2D9TVILytnAPSulOu9QGm1c+2cFHxMaLVCO2lVLvm1CoxiVZKT5vJBO0JUoQUzNaJGELOpGc20HFBLcPbKSLZeprqqNpo6gka+uFGuuddrLn5Na3luQr1HSKj0NF5tk64r7fo/gGPf+RhjqqqyKNq+z/L1/0J4E8ApE3s6m99XGBswXI8PrXrmCVsoZOuPrehsjzGHCjY18jZ98nj4Ef7Inh+GG7qq9PZ44QYI7vgHM7nx2GJSOcBrh7j8TZi8sR0boo2VKuVLc5+PvF2QZVm2R3a7GI3pZP9PM45UkomEl4Lx+PJdvOaWbJRVnwvnVq1kqitNn0UhNKKcQhbtd9EMFiv74BS76x8rY5WTGArFZZ742lSFXKhjkaXSd4zDROTRt7bPuN6ukRqQdvMkhdSENKQDF5ctFvI7IUNITB04reTSAgmc9nsd+w2O46nI5txY3EBIdJU2e83bLd7PvvsUz799AvWNvPkckdRZWlKy8rmItFUKVVZ1syw3RPHyPHtG0QMtJDSSPQTp9PM1dUTSm34FKmt4tVe+3OU6lILu+0FtSqn3Jh2ew43NxzmhSElltUm3yklQgjkdbHpq08GKmkr4h0xBkKAeTmQl4Wbt684zTN5PeG0MU7X1pZZV9QPOIU8n0jbEX/xDJaTwS1EKfkMYv4KRKtqE1/FLLB2mdtA0iO4JnixGF+rmSpGDO8KiP51rl8X2nOTqjZK6xbCviBKPR8SAqJieeEZK7er/XfutktrZg11Aae22Fo2lf27j6l3Nxe8dKyZb4QxMm4S4xiI0WR6IQheKt5b4mnwzsTomKnCqYWFoUZ0cZ2BYKL6/DgEPi+Ipa4WIZV6iF+z9pFUaC2YxK5/bq3Fbvrwj2eh/OJcUovIC+DL/vFPgG9+7fO+0T/2mzwspdDqCSuXnesWw3MZ3DcJawz2kww8esLPC6ogJlDtZbPBhGzAUavRb+DcMLYTl2CLLZiJP0bPNE24YFksSsM7oRGQXCmzsGrFhUoazPUiYifcpRZz7DQ7aTnvrW9SK7VUxtH1E2gEjJ8XwmADkpRYTgs3b++ZT6sFQ9WMSoNo0oil9kHB3H3d+tUJWfsGYpipRgjCOAph8MyrkjNQlZxtFy0HZcFu4roE0tYTo9KSY3DCe5fv8GLznAjMywN1PdpFoxEfHMkFTg/2MW0dsXUG0voBi+MVnKg5UQbl8uk7DMGjrbKsM29fv+T7373lar/j6sk7TJsn/M2/89f5wQ++yzc/+CbiButj+MC8NkKfzg9DxEng6uKazz/7Ic7DNO54ev0ub29ekoYBX7sVVgEn1OaJwx5aZYojMU5UETbDyP3b19SyEskcbu9Rp6QYCSHaACEJtWXmZUbWSBgmNrsLy1D3Ank1X3ir1LxQl5lxt6c0oS0HxsHTnBCGxNWTJwzjDkkjNSS0Cg8P9xzXmXmx/KSWa6fsFKKzjjOirFqg21/9UpDRohGCWPqnLZTFrhlnEi3FfN7nU6ddK0qu8tWgsTp86zdZw3gG1a6xWruUptsCRYQhDmY0kC4S56yXs0lyCg5HI4jQujQnDAPNK2GA6PvpN9ra6rzBjuXc66yWHSVOqE4QXzpQTOygoRCdJ3cq0Vkt01pDQmPVRqqx991NmG78h4arXZPpbOBp2MJ/PAvlnwb+28C/3v////61j/8rIvLvY0Oc23+Y/qRNtr/K3LY+o10Yj9KaRy2jyWZqO58srSfSsBOcc+CjY5yMRlKbdN2VA61UZ1NIuhTTe/t4UxsYxBRJCdIAafS0NrDkbKAAKl4aYfDk1eQVIRlkoonaabF4cJXWPOqNtu60T30k97jYXkJhmsohOoaUUBlZpoFCYy0r85IpLZvzJxSjtqgl0S6lY9cE6JAAu5ABGt4pm62wuTAg77BNlOw4OkN/rXOG6iiYf3ZtjbYqZRI2l549ntCUNWdWb64XbaUH2w+cDstjBKv1US0LPLgRn86IN2vy0xrzw4z3DzzUlS8PB96+ecOrL7/kzc0b1rziinJxccG3vvMdfua3/Q7+yn9yx/c+/pSf+PbPINOeEgJ1ntkEE1bHFJAKczYCfIqhDyoKm2lijIkSTZ517l81GVjzkVoXhnHHNG643F+Ss8V8HL1nnRd8qIzTlhRHcjbOZZ6zTV6xPGkfEoKz02BtaF1Zjw/gYBgCISSm7RWH9USIns1+T5xGhs2Gp8+uWJcbUi2spfHm9Q13b+85rQvrPLMuC7VYL1IECoIGR5O1D1YMSrvWQoqewZvkDK04IDhPFes5S1RIkeYwD3e3NauzDbWtjeaC0b+R7rdQtHlEq7WH8J3d2DO0e4yCk2D3qp4NDXb5xSTWKmmNIfXPHSI4YWmrDVZao2VAAmFIRIOk9v65xfCaeMPSMxuR6kwFca6ocvasWbg/nfoc43zCxDK7neu9SsPAgeWmxxTwwRH7vUjLRqn/EY9/GHnQv4cNbp6JyA+B/3lfIP+vIvIvAz8A/sX+6X8Wkwb9KiYP+u/8Zs8P50mznr9fx5PZ388CVM7lcV849XEE3j3VfWH1ITBtAymBYBnYea2PMQuus/3Fw9mTa0d3O/mF5Bh3jjRULFlAWG9NZG07j+Kj4oP1Qix/o1r/UlqnCME5sQ4ailn9vG9A7uWTkNKGFCJjjEzTSEzW04nBQSu89ZnjQSEU3NDQ3AgNVB1hhLraFvHVxdrfeAdpK2yeBoZNIcZADBOH2wKrUHo28jwvOIXJj3Y6ADZ+4mq45J3xGdfjE5TGaT0g60J0yfzRNNacKfOp29aszAt9wuglmGatrASnlGZyjMP9gbu7Izd3B968uuGjH97y2Re3zHkFcVzuVz55feDlq3t+18/8LH/1b/x1vnj9wIvtFXJaeTZtOS4ny/LWgI9wpBCSYz40njzZMw6RuN8w7ffMuZBiQmLkcJg5nBZOOZOXmf3VE6aLHUqjrCfmwz05z+z3O0IIFBVwkeRt8ife2g1rj6prJVOW5XE42GpDXOHpO1fc393i3AUlW8b21dNrSmtcXewZdyOnUhgfDuj6QBp3LGvmdLzneFxZTittzZRSCEEI0SHBYovPbZwzJ7IKrKWw8YOdInGmJewHCIdRwH08awZt+q29f39OFNDWKLVrGJ3JfywCopm1E48TMxS4fip1HS147v1Z4Ffrk3khJjuZetfbE8mztkxez9k71RiU/YQafLBExtKsYqqOVsxMIa0xjgq1sEozNkBuPKyBu4fC3cEgKoINfoNLBBfs1FwypbUeD2NcgCEFNpuRlIIlYKriwo9eCv9hpt7/zd/gn/7L/4DPVeC//5s959/3dX16bY/zpEz6DmWny68yvr+ebtGPzGr/4h2kFBinxDDYpLs1QdzKuq40VWLy5oTpK7DBftVkMV1fNmwdw9AI0aCih6NSik3Pncc+Hs/DIsA32/mcI/vWQRumZZRzhGYwWorzxv5x0khJGKK3iM9pYkhW5u13O3wEGRv5yy+puuBGIAnRm1tBFaiGRat0X7mYNMMn2F17tk9gswkkn9AcyHOljFZylCaMLjKNic20IabRXEatcHN7z6cNyrrSlgPvXT1jQ0IxfNqSV06HI+RMa4XNZoNzdqKrZcVVR/N0CU+gec9ShZv7zP0p88Mv3vDZ51/ihpHdB99CDw+8evuK129O/PDlAy/fLry9P/Htn/gWt/NKGiaaVgqBdS5s04Dr4VdOhBQCLRjwOKZIGBI+DeyHDY2G+khtKxVhXjPjMJKmCfGO+eHAepoJLvDsnecUK2QZxHE4HsAZ2GMaJhsWBiHPJ3zJ1HrCe2857SGxu9hyOi08fXrF61cmhN/uJlwM7Lcju8utXb3OMw4jaOH164+4uXvL8lBZjwt5zdYDrRkvyby7UXsPshPHxbgB6hyZxqoFXxvqPC4kfAXnKzS1KXgIZlIJalGw3TkmfdhYtZk4nPMgp1hPsrtsHF1m2DcM77w9J1bOf5VkaPALi3IuDPHcBqsIjeQhOxO+24nPepdn1YdoRKqH7EwFki0sLy/WzhgHiD7TamOtjVoSpRSWJZPLuTo0HXCT3svUhObC8XBgWWcUZZw8h3FmHJP9HL5HZ/yIx4+FM+f8+Io599XiaBfGV4wg1a9m1q2/UX3UgvdKGjJxSKTRg7PeZ1axKSE2EHEu8FVQl036SlV8UYOcDlZeS8iW+zuYC0dQnFRCMjeDOOlZyGLkFpdoLTBuLVXQNd+n7EBoSNLuLHDUOlPKQEpm1B/HgXEYiTGxE9uti8vMes/DycCxHgjJUY62i4fBGJO19Wa7s+xoibC5hM2VZzMIrtopIiQIo0MdJFFCC1xeXpNSIrlADJ7sCriFsIlsNlucjxyXSvIN7ypLzizHE2XNaFvQUtBjY5q2HO9vqfXI5CJBLNDMp8QwPuHq+kNqgF/6/i/w5c2Jb/323wMevvGND6ka+ct/5S/x+acf40vl73zyOTeHE9vLK6bLPSrCO8/e53h4Q1UbtjWiZaU4zyErm2Fkf3lh5b9P1JoZtluKh5oD4hpzucPHgWfvPAM/cfswc7G7oKlwsX9KySe0mcSkdFnWupxMtKwWTTsODqGidbVYDSznOyRDz13sR968OVLazDiO4CvDFHlyvSd5RbxjM3hacjTxfPe7f5dXb15zuL9jOWRqboTg2Y479ruNTevdQnE9HtnNLFJxJFQKuSm1RUQ9RQ1YHKMQi7IWM2oE51hd7eYH+sLQbX4V5OyV7veXrtbPL80OMOLO0/eCw1w82qmRwdvybVWFoxRFKSzryQDU3nqKeu5V9jZaKza/derxxeNbwLmAb4I0MfwgjWVtUBuiBe9bn4xbeyv4TBSFutCyQB27jrIQvMnsvEZG5xguB3JV7h8OLMuRXCrLks3C6Xt+1I94/FgslAbPLKYT/DpLzp3Duc5vYp/5qS029Bm4YDtUHBxpbISxETcdMFqFXBw5O6RVNArtfOTHgLtNrKFci7fdTwrOh04vh80mQnEW3NS1XjEJ0kv1NHpEmsGAUXjiWQ5KWwAqIQZSdKRkEqNWLRfcHBYDZVu7visyTAknwr5d8LRk7pcbXDzRqml//CiUIAag2ArD1pOrMC+9UxtMgBtSZpgU8ea3ltCI20jSRvPVOJMSGXeB7WTwixgCpzYzrzPNFQiNEAO+5wJZGJSBcptW21zEk3Oh6R3zcqDmmTUfEYFxSmxD5PLZe+ze+wnefPyGkwTe+eY3ubs98sXbW/7wf+2/Tq6N/+d/+Bf57O2B623gG9/5Dh/9yt/lV37tE37f7/29pJhMFaBbDqdbcl7Y7jzL4cS6LIQYGEIkpcg0bSmtWgbOtIEWrORMmSfvPmP0z6l55f7+BCiHuwfGwahHU5zAKTFESjgRfGA5PvSMm2x+5SakIVDXiqiSxpE4TGy2W1KacOIJ45Y4TtzfPBDjwJMne0vZbJnYkW7j1TXL/S2//Ct/l/ubO8hKaKZjncLAdtixHXZspoFM42Z9yeqOaBEbkPURtu/FS3AQxAhNpm0UgmrXKxrh3Prj0gfT1u82T7T9vXYroAo9E8ch9DwatY3YKFVmwggx4sXuXSetn1Gh5kZeFJ3OmmYzO2g/qLTVZg2tTxdyy7boOkhhIoWISAQO5JKptXYqeib4AjSqrlT1NBR0YZlbXxE8w5CY4kT0sUdLDDjnSWHAXQq3p3vePLzlVOyE6STy1VHsH/z4sVgonRdi8hjKz0547txDeZzvnH8RO08ap+Ss52l9YbP+YUiNkCzRzUjk/YJJlpRXvZXF5xLf9Z6LasT52v2fgo/RAA6TUNbCeuoaLhQ/tMeS5LwjBS/kYOJuEWF15hiKgxCDTfqcWOZIycrx7gBly34zUy9XQtqaHtMJ4zSx2ezY7besmii59p3VFkm5qAxjJI6BJpFxscGCiwlU8H4mDrOFzjulUdEYqCFT40wYIjE6QlJ2+8ko3F5oPRArBEfrgy87TUHLhVxW1mXGK9ZXao1aVpaHoyXhsQFvofNpc8WTF9/m4r2fYPFbvv/9v4kEz2Y38tHHH/NXf+4X+R/9T/41KsLLl59R1sLd7cw777zHe9/+aT59ec/vOs5GsomR2CKlVpaWGUKgBoefJvb7DdEFpu2eVhem/Ya0e4IfrokKOWcurq5ZHoQf/N1f5fPPP+Xh7rWdjEtlHAb2V1ve/+a3eOe9F8RxwEvCBSGlxOHhxmyh1cLhUhjN7dSZAT4Fa12onbNiDLzz7hW7/WDdIXcWWZvkLA4DIQR++INf4+btS77z7je4PK782icfIW5lk7YMITJ4xxQDyXtmtrRWib7gbAqCx5l0plRcSkTfJ9b+PAgFvCPGLvT2ZvNTNRq+NBvwtLN0rv/ZOYu3MLWJrZyiVokJ3T4LeF/79xTKOd0R65uu2abqUb/WgqKi1don65w5uch+v2FMCQmBGL3BMsQjUWga0bQw5861bNkODKKoLtAs2VGkorWx5pkURtLkGVxkjHYACG4g+YjDLI3X22s+ePKCL++/5M3DK5pbTRD/Ix4/Hgulc+wvJo6HtSfduS5noAd/tUf7E/SBDnbznv8eovUOXbAd1PlCCNFE2tF2U5zrF4qBC76CdUq3E3q8n3DST4y9bZHGQJyVtZ8ifKj4ZH2dELT3ySIpNWuWy0COK/iCawMpDgRvC6mIpyHkunK4mXH1lv1+x8XlwPZiA85OQE0tnnYcB9I8IrISNCHFW78zKOM0EKIjhg1Hv1DagLiB2mZwjhQFKWb3O+qKS4ofIIizzSAJMViYU3DeIgacw4sn4sApczsSUFKd0GVFy2LWSlUiDm22eLZWycXhJLHbjlxdbHnxwXd48v438cMFh9sDr96+5cnTp7T1wOvXL6lN+egHP+gtFMub+eN//P/IZx9/xJ//D/48r7/4Hm/u7voN3cCJAWfrQFkr0xSZH6zNwTTRXGAYBJc8w8VTnJugmMtEWuZv/rW/wcc/+Ix5idydIutSqLmxCYXp7S1ffv4LfOPbn/FTP/3TPH32Lk0cazn0pEyhZnvPnXg220vWOhNjIDmhrtlsgh58SAamjVtag5ILznlKbbgYGC/26HqkrQdevPcuX37vwGm+tz5ogxgS0UcjxkeHD46heEqNIBOZjIojOFMiND2TthwQrJdu7gpTk3R1glPXN/reN+8wCwNV2ILmz7kywXqLzlVLZzyDcLUrJUQJoRIj1FxxtVFVWEumaEUqFvyWrMT3voFmI7aXSs2QW7HFNlhQmXSJHs4UJkMMBnAOjWNWtGZULYzNqWVEoRXRAtqjnZ0jkvGyAoE4bNhPWzwjpTQ7MElgkBGXHGPy3M+vaLr8yDXqx2KhFIHdfmAYHPcPR9bcobzdn2lj8fPn/r1H5EoIzk6RUe3P0RT/lhMiDKOjrELN4INx9lQDpdjO7MSxrMWm3tHjgsFNncHrcKIMKbBGw2Y5XNeKBWIS3NndEhyCI3mbTscQSH7ES+zNawDtjhOlnpTb5cR2d8vuYmDcHbjYGR2mNqVpNrRXCASZCDWCsyzpmDzjEBg2AbpedM3Q1FGq+VmDRNRXMCus7dahMgRbcEJQs7sFO220ftESPFd+YlBYasY1j6+ClgzrQvKBNAyUdTW4bR+OtdbYXW0JwbG/eEoct+z2V+AHeHug1ox3nre3R+ZToTRwPlrPrzVOS+ZbP/Ehv+d3/w5+7q/9Ap98/F1cHHuvCVSFVjK77QX4YK9PjEjzjNst24tr6nwgug3iB1qwfB/vPL/6S7/CJ58+8Es/PHASz+Ziz/jkilpPvLyb4ZC5XDJB7mj1u/zU7wxspomyrDR1xGF69AY3VTR4xjT1NspCAzZxa3g87/uiomirxOjIpTBMI/snTxn3F9ze3fCrn3yXX/nke/zwi89QiVzvnnCQO1pZiHE0An3ySPSM1ZOXaGWuehRP6KxUVwvSGg5PEE/0tjhNTTm5jMj6KNwWbXyVFmMtKFUTlYfg7TQHxNCHlr3MrrlT1hlM9+2U6I2SLq7RnOkec6nWQ1TpWU5Geu93L42M6ydsJJBXyKujhgDnBNNuc8RZVMmGhmPX9aAFEZPKeRGSd2wGxzEVqMo4KM6vBssGlrWxnSY20wVh7nOKzmudckF2l3gpHNabH7lG/XgslE7YbCJtslLy/u7EPJdOKqFrgf5+tJod/x3jNLDdeUIqBG92JOfMthhTZBiVvEDpHtMQ7KaOKeHEAqVCzMRoR0jTNwfD8ANZq3m5ve3ZpdpFNW0T42S2htoKTQ0w6pPamz54i6LtkFDvTedV8opmAwwgjdO9icw3+20vcTylZpY2gzaGYC0A7wSpCdxoWc0BxiHQqOQipj2ripaGo7MA1fpSMVijPUWHBIFoJ8eAoL6hriBiInVRT+s+3k1NpOptkipC9YHWlGVesMwdYbvZUFsjeMvtnsaREIXddrIFxkWm5NlsR+7uD+y2Fzbh706q87m+Lkf+1J/8t/gn/3M/a33EhwMxTWymCRFHSoHdtCGmwFoKYwiUPLOZRkIaUOdwwePThMQRfCJI5stXb/je97/g57/7BcM7z/ng3WdM4Sm/+Eu/wMvXP+Db73+T9z78NrdfvuTjhwMXdceXX9zz/B3HGD3UvkiEYO9BN0B4UQpGf3Ldw28nFluKqjRilF7eR9JmZHtxTVPHL/7iL/DX/vZfYylHLoaJ7CZwnuDgfnlLaQuWXBFsc4ueXEdyzSSXzA3VVrz4Xiobjdw7IYbEOCbi4AkBwlC4XY6UdaH2XqsL7lGfrCoEEVyzoZQFojVS9HgP2rJlIK1iCDkx6rinPnJXg3cMQ2RafWe9WlwuVFQs18luIrEBSgzQJkQSx6OdXEMwIr9xAo3CTw8mm+KES+DSAdXZhj214T2kQdhsLA1gMwSmjZlAvGvAzM3DpyCwS9eEZnppvGOQACRq2tD0+CPXqB+LhdKJME4jMSQ2Xft2d3fgdMyUUh+xYV+/rWyhVFIKbDaezc4zjD1LWHIX2jeUTBo8wyigxZITo+02tZpcIvrE6j3eZ6oWpmlHSrZ7abMppzXFXafhe8IgpMkRhz4pVIfHdtA4RROhV2fi4I42E0yvVkqhrMqqjW0CzcLpvnB/+4A0YRgCa1lY2kppqw0IQjSkFo7gBxtEqUN0wLnZSg1dac3jfSSGCSczLVfwnqTC6lbCACmMphurvp9OMgiW+eMxH7AaXzARGJ1tGr65HmuhZq9cT5aDro5xu8F5YUgO76xvdn/7luv3DrQ6sd9s+clvfYs//5d/jt/+Mz/Fhx9+wOubW758eW83Lco3Xjzl1ecf8Zfv7/nV736Xlise43zG5MnzQogTpcFmCDgK3gsVYUgjwzhStBIvLlBxtLVS1sbHH33Bz//yx+h2z/76Gf/VP/Qv8Gu/dsOf+XN/kc8/+5zv/+on/MHf/7P8zM/8FMv9wpf3le/85DfZbrdofstcDkirhB7XUWtFsaxu9b1gDYMRiGo1sbUTWjF97TgkxjSSdnvS7oKH04lPXn5EazPbIdIuR45LowqEtKP6E6seyHVhbZBawEth9J7ZeZJEvFOaeFo7kdzExm0Yo+s+aWEYIuNm7L7qyJNcuD8dubu/57QcyXW2KldsEBaSLRopBTNRxIxnxbmMU1ikIQSWaid7ayC1zlEd8M4bEb1ByWrJlZxMRRKVpoVWGhBJaUR0hLYlhg0qK7U2jot59FPyuFjBNYJV2rZBeUcIHtzAgsWieFVidYxDwbvGMKoNc6P9bJaj/sBnb7/Ps4vGs90znDRyWVEyIXqGKbBK/JFr1I/FQilOmDbmwd35LZvNhiEm3oS3HA/K8VAefd9n8o+qkobIMEIaCzEpwxgJHopWKyWbMe/iMJIGAzSIKyYK9z2fGEuQiwnzHhNJXUMpmF6xZJvsGcW5n9Ci+VXTUKyRLWoUIQl4N9kb7LAsEjVGS82ZUnL38YKZrgMSHLUo66myhBUthdJWlnKi+bUTjCLRgVYjnA9xIjgxeGkYEO5wmN2tNkP8J78h62olUBfcBx8Z0kSMiVY9mgPa1NIZk+HYVAqeiJeAZGfavhaorbLmDB1OW6oyjhu8i50Pmix1cM0cTydO80fcnW55/5u/C5cu+Nnf9gEff/QJH336A95/8S6lKt/73kfMZebJ9VN++qd+kidPnvC3fuG7fP7ZZ3zngxdcX+7NO1+M7u37jRS8oMuMV+Vit8MPI+I8+Ij4idaU6AIvX93xd3/1Y764L3z4k9/kb/z8z/P/+at/i4fDiU8/+YSyztTlwM//wt9ms9vwjW/+FLc3R0q8prZE1Ad8j3co1eAkzhleLcTIWjIhBnKx8laKwirWLvChU7RBAoxP38NdP4NXX3BxccX21cjycM/1xTXTYlrCRRwatzysGRcL4lbEZZKASCRVT1JHCiZNoyUGn4xXGiNTMlZmjDa48NPAmCe2ubIrWy4vrri7f+D+4YbTYcaLMITAGAdi3JBSJA5KiIVajsCMFo9ggXHVJdZs0SGWJVWNeeCisUrHSEtQNNJcIbhKCgb6zVpt8OlNwE5NDMPIOE54FdaamdeF5s2yOQwQgyEODVWohCg2bxAQr6g3OIYSacX6+ylhJ14RIIBvFIHb4yuSF3bTHpWV1lbEKWmASdKPXKN+LBZK74TtNFgokrcw9xgc+IzIgfpI5PG2Sor1PLxXhskTh0pKQkqNGDB6s5693hXIpCGQV+MXiu+Ec5RWCiLmXTbmnzBOQgrWJ3USaMVxdAvOLwR1hqnynpgcIQqq1oh2QbqOz0pulzx5Lb3nmCnMlFotibGYNKPklRA9F9s9AwmKkFujOiNMB+fxPpg+rJ+qa1vx/qIz9aqVwKKkcLZxOQKm4XQSDaMlgmjCS2CMO4bkaVUQP6BFbWjgFYJBilvJ5LJYydQ3JlkX6prRWnHOMQw2GfZOmNJI6DY4UTje33J9vac8HPnik19jc3XNZtzyz/0z/wR/4f/71/jy5RtevPeM7eWOGBzbccu8KH/7b/2Aj7//A37iG8/4p/7g72eIwjo/8HDnqPmeGKx1UlrtMBEblhnFxoFP+HFLw5HvZz756DP+0l//eWS8pmrmcP+WX/jlXzMJlQrXVxf8H/74n+Tf+T/9W7z87DXXly+YNolPXr/i3asPeXh7w8PrTzg93KBiLM6YBqblgj1P2I4bjl1CFOPwqAEG7a2cQNPKtN3i9ns02sR7ksCz7QUxCGuF5BOtjRxqJa8bVhZwh844qCiO4Ax6ER1shgHvA04S0kxD7P1q1rzRsxkSIY4MbkspiVwaU/ZspsYwbpk2W9bjibrOSKvEYFxIK9lLB7BscS7QygltlVUMTFGK9SzdTnHOeobTuIWWWE+eEDYMaUPlCVW+xLkTyr0h+ZoyjBOi5ohJXtlvNqQwsGrmcLrlsB7NgdSJ5sk3U64EjwuOEDoJzGl3ICUGHxCZsGHW2XY5IZr6/eFwNTCfboi+EWPCuBIZF5SUfgsIzkWEzTDYhSaO6o0efnV1wVoKWQ3ftS7moLGvURrm0UxJGAYYR5MBVfF0lZF5UJ3iY8YnJS+uw0SNLt4aUGZCSMSU8AFSVIYUid4DEXRgmQLLfEJrxodIjNb/8d7bEMcLEkCrOV+CRLwLqIclz9ZzctliYZ1RzH0QxFX225EnT64RgeQ8hMqs1SaT6ogECrOd2rxAyzRdUR1NbupMkqLirW2uABVtPSfHBdTkvgw+MsTENgaKU7wfLdBqcDhvi3Ht/c2WMw9rwxeYWiKUTF1OGK1nheIZUMJg6P2KlZtlXgkObl8Xnr94l+PtF3z58oek/TUvXnzIP/sHfpZf/e73+OjzV/jmyEvm8MWXOCd849LxT/3z/zTvv/8elABl5s3rL9ltjDCDWI6OQbmVOI40Z8R0J55xu0O73/fVZ6/4/qcv+fjLN/zk7/4GdVVcc4zeLIoNi0b4mZ/5Kf7AH/gv8P/6M3+a0gqtGmh3mAJhv+P+tWOZj6gIpVaGcUOu5uTabPa0VljXEw5HFU8uM+O4YUiJMBqB27sIpUBeONy8wuWFjXccUEo9cTFccahKW1bSEJlkoLCCVJSB4IXmKuNgQvUxBGIyu2JwkaYZ54txBgT8YMR49RNKIldYV8/xtIC3HmFOA1oy2mx6HlwgDSYZUykd1uPI2gjRAZ61WGwypRJjZZOsHzmMAa8jyU9sxqekGMn5RHURifcUHbg/3LHmivOF3W5PK57kjT3pB4sWaUTmphzrQsgB7yFMzTS9yYa1vmfQqxS8iwQn1GCsVddjVsRiRaE6RALRB6Q2RCqlHYjiidGzlhnV+ki/+o0ePzYLZUxmUqdLRQb1LNUxbSKnxVt0rVuY52qNdd+9364aATlVfLT+49oauVhei3cRg1EUXKiwemqzXBqbekOloJrxoTDE1PFZiSGOCIawutrbhPP21igkIVpPLwaPnOksQWgkok+kMBg0I0KuJ+htAB+UmCohCi05nlxdcf3kCdvd1Kk/1Zh9xdG8Jzxq37yFkYmBeEvtZVnNeDWWoCkCFnwINuGn5xeLJwYl+Gbe2xAseF4V5xIwMKSIiydKq5AduTQWaVRgChGviVAx+tCaabX2XlNEq9Bcn1ai5NaY18zh+MBxPfHBB9/mvasrXBz49KPvMmwu+PD9J/zEB8+sb6XWZzQIw0JZFtZ8ovpEqZkQlPl4z2YcOC0zab9hmWcGp8TdADGx5AWJkXqa2YQj80G5eziQFyGEDQ+HO148+YDrp08JH31ssR0Ip9OJ/+Uf+18x393w9J1rnLcWyCYOpL2w3b7gePcFTgs3dzdQFlSF+8OBnI0RMAyJEKZHUbqLAR71iTAEA0RIqWheGMrC7mLDfd6yryeqztwsr5gl0mgE77gYI8cyoGG2DXQw5UQWCNExxIiLpmekeoagpIjJhbKgfsBvIikO4DeUIqyn1dopYhSm7EdK8Uit1Lp2ayL4YHpQbQUtFS+Cl2aRwypQer6O2jAlBFAWNpt3ibt30BYJvuHcwNoWQtqSOkFKWKjtjnG8xKURpxWRGXUDjkpIlbEImkGZWckMfXgkrqAy41wjRXOJFNeVKT6bKqEt0HqFoVY1tdIICjSrfryLKCshGv8yl/YoBfyNHj8WCyWoWYg6/aeqWkkXHeMYGQfHqgXnEj4U5pOVs965zt2rZtLvuCQfIqV16URvAreWGQaHVsgrtJZ7Zo2hpHI2HVwasVybMHYARGTC4TXQagGFu/sjw6BGXI4NCR3X5EZ8GEhxg/eOtQuDh2FAfOV4OhCSQydl3Box6P0PnrPbbwyHr+b2aWQykWHY4HzrRHe6xF6pJZN5wLk9Uiu+5Q41NVG0E/MkmzWtW8O8IxeIwRH92SVhVjYvE0Mc8BHW1jo/c7UhjR+RtVF0xWG56A1I04Zxt8P52KetIGqgXZ+ibRzNQAyHwx3bzRZH4/l2ZC0L+Wij4xgc+/2OuLkgpQ03b19x9+YVI57D4YHD/Q1+hotN4O3xgf12QksleqGsq31/71kOBxaEi+sn5OXAq8/vWFZ4/uyCF08v+Pyzl8h3PuDDb7/Dxz98l+//8CNQR22Zv/Dn/yw//dM/w3c+/AnSOCHLynvXl1w+fYI/Ki/e/wZjFCQM3L19ZaLxYPEF63qkaSb42JFrnhiHbovNhBxgGHCbDeqEdvuadronhcQ4bhmWe0JNxDYTU2JQWJuyaKOsnpYS201iDHuGcUeajtyf7i0GOIgFelUjvldfbdM0qxoSB8IwksaJWqyExwkSPN41Zinkxeg5OZvsyDmP85bd3nIng0skyED0yhAdRxXjQrpKU4v3Fb+Av8OFC6R5m/y7hFPz1Fc52pDGJZYlsK63bLcRSleVrJkQHSEI200glsi8FuNIiiEXG3ZvKpXgRjYbx+rtGisl96FA6OhF850b3b+iuiA4grOhaqvKklecM/JXLb8Fcr1R844ahaVSW0Wd2KTZ0yfW2EKgdpQuOeOdseScnNFPpucbR28Rs2d/I1aGhwhpdF3fVbuf3CbWtdpiWVsXu3ujtTiJeIlEGdCmpDSymQ4oD0xpJQzFsP0aCLIHjaTQYQbN3rwQAj45SosoJsPZS2MbL7i6umIYzOHjXUBbpmVTAsTBEhHFQ1lOgC1iazmag8Jb9ouElTisKJZ2Zx60YJo0b69R6VBW701gDibCj8HjCQQfiX6i6AxN2bgRH7YMLTJsBkYG/JKpYcUl22BcCAzjhKpFQzg1EDI4k9Jgms7TPHP38JbtxVOCD1xGI9pcPH2XabNjGCfi5gLUcXfzGm0ZaTOnwyvy6cD77/0Eh7u3ZizYbWi5IJIp2RweIieoK6e7xUAHYWB5OELb8sGLd/nn/7l/hn/v//Zn+OTjV7z4xnv8vt+f8any8tUb9rsdP/mT3+b9Dz7gybMXlNPCt59d881vPiNdX9L8EQ3Kdr/hcNpQ8kRT5bRkBM/hcGK79Ww2EyF4xHlCGA0aIaDe43xE44DmmeOrL9A1czFesq4rrCemkLgqC6eSOdbMCeVBBMaIDontcE2KE2kY2IULpnnL3cMNSzmafzsUlkVRybgApTUgUFuiOo+KEseIumTXCIJrgRgaOTjzPIuj5hOCp9Uj0a+IW2isffFUUnK9zeWp1QwUzj3YAkymtHtce8UYEqqpb54e8cFO3mMk1EAMIzlDWY9Ev4VuKlnXQgyRMUDyA04bczeG1Go5OjFu+jAoI/6eFhdAiNFbamQN1NIQrTQs4iR2GrqtIeekxmq5OyGCiLWSfsTjx2OhFFjXo/2yuTKXpcOECiFk0mBkYrR0SYOnxq+QTiGoiVidCcFFIAWDRbivwkb6YuQI3qHBsmqopsU0Z4LRoZs6cs6MSczPGhJNBi5cZFr27MZM5Qb8W8SfaN6S4Zxs8LJHqajL4EeW9dDlSgbtFfHUIuw2iV0yfWEYrCHufSRL5Xh6ABpBJgOaehse5ZppLFQ9UbMSUzTJ2SnTxLyw2hqtBKAR42jGNAeuNfMDO7U2A5ZY593ZqRNIwVssRggMybNJV4Zfaw3fGpZGoTgfHj343kec82iMHB9umOcZaY3tlAgukpdKGnbcHmYyNzx/8pzSKl49h7dfErVy+/pLWjW/ez7cElzlcDqy5plhCjw8vKXkyna7oZUjc+lSMec5no7M8wGLVFcON7fc3Rw43RXiBiQ3fuqDif/hv/SH+HN/+W9xeHvLi3ff5/l/6Tk3t6+YxpEnT95nGhOfffop7/iR3/6Tz3n6U+8gQ6IuDwRpHOcjtMJ2e8EpZ6QstNZYlgWVE5v9FcEloHU5m212wQmSRnzc0A43xPVEaY2QMzss3/vgAz4HaruheUfxwoCa9XHcsxkvScNIChtEHLvtJWPa88XLj8jl+CjNWdcHQNnsPLV6Y6ISydXaSw6zCte6otWSGWvwJisrR/AFIaNa0QJOqkUou4XYM2vSIAwbz7rY/SKdWmVa4hO53EFLBD8ZdV8y2o40d0dMHt8c+I3pQleDdjgvVM32nrYFFyJehE0KRm9vq4W8MeBIJJ8IUcHtoT6AO2J5KLY4WkhatXvfC+tijjoUYjSZk8VJmOY1DQMtDj9yifqxWChFoNaFpgajXfOJKhXnIs4XUsJsSwTTj5WCdxMuOHxccMGDL/hkFjKlEFJE1oB3DpVIa0e0FXzoF0jt3lvMPx1cMKeKNzBHKQZJrXXBucCQJrw0pK1MaYf4DbkNLPUlGg+kOOK5QHRDFZPTSHggt9Uo5VUIIdF0wXtlSiNDrMR4j/OXrHmm5MqyHjid7tjt9gSXEB/MRVMTknsZLo5ajSjtgvWKymqgkHUtJtZ1AhQru51DgpJQUujZxtoIGAgjuB7E5CvUzFzf8pAfmPWaqV4y1oQvQhTTT3pvz+ljeAwXO51m5qXg4xapMw/3R3KezSk12Ot7VMeX9TVX+z1ffvkKR+Xlyy9tICWNNIxGe8qFdc3gAw+nE29uPuViv2XajNRckRDAeVIcWNcV5xqhx/8uEri/ecv97QPPUyOww5H48J0r/tA//dv59It7fu2LA69ODzzdXhlR/OaBTOUnri74HR9+wO/8fT/FNjZuP/olyhc/4P71S06HQ+8/RutBV+kcgRGkcHv3htp27LbbHkfQbHPyEb+7pIVAnQ9ozZRlpqwHEpUwTKZSaJU07VlkxenMwIALkF1BQmYcnrFJGyuSfGBMF7S18EXXY3pniYy1FvIysywzY6tkGqqFYGB0YzLqDHpCW6JVy98e45alQW0nSltpLRN97oMOj/Mwjp66M8PB8Xik57NRW6FUAZnJemu5PC0BwjQFkGPHE0YzO4RGK5EQEmU1I8k0XNIq1LagZEQ8MYxAQaUQvLWPtPaQPXGIDFZKR8+S7+x07bQrBBLaAqgnaDEQMgDeFC4uWAIBitNG9L8FdJSIUXXWdabqSmlHKgVpNh2OUTDkjxCdmBuBZGy8QRiTEmTFUcyRIg7nIlk80hwuDVTf0HIEbYTYeiPehjGoM3/t0Dqg1OQ4TW26vOTGOF3iW2AcB6IfCXGkSuKweJb8ipi2BLnEsrsrOc8s8w0hQOlDGgRidBSpdkqMDYkz2uJjjsfD4UCpFZWp9117uRwGfPBU1d7PxMryDloqrVFroTRFWkVw1FoY49BzvCvB20ZgPUrTlJZ6AB0A+7rGSpPMiQOtOGP+aWPjOlmaxunh3uJYQzTAq610NFVyzmhZSUMyB1ReOZ4WxkGoOoNCCpHd1VNyXpEY2YwjtSzUvNJKoayFZTmy5hVtgRBHHJGgIyFuSMPIvK6U5qyNoIWHuzvyuvBwdyAMCfELeblh2kdcSuwvt6TrHUP6mA/fveDt2wtuHirNOcQHdtuBFy+e8+Jb7xF84e7j73F8+xmHl59zuH+DisfFiSENeDUlwXw8UXLheDqiBIL3GBzakaaRcdoQnjxFpw1tPpifPgykVFmXo3WFnCfEyEY2LIvSfMApHNWRwsiqK+vpRLwcSHG0MjqNOIm4NjCEiZdvPuJ0+hLF9YjYO2o5UHImrwWi3V+A+cIr1FKoGWiWeRPdhjAM5PrAvMBaFmIseKlUjUZLR9luk1V2Esg1oi3YZiVicA5Z8W4GbGjSmiOkobvrrD9urEiDqkQ3GkPVRetl1kCtK3hvaYoe1loeHVEIHSp85sg2C9RztoA7HxE3oS2Z11+TEa7cwTiZVRAxDiUlgPapv7jfYHGyx4/FQinY5Ls1pbTcqd0KZNNJuQCxH5VTtKq8x6vGFAmSGULCaYNqJy6nCXGjDS0UxA2oy9bwDuBjpla6s8V6ddNm7CltDtXCWo6IOFo5cd8+Z7/5Fiqe4CPjCE0UCZe4ozkKHkN2xUMSXLYTmi3MBrk1qVKPiA1q9i49kGumlNV2VAnAjhBKJzN7YtiSoqdoJtJlT95E8CZz8uTqWdaMbwVJofdtza3hW6BKICWHdzNrXeymckoVE51LU5TZXncMVOAw5H8pDS19Ku8VLYW1VGq1/46nB0rNZqeLCSeOq4tLSsksxyPH+YRooU0X3N/fcf30KdfXtlheXmzxw4Z1Vmq3jcYczQbnYDNt2W0v2F4+R4aJXAvBDd0P7FnnlePRTlExZaI0G6C1RmiF7TaS65HLD36S8Z3nfPF3fpGwdTxtNpTZXV0Th0BwjuPNpywPr3n98a+RDw+EYcQNvovqM/fHBenfd7MZWZdM1blDimEcIuIbwXt2+ye4J+/AtMc93JulFAsBG7yntICjsfEOH5JJffREAmJzVO9YZeLN8sDd4Us2008QfTJieExcViHJd7jevsOnX/wit4dMcw04seZXLPMFfgw0N6G99FZ1VAZqXaz8JnLOz6EKye8IY+S0RGp5A3Lq96IRvXy0VthOJxoD8yLUaqL0oqu1kMQWTnXK6uiQ7EbJELyZEnCBliPOD7QmlLwQYsW7geAucJKBlRAnc/+EhRSSGUl0JbeMaKPUmXk52DXrPN4lnIbOQrJyO3TLWWumbjktt2yGJ/gwgAbL56nt712Wft3jx2KhhHMAlGHJFKNjm7HNpDfqYRgcXnrqm/Rhj9POVrTgdsHRqqUAivf2Jku26S8BdRXv82OusPXovDkaQsDFs13S0cpMFSG4kbXeMec3jOE54sUIKc56ns4H83q3hRiVykrjiItK1ABUWouUZnIke/8WVEaKFmIcGUZHW04kPCUXqh5AClFS50w6hmFP0QMwIxIMYorQtKBNqdWGUhZ6J71EL2ifTCOBVk/4UIjeGfW7HcBtaNXiBpQVp8IkW7bxkk24IGrE+YaTSm1Q15V1OSAKSy4sp0wuM7VVs1eq57DcUY4PeG9N9lpX0MbrN3cM40DWwov33mN/fcX9/Wuurq642j/l5u4B/MpSGw8LVCnEccvls/cYdldM42BOnVwIweG8AgV/GpH1RCGQJFHbHQDrfM+1PKfWmfWLH9KGwPNne+o68fbtG2gzh1cP1FaY5xNlXZGWGaYNTpTD8ZZ8zOx3F0RnCDuXAnGI3N3d98RMA8se7+6hFi6eXjOOO/zVNQwD5BnyjDbzJ9d1tVz0bMiwFA12sXEeZbRgOoS5KRqU5CeW+Y7706fspudERiYXGXaJ4CbA8ezphwyT8rB8TOEGWWfq4Ya22aLe9LQqkZYbrXhEJmq+o60HpO2hRlQXnCrBJ3bpqW28+RVLmy2rR00al3obqhRwumVdR8t4rxjOLfVICYVa1MprX3tSZsS7AVWHSkGoOCKoI5fKZgwMaYDiydXyiCzrR6EeDUqTHLVDFkvNqAra7HQa3Mb4qQ1olv8NHu+sjEcKrTaO81s2wxOcDpbOqb8FdJTn3BEz20e8AWwAO03hTNCMs+Tg2OUu5+he8bZoNaxsaerQapSU1kGkzlugl6fivU3Kvbfg9dhZeMFZ/0K10ppRU9byFol7VAcejl9gCttKEBgDSE9+KyVbFAIZdSuNmYYQ0xZV64msebEpe6ugi+WsJCv/ox8ZxsCyrDwcXlP0llKvST3I3WyLW9YyoVIgG5HTMNF0jZhN6O2Vs99Rq1JyNZeDBhO7OzupewJK6VDgFbLrKYJqGc6+kssMWoja6Ue9j7wsJ2rOzEthXRt5XSmtEUfh/jgzbRJrwRY1gRRNxK2qhGHLuLnmtDQuLxP762dIjBQXGXbXLBzh2MjMxDSyzCuffPR93sbPeP/Fu+wvn+FSQkJCO4tUpwbrwhCMn1nrypqVvdvS1oXh6gkaHe3mltsvfwgijClxf7o3EHFttFzZTiPrUri9u+F0vMdpZhzMElnofWFJlAUrw8UxLwvemfLCOSGFxLS7QKKnHd8iOSMtP+bDWCa9RRuDohmCSD9lNvYSCQKtLhTvkLjlqDOnty8p+R7vHfuwIaURUqOMW47LyNQuCWPlsHhyO9DWe/LxASSiEnpGjclmFEFd4zQfoVZCuLD8bW+gZpcq5RDwbHGotU0QnEtm+fXRJDVNKKsZOFoN9jupe7wWnYQOC3HUunCaHyCNWC4P6HqEGnFuMPQalSnalFGas7ZOM3xaqwu5LEAyALZYBWrJCMZC9XKJcxmcldnaSheg92x0bThXKHXmtBRSuLSsp1z+wYtTf/xYLJTQ5TtnG5IMxq9zjVq7KFyahcQHRwzR+HreTn5yPj2V1SAN/QmbCireFlIpBBepzdT5Xhr4oROIPM47fIj47gpyUlGfkdbI5YCwUhs8HL7A7wr4xLIqtXaSSavk9cTaGj6BajYPtQtEP1JdYHBbam3m/qDi3UhwBnx1IsR4CeGetQ2sy4lSH+xmkEjtF40Th3eO5itoecwKRx3SILkB9ecLw6AHtaih1pwBDWgYN7MDW2vLaMnU2iPsqwmKF21McSDIQFtW1pLt34u5mmpRSrFYvxATwQdcCKSU8ENkXmbi5AneG11oY37atLti//Qdnr//TcbJ5DbzunJ7c8/+4oJnL96hhS/4/Hbmo+9/Qp1f887lgA6eX339fWKa2E5bYpoYL68ZN1dkBDc9Y/QLKTSG4YIQRlzw5PzAIM+RIeFjohVYdeF4OhHH0TZX11jWB159+QXHwx3b7ZaL/RWh20fPvwPFhjEhmUDaSWO7GaE1QvfSp2GEmKgq+OMJ1pNlp2MhZMFbXzJrRdtKCgE1iS5NIahDnMlWxlJYAJXIg96SDzf4EBncyFQvjXiuCzEM9roz0bhCMpRyIp/uUUbUJSrrY4+vFEPbNVlY6wnKA2PYIjXgIigGjbD1w6FqtHNr5TVSGNAQKBlyUPJqSZiqSgwj0Y/QLcAi1iIqRWk6Q7sjjRtUB/OBVWiFHhq2cN8au/3OmJ5qjIGmhZzV+uCrnSebLNCJ506C9Rm9Iw0TZTXUl7JawFjONC1Y3lDGS6Yxc6p3DG6DxN8C4N6zfieEiLTQHSy58+gMblF9F5D6FY3mNhFnmcSms8qY0jRYmX0OXdfuTmnaG7Zmi/LB3C5OEx7HECIu2NQMsVRHCUqrgVY8RU9oCTS95xCEi3hBEc9aVrRllEzRTC0rQbpjQhSpkehHojbEBzQIy3qkNYgERK009aESYqVko7RHZzezK0fb8VqwaaDzeI2ot4meet9LeaHS2wtSiT3nRBz9IoEoFjVg1UjrU9A+aDJyMWtbjKzSKolAbWbPHOKOlg9oFIbNjrxkJDZ244a8ZNQZf1DE8q/zqXTBfmUuBcTh3cDzJ0/59rde8I0Pvk0cJmQc2E5bnsQRQSm1khE2S+ZbH8D93cr3f+kVrz7+nHevB95/ccF8f8McE9uLS2pdOdy/5Ngi189eIGnAp8oYR7bboecMBZyr+P0eSVuuJXD38lNOD/ekaYOWTF4WoiibNJDctWlUW7MNAVMWpDTi02hDM60EnxjTxgT6y4zSGKaJaTDDgYaA1A1ajtTDndlm28yaTdDtk6fk0Mngys57AoX7srATh8ezjRP32lhpzJrI9Z6H249wecsYb/FuY4i9kqnVWUyEmp2wVaUcZoQTJEE1G5MAR14Ly3GGmml6pNQ7Srtl1AF1kRS8wa+dUHJFtNli2SszFyN4Zdj0qOjouxxImMbJpF7Zrikn0bKIxFFrZuWEuGicy1JoJSLlgHcDIsJ8WnHBbJOtLZT6QFOLjFZqzxzHOKwtPoKVLSCjoC3jfOqVas85p1euzqzM7jEjKFO5x7t/RCiGiPzbwB8GvlTV390/9r8A/nvAy/5p/5qq/tn+b/8z4F8GKvA/UNX/4B9mqXTSO4M+ccbSi/bBjF+pwZiL2qBpxoKOHM4nUE+l2MLkQNQmYMFD6RGYPogFhZ0XD6cWpN5ih4ha7o241st401c6cRS1HQ+pxsKrsGYl6sZkRGQ6m9kcAtIIPqGtC6+9+cudjP3nyuTWKNnADs4JKrlrHoWUHKJKnm+RIjS25iKpFXzDOU8QWxxRQTsxScCUAN6yf5pmalvIxS6e6tRQasFea8OFmYvBae1h99U0bdKoeiK3meK8sStj6q9LZncxcTwpd/cnfKfHT5uJFEfGzQYfBhqOSqVWJU4bttsrovc8efEuF0/MD/xwOnFz/9bKtRCZtjueP3+X8dvX7C9uORZ4++qWH74+8IOPXrLbOTaDMKQNr9+u/PJ3v8ez955z8c43EZeQ0Hj67AklH00SEh27zTP85h1kuCBeXeIuX8C0Z/mVn+fzTz7iNJ+MlzntO4ihsbbeqnGe3cUzwjDgQyLGCAq5ZBAr5ebDwUrpVi0mQRVdT7R7ZxtaaazLkZwLlEz0nkqg1kyMkVYVn+x9K0tmEEcBC8BiwXkhUIkEqjhWveNm/i5peQ5qyLJWKqUWQvLQIo4B0ULJBeYTtRS7ti3AxDJo8kxdZpyYh7rpTG4zvgWCJHx0RFUajmWxpaIUs1hWKRbdEBppEMTFTtHy+J7lraX2g4ezfHBvLaTaKiUfQRytDpRS8RVypQvAK5xeM2ig6mwStBAfYReqhdwOOEm45vtw1wZlqo01n6zaqtJPxhZfoT3U3IlDxZQLQkUVaz/9oyyUwJ8C/g3g3/17Pv6/U9X/9dc/ICK/E/hvAL8LeB/4D0Xkp9WaMb/hQzDRM+KpDdNPOgduQdXR1JvkIppcqGnGwo0siIgwUGrs2cSmGyvNU2uxC9epqfCdyYbAiCSlORRnzd9OAFI1/WSvfB4ndtYqMa9sw7GsBzttVBO6qoLld2h/4YVafMfmG/vSBeuDqq+0urLmmfm0EKKjthMBjw8D280z4Cm37SV5eSDX1Zr2atN43+HEIvbza4DWrB/rRAxe7E0SVMqBUpVSPAsrSQeabwRPb2uEDn61k7u2ajgxFk6tkSQQqiJYTIUbPIEd43TB9qrx/H2B5sjVQsecC6SYTE4TBqb9nt3FFTiH69Kgq+s9LiXUB3wUDvOBzbTl/uEBNzQ++uwLxu0FabPlOz/902y2O/7C4YA7BObVhm/7dIF3G7793s+y2e+5enrF7onwzRdXbHYbTg9H1jnbjPnyfbj6EMKma+cgXTxnc/nCZFn+Dmrl9uHUYy3Ac+LwcCD4QGmZMG5oKoQU2IyT3eTFTvgxBjSOxOAZL/bUcUcZRtOyrovdxC7ixEDSrVVaUevbFTvttPqVUD3iEa0EycylEJoySmbCWjkiylLuaK6R10ArAVfHx152iFtyKZS6UHVFTzd0s3S/1p2lLK4z2np0c+08V1ZwC6WdbMgSFF8iwUeWUkCD0fdzNgtjm40XGYL1irXhpdGqUaaaNJo0nLcojxgTeW3M6x0iAVpFRalE0EpeGylFalWTj4nDhz0+2kYi0hMg5weKHm1RxXq/rcN8Q4i0KqyrcSFqrVRm+9ncCNDRbdr/l9E2/8hF8B8m1/s/EpEPf7PP648/Avz7qroAvyYivwr8AeCv/KgvEpGOpPJ2IXtDwVvMZYVq2sGAIfWrVEQcwSdSTNBGsya5QuNALRXvPa3mfhy3gCHvjFyi6o3kjTfSivcdWdUvBF2hGpAULJ5Tiu1WKpmc7fzonKMWy+DJ/TTRmoFtq2ZULXMYHUESy3qwMloVpCIilNJY9QEpmYkN2ziSwp7od9BGXufPWLMNiJwLeOnTSafYbFRMUC5WfHuvvd9qEaK1B2LlstpUvGVwjuqxE7g6s3u5ldpWxJmjoVVL2SuaKb5xcfWMjdtZX2vJnN685eb15xwebqhZkAi7y0t8GDid7qhNWHMB56kItSq77Z7nz59T67s8u74khYEh7Xjn3QtiGHjxrQ8ZtzuST7x8+5plPXHx5CnOJ37Xf/6f5O/8pf83VMftmyPDduXd7/wMz97/HZzWt4ybyovnzxj3l6TrJ+R9Jq2N9VRheEJD8MMe7xrr3cycTzzUwsu3b1jvX5EPbwnOGfA4JOJm5MnTyQTbKJTVGJxzQ9YZCYEUAjVbxnaKkRA9m3HDEBKaLbGz5AXfKs4lUvKspztKXmgtQ6tdG2gLZKmlE3DsqnUVJhxRGhoczQVu1E6EKcKaD9Q20LJZBFUDrXqjZxEpudGolh6Zz6FgJp1pulCKDTxrtniGGhLD4IzZGlfQhSgDUgUlsDSHVG/wDO/J+WAUq2aHDxN7W5WCOmpZzFvuvCkxmlHVVVbLqGkFRyJ4R6MiNaLZo0slY31ag2zbidUGrWYLHoZrfImo3KHN1Cm0xlIWxF/asEiVpgHp0OyeTG2IQ7H5hfn9I6X845t6/ysi8t8C/lPgf6yqb4EPgP/4a5/zw/6xv+8hIn8U+KMA+6sR703J7zDmnbiA9xGjb/dwIz13M42e4v1oOCWxEHqnlVwqlQWofTeKBD/ifTABbchkFwi+UIujZNcXXVNu1wrVeRrOfN5uILhGFcOktdYsO4ZGrfdAoLVsJaZWaukXfV4A19EqKz6YP7oUQ8EpfeopDe15uSIDKVwbY5DK4PcE/8Dp+BYXTYqEWLnQ1Kgy9N6RWbgardW+uztEIrk06xnWDDXQfKNWk2OJFxyJ2gpVC+LKo8Vzs32Hi8377NNzWLcsh0Iur1nWGSkgZcWlCT9W9heJYRx5e3PHsr7m9ctXOB/44Ns/wdWT97l6+pw4JG5uXpPGid31E+5PD7z8/CP2u2vG7c7cLpsRHxO77SVpsKzq+4cj4iK/5w/+F6FlfuU//SvmHPFXaNgisbEdhWdPr0gxMB8zx3KLGycET7q8IG4v8NMefOyTZs96c085PHB784Yg4DeXHB9uCe2IKzNLWyi14MXjQ8T50PmX0U6F4vsG6zkdDxSZefLsJ7l4/gHsByQMyGya04pSnUmVNBfyOgON6M3L71jMaaWNGDzlfI07aHkle1ARAkJqE0kaxRWczyBHREaQ0Vod1RBj2hKqQi4npFpvsuTOUWjSKyQh+tDjFjLNQYyRGFN3t1VEjwzTBp8CEgbWQ6Ss3u5XZ6V4bTPSs2xKzaCeViK5ZmoRYhjNPhwExSKivQ59Kk2XA5peV9VbUF02Tqw4JfqAVg/BFrxaTQ/Z2mz3j7MhjzRLG13zA95VGh5IOBlIYeQxRE0st1xFjHxF63nuv/Hj/9+F8t8E/hj2O/4x4H8D/Hf/szyBqv4J4E8AvPfNS00p2YLhLcBLMZQaEnso0kgpM4UM3uPDiJPRTn3Od0SZM592aUZ4cR4vNnUOwYTQ4he8H2httmYz1Rq+ONbSaHWlaMNFR5LRjuqi1GBatFItcXBd1TzX6hCx/ocx8MQQVE1MV5YrOZ9wAXMUhWIvulqQUimNtVZCbMzzkbJdCN4EwYidFo3TKab3ktYjALJ5yInd04uVJWJ6R+esp5bzyUoWNZ842IZ0zm6jqwlUi0UGXD1jiCNlLszHmfn2Dfvg2MsFYxoJLlJKoywjKTR21y+4u/mU7330A8qyMI2J3/v7/yDT5pK3dzd88cX3+bVf+ztsL57y07/r9xCD55NPvqDpynsvPsC7gVIb81qY24E0Fu7uD0zjRC0GetXaWNfG9tmH7F684u3nP2R8732efuNdNteJKT5FNJNr5XB/x/38paUUDhO73XM228bF80jC41IgbXdcPHkftx4QX/j8s89QGrvtNa1kNC80qVbRVGVZas9ZyizLio8D+4tLWqsMw0gb7XUJ0waXBrRF9HSk3d4xLwvacndmNeK0R0KknMxkgJROQ1dqq7Yw+8ZaVijZ0jJbYTRXBrNsSERauaNRqDWjNYDMxGC6Wm2gGvGyRes9ra04r7Q20+pKK47WBlpfPIMzo8TpYIvuznlSiOYek4zSiBIJbsuigdUJpQgubE3mpg89aVKsjO3k86KZXCOVbFlGARvuOA8uIGqiXydq+mgcxAjNUzB77OmUURy41M0goU/u71EWcxipQZKd80irzPNM8A3Rvc0GMOgveFqFrCt2cwKYcgX5R+9R/oMWuS/OfxaRPwn8P/pfPwG++bVP/Ub/2I98GFfS4WQwB4G3yATnxHYutTzs4Dy5NhozXq08EfE4F1AvtDKbaN13X6uMCJM9t4+k5KhaEbf0qfRKpdoOWC3yoRZPqZAwSU8VzAHkHXY6735qUbKu5LpgueKuC8CdRZRWqNma/ZIboROkvVZ819Npc/b7NEfJgBbu798QLqMNrWohSGATdqD95FozVa3vA8bh02oiYsVymkWlt25M0mF0oAH1GRey6VHFYitamxFXef7022yHC47zSw6HN7gWSH5PHAVcY9XZ4LgY8iptB5wE3r56yZI9H37nZyyLZFn5+KOPuLt7Q1lXvvXbfppvfPgt3t4c+aVf+jm0CdPmGb/3n/hZ1HlCijgVSm5sdzu7OcqJ1y9f2SlBHVIqy7xQVHHTlv07H7DUwA8/f8vTtXJ9ObAdhNN8R0qB7S6R0gZ1A8N2tDyYViGfaNmhEcL1BTu+QRsG3PaafLi34LeW0Ww98GWeebi/pR3uOR4f2EwDMSRaWZgPd8Q04n1ms5nYXl4xbQa0zjjN6LqSc+kwkWz23Go3dUgDcX9l10YtiBa0rogKFZNw+aZsgif0dMW1VVSEyTkWhBMDLBnJC0u5s8z3YYdEGwQ69YzuAuJKLrZRgk2lVexUlls1jaOLNlRaBbTrQX1kGCIpGK+xakG1EsOERsH7wYYvrqB63xUCluYpWIifSIMmvQ/ou96xIE0ttExtaBt9JLqNnRo1UNZKdAOqlTUXDodblnJkLb1l5grIamV/v+q9N0mSd4nSDCrjz+4tPNFvEA1kzRQHwWebL0jrFdo/BmeOiLxQ1c/6X/8F4G/1P/9p4P8sIv9bbJjzU8B/8ps9nyIUVZL3uGZNWZGG1taP25ZA6BMMFfJcqKJUZ7Qf6ZIYe6aINo/SaFpwwSPOmSFf1MAaQVFXCa6RnDkHVAsiSqmespqXvIiRidRb/8I57FRHsoFAACGS653h0ewTqE0p2fzkrVlvpOFBKpWKlhUhWKO5uA77MJrP/eEGlcqUzKEh0nd32ZLzQi1G5a7qLHdZKyImkmyNnqrney6P3Vwilu4X08S0iYifadyR18xue81me8393Uvubz9hHALOjYzjJa16mq7M7RatjUn2dlEPI1qU+/sbrp88Z//Blk9efsyrzz75/1H3L7GWrWuaHvR8/21c5pzrEhF777PPyXRWZlaliyoj0h2MVUiURAs67lmigTBCMg3TQKKBRYeWJbeQ3EIqyQ0sIcASSNBAIITKDctUGVe5VNdM0lnpzDxnn32LWJd5GWP8t4/GNyIyy6o8WVKCdc6UQntrrVhrxZpzjn98l/d9XnS7UbfKj375n+PzH/xz/PgPf5+//Tf/X0zTCYYTX3zxS/zXfvM3+clP/pBf+dVf5f3ze1JMtCaUUtiy5Stfr1ZV3q43jqcTbBvltvD+m+84HWaaC9xypn3/ni3PCJn748S6vudwOHF3PzAdA8c4cjiedlF5JgwzPW/0kpHDkdg778Yjz+/fk9cr29O3lG2zWFjgdDqwirI6sfdR8LTSeV1eGA8Fp8o0jEzHI847at5wVHyciA/3+O1GuXTKtlK3la6NVm84HON4JI4HW+bUTqsLdDscNOwdUjVtYW0d5y2+IlbPIAdyUeq2sm4v1HJDe+LEl6izXJ8YZpz7nBqOtL7S+xNNz3ifEUxb2FqlObVFEDMle5YLJqvTRLivJtDeq1eHEPxM6wHFIz5Sg4f+RG1PFs8bTLoXY0C6/BGYxYG58PSTYQStqGScT7ubRxAZLG1UwLvEVp/Y8jNb2RjGXToolh0V/cQQZxuPSDTFh3vYr1dbtirN9h67zMu3ju5M1qoV8ZbH9LMe/yzyoP8d8FeBdyLyY+B/CfxVEflNrHj9z4H/MYCq/gMR+feBf4gNzf6NP23jvf8Ueheasy0s6s11wseq0T4fg2caPOuW2bYVZSEEt+ujbHPYajdZYFOaK7ReiX6g90arini7wyAJHwcCxqFT3aVB3lFV0c1yg33sOK92pxSTL7Tm6L3hFMZwwEmmcLMXnl2bKLZs6f1j5nWh9IpUqwR73yi503ZKdoojrUdGjVzljPaNMTyYpzlGi7bVAWmNsm3UrriUCdLw3oE3AEjvQi52y/DOQbdBeEoDQ+xEr6Q0IOGHpPsjr68rP/nJ7+McTClZMiPZhMHB7sJRPKEp1E7VShDLVPnBD77k6emJP/jqJyQHD3ef03rjiy8+5/Xpe/7B3/mP6a1wOp0oGvnn/8Jf4s//xl/kb/yN/5B/6b/+L/Of/97v8/btW17PL/QujGm21M1ui7IPTx8QhW/ff0e5LuTryuuHF+qWccd7hmkA8Wy5M04jRR1xOLBWh1sbS7nSW2LL4ILnIZygdfDgxwGpwjgd0TRxPwycnz6g2rixV5X70u3w7sD5+ZlSVkrNiFYDTG9XmGemdCClO9zpHqKnriuoQsl0YMsr5/MLTqtdnMV+P9XCtnwwmUo00bpzhkLzcoReyKsSUYYa6HW1A1YHtuq4tAvaEtsyUtuC1xtTLDs82Gb8PTd6M8tidI1Sd9vgRzivGiKta0U007ptlb2PXDEg9JActSk5bzbv7LYziGJLpCk8QBOKg+5eUDqijiFFAtESCZypMexaMEyeD9BqoemN2jtjfIMnIGHCiSECW62wAC3Q5HXfC5h0TxBwSgjRHEM7NSME6Fqpte9LTgfSKLWzrFfyejG3XhBwzTb//Blbb1X97/1TPvzv/oy//28B/9af9n3/C19DzZVeK84bTaQ1aBhVZBxGSs8mhB0OjKNSbh9YyxXWyjSMBD/Z3bpnlEZr3VLWuOKcHWzjkPAuGbOug+CJHrrLiFhYk+kt7cBtXdjygoRsZGU1z6qEhBazg0WvRIn0Fu1FxkBHJhmyFtuJcTPp7NKMak6erbIszSRHo0kfwr5Y2iTjB5tdmqc/UKWZuFaE2iqu9l0kb1w9nLNNa3MonRTVblc7cy+6QgiZYZiYpjvef/ie2/qCiOG2cu5Ev9AlUcvFMlT80bb7WtD+yiCPzIcTdb3yzTc/pmxwHE807TzevUG18fU3X5u75XjEhUguwm/8+l/isy9+xF//D/7v/JW/8t/i7/+9f8SXP/ySb376NaqNt2/eUKtpVK/XK3/4Bz/GfwwMy9UIOj7yg1/5ZVprvH54j/bMZ+/ekbyBMWrp3D8cmA8PHOYj03wkhATORiNb3vCDIMUshLVs9NrI20LfLoQQOH7xI9LpgZevfp/89C3LeiZ4h1ODQpTacFKZhoGHNz/gix/9OdLp0V6f8zNdF5x6tmZjiLZu5NsNRyevC9HZGEdEkA4hDrgQLMrD2WKh90apJvz3rTPjwUU0NFy3TCOHgUl63Yw2pZGaYbld8GEwOMTgEIK95/tKyw0tFjmMdAvZU8zSqB3V1RZAeYB+RNSzXAut2pa+5E7LF6IMZt/0CZo9J9EN9DZS2wVcBh8IYvrbWtS2+K7b4oQ/6hqR9olEXuqVMNwzponJn1CF5XZFa4QeWauwc7UR+i4mb5QamabRrjVv1TEaEMkmyxNHaSb7KrWyVZsZBw27IP4jquRPfvzcOHNyrZSSGQbLTy61kNtKbw4f7gn7i++94zie0N64bJnSzqSuOEz60lqlVhvsC4oTz7I0hjDh3UjF8rHD3ga48BF/r3ivJOfoMZC7o/fKmhf8UHDOXCYgDMET1FNVrL3d85HZ62DV3Zljtn26Cl4TznvEffTFWkhWq41OwPtOi0rxHYqSHCAvOGeAEKiEJGh2HMaRzszaVpNC7RQi8Yo0b0SiDrmYLaw206m6aFzNeY6cr38ALhPdI1u/oWrEpjVXvBacLoxOyH7GD4GgkSk98O7hB3z/9TdQK8Mwc5hHbi8rj2/fcF2vnF9eiOPIydsYYq2VX//zv0GMB/76X/+/8S//N/8b/Mf/77/JL335Iz58+AbnPLVWrrebMQ4Vvvrqp6zXlXePn3F3/0DNhTqNdhNJMyEmzq8vtLLxk5/8mIeHO+7uDwzTwHnpqCt0zJ5IU0YfiUOwAwUzMXRx5KZs68JyeUJbpuWNy+3GWhpf/tp/he31Rzx99ds8ffNj2nojumZLRDfRTO1I7Z357o4w3NNvL7TzK65t9FqhVHppLLcz6+2C007yzto/TCpTaiP6gOtqVPx9bidV8dV8zrVmtnyhSaWKI6uylQKt4Kn4bu+h3uF6WxlHuMrZgvNUQAKtJXLxrBmIUNWSG534PWhvQ2S0BFE3oC0S3InoOtI3er1Qyiu1vNAlAhENQmveRmQ7yFn1I2loj8To0Q5EZzZiYYc9w67cMESfUnBeqe2CkxPBmxojRPBVGbCIjNwsH4qcab0YBi5/wIfGNL5B1RtzoWdqq9ScCN5ydVQt7jZ1b2OlZj2rvfd/AXiUSif3C51O6StKQEUo7UztlZSVh+GX8H7Ae2VMido61+WF2h0lZFvmdPNcG3fOQqsKC0jB0QnV4j7FBTRa3MNHIIfscRLeQ4o2F21qQfetN5ouNift3jbLwejdrTdDp4vb3yg7wEPc7qVutOY+uQfS4D6NE0SEEAKtOrSZ9qm1Tl2Msl77th/EwhBklzkEtAemdEDrtgej2d3Z2gyTfbAvqWyrV7i1zjg9EobE86slHsJsz416mjRyvyG94DUS6oDISojPkFfeHn7EPCW++/bHgDAMJ1pRXl/OPN6/5bvvvsV5sWgIGvFwz/Vy49d+6QtKrfyj3/pb/Au/+Zf5/d//A+5P91wuN8Zx4OXlhdPpxIcPH6jVNsCHw8jD8UBwDu8ab794w7aZx3pIEzFFjnOg5MLleuFyubJuK1/86Ee44hg7gLBtG6REa41aKtM8sywLp9MDTWCcZjyF9So8Pb8nL1cOKRGXV37rP/k97t5+yXR3x2fjn2N5eaZcnvEqnB7fcbp/w/39Ow73D1AKTS/UJsh4z3L7QMsZit38vQvM40jeFj6OwoyoX/Eiu6TLyPw2w+v44PFpRHvEZYcbIkvbWLt9h7Wv3PLI5CpT7NQGRY1qtW03ROyGjJpmsNTCsi4s5YbojTAYdq27bnyDZvlOwR8YUsB566ZCMieSd2FPAc30fmXN5mYRN5KbhdJVtc1339c3VrVCTAnVhhOLykDVxhe+0/oGauYM8Djx3G5nwiHuipaGuG7RtA58D2jbyK1T64pTZRgc19tGKVdiCKjru9xqRsujaaKDMwF+z3TdNa5NoHuLmfC/ADxKC/jaAGt5JYxmiI+Omg1W0FmJ/kSKA5s2Um0EN1HrM7Vmmxl12eUAzjRVzTRTVQuim7HwwMjovhGCSTJa2+1OKkZiDmB9suXPNJSqDe8C3gdyLZRcyMXiZz/5TMXsk8EdEBdoslK7w+1OGq+ClmpyJ8vWApx5fbsg2Cyx1tWSDhV6u1jFOnvmZO152TH3InuYUqv45Gzb7YWmjlY7znUDZrSdlOka33/4XUJccJJQvbMNuLdqsmqn1wLirNJgY81nDvNbpvHI69MHpniitE5eruRVuT/d891337CzwZjjgXE6sq4rX/zwRwjC11/9hF/91V9j2yqqFis6H0a+/fZb3rx9y/l8JqXEOE42gC+RKI55ttC1Vlc8ncP9zPF4x7asyJBw7sjj2zvev//A5dU208NhMtd7cDhn38MWB6aTDSFwu70y3T1Qa+d2OTOlyDSM+LxwffoeEeXNaeL28hUteqbDgbu3nxHefU5ZNx7uH7i7e2B++Ax3eoTuoN/QQVBN9OXM8vot6/WF3DIOR3T2OrdScMkwYM45fLA/Krt5wHsL62o74wAIPn5ymOSWGfDcuYEt3aHecc2NRcquDvHUupEzhNDpXenN3GW9VXNftQ3XCyl6RHcbrDMUnvMZ5YbzQneBXJQUG7Xvcck4i8QtV3KznJ6iK6UXWl8sjhnrxlrfrzlnvmvnOorHOaV1mzUGP9BbBN9oveC00vrC0+v3HOa3KCtNbpS+mPa4g0qgVEfRgOxLN/pGLQspJnyMoAlBaHWxVr8NiDOveSnLvkj1Jkmsid7/jF7v/zIeguIdn+yEiG2qpiGxNnN4bDVzdxiJ8UjXhVBsaO2a0JqBce07Ye6XCrWDNkdtZrUScfieaLvns/ZsrXDVP4pWcA6JZg0sTWmlQusWQbHPR1SVvBUD1Xo7mJ3seCqdceyB62mg9YngoZOpbbOLIQi6GRmpVRAdzJ/eIqKJMQVwma0ulvdBxukZ7+1zHosZkLr7VLtHW/hkb5Pg0K6m+WxC6EfePnzO+flbxjnid36lfAyG993cG3sGikmuIl0z03hknB/56be/x8m/oRPJ1wXfEw+Hz3l6euY4TeDshhLTgCrc398jznE5v3J//4a74zv+P7/zD3n37h2gfHj6jsNx5Pn5vV1MQaAr67qSt4U0T5Ryo1THaTrwcH/Puq44Dw+PJ/Ky4oMQ0sw0el5PExqENAjTNDJNE/M0Mc+zYfCKia4PhyNlXWC7EoaB6f4Nl/d/yBc/+JLzcOAwDtTlQm0b93ef7Zk4wuAD03xi/OXPzOs1T7j7t3RnQV3a9vGJNKbPfom0Kk95ZV3PeAGXImMazBnFfkh6s+YaLMPkKR9JVl27cUq67A4fy7af9gVKDZ7zzYAtYTwQ8oXeqjlscqbWjVEE1DLYm2ZwyjAGKnbj6Fotu75bfAJSKf3FtvcdgsxsWyH4RCmNVos9596CAPP6TGehNChtQ1lxveLEnDrGazDZX+8grYMkW7z2BdU9z0lsjlqKQ9Wgv72/UNqNGNzOdjB5k4ij9Ubu9rv5BlRoInQpBpmxkAOrprv9nk3VruFeTVyvihKoFcMN9l8EHqU137vH27BMqp2+J6/lGqgt0oVd2B1MvuBsKKzVhrpuR/Eru41vR5CpKrlUxK8IlSoNNBJCNmLPTuoWTOxrQ26xwHq6STfodAlGUdl1V62ZNYxg7gPvBlobsVkmBFFidDZUVlCJqBZTfrlIcJ0igmrAMe82y0QK0PCMcbLKdVW8KtfLjfhwwMdAaJFYIlsxPWYv0SQlpupFXEdrpbWJt/ef8fr8PfOd362RgcoVkf1w9A3Xm1W8ugt/tTDFd9wd3/LVt3+LU/iS2gulXqEFHu8+53y5cpgmRBy5VFy0aFxldyAFO/jevPmcP/iDP+AHP/gR67LighBD4Ha7mqXNN3Jerb375M3fULcP292Iat0xejdkPDAfE32fYU3jkbuHI3EYLVpiOnKYDwzjiHjHNM2AEoKna2U4Heh9JbSNw90bSr+yvf+Ww/2ReoysH37CttnFe3d4i/bKdDyBDIg6xvu3pDdv0bXhdhK/UtGie+cwoePMdPeWrWRavVFbR+tqkhnn9nm1A4QYo9kWd4mYAls2n7NzRrxxe8qgOKG2Tq6N255OmIaD2f16obSOuBe2vOFytENKK1u+WKSK/3gzVLP0fUwTEPDSaSwow96dVaQLvTuct4qRthBCA58gKKUsbFug1M0yq5xDYb9GzGtu2U02Buqlo06oTU0z7RwGdwm0IrtzKFu7HzZqtJuKikPEqr5SG2jAecu/6q0hfvexV8dWGr1nUuwEMYygD8aB6FrsvOmNruxWx9VSFX7G4+fjoNTdRI+RyZMfEZX9nHI4NWS9SPyEBnN+Zyk2uzt4nN1RJO6AVL/fvbp5THtkKwWvJpdwKKkbXqx3Z7IRESp7QNE+G6mtodXMUFVs01a70p1ZoMynK5YSp5M5DcJA6RmkmA1TLA40hURusO5zqxDMulWaZfYMYeIwHJjGQK4ruVmswbKtBjV1G1uGOAyM0SFDZMk232niDSVG3Dd/Njp483DPy/unXboUCT7ZlnuXhXgRomd33DjiYMuKWipvHz7j6ek75mEiqND0RlbFx5m1btRucRC5WMzoPA0s68o8TtTSeX19z93dHet65f7hSK2Z+TByPr+SiwELTPpltGz1Bn1wITEkx7auDMPAti0cDge8E0re6ONAGkaWS+FwOjDPj+A9w3wgpME2/MFgHyEFYpp3EbXuc9xg75PlAuszd49fclFHfX3PcHwgDRO3p+8Q8QzDRPAJNw0GR26Cq8CScYcT2jvbdYN8JeaF/PSexQcGH9HWiVHIa6Gp2/OnK04dErzhyrwVBwAWdRvIxcYfXa0Caq3tyxFHq4Wl3MjqmOLIqN2ilUUQTRAKcQ2UvLHmKzskzsTbueDlo24SvPsocAdBUDXQbqmZSqcXt/NdK+MYiKmjYqAa5wUCtK3SykbrQDUCV2MhOkt77P2CuoGIsV977ZTaKFqQWOy56MZ91Tru9tpiiZbdnDbqbCHanac5I/+g9vxJEARsoYS3+IlNKbmjMcNgRgzdv08jG4CkOtCIqIE8mv5CgHv3uws2J6zVEyVYFdEUh7e8lmacSFXFO0/0A4JRWYRAb9lClzAupWJzS5FooV47Gaj3wrYCwdppRBDS7qU2P6ol6Zm3O+dMDANbVTyd2q2iFKcGEO0JYcAOogEJnl5Nu+djRomoVFwQGoJXaN1arxAcrZjbpdVOr0IMR1I6saxX1nZjCEKrWPb42gjOBLQxJYZhYlkyrdpstLmC+IoTz/39Ix+ePyB+Ygz3ODFxewjJfq9ywfvA6Gx8kVwk+geiP3L39shyfsK3Bg5ye6X3I0O65839I6/vL/RtpeyBTuM8s+XM6f6e56eXfdbaSClxuTzxdp9F9n3ZZlEeDuc+bl1tbpuCw8eBWheGNLAsCzFEct4oZWM6Digmwja/fSEGwaWBw+GIC1bRhhhIMdLUlgnOCU0b3ge6Flw4Ii7S6pmeO3dvviS7gJYN5s+J8xu2ywsBGIcZGScjUBn+k7ZuBAfueGJ6fEd9BkonpQPnr36Ll5yZDycu18sn0lPv9nsaCxVqrfbec84IRKosy2LdjbP89U7Au5naM2teiCnw4O9ouVPZOHSP9kI28TCexjSNoI7LpbBtFlYXguIE2tqQJFSxOWmIHaUTvECPlGJxD703etl3Bq5S6sbdXSL5SOsF8Rc7qARKa5TmLGNeAs4NhiUMStc9bkSrFSXNfNYdpW/B5G0s0B3SjfjjnN9TCMK+6NlRG90jPSKYuw7sefQhoK3tvnGlVGXZLHUgdxDfLf5ln9m35nfOLKZ4cR+39X/y4+fioBRnlYyoQofSOuoqtgE0GkkrlbxuFGfotK6VGAdSPJKr7CglQWikaIy04EaTXQBup0Kr7FaypuTmwEeg47zgo0laem80dfsAXVmzzWZkHD5tl51UWq+fIKgtmXlfejL7YXcstysxbcRUwDVq3UCatae62gUdAs3LPthPtOrJq3I8zrQAxRVKV/z+9dvm7XuaX4khBHpSXJkQnYAbKAxjMoGwM+iqBMPXKdbGBHekYNVN9Heo2SRss+5PDOHAKq8Mwz2hqmUp84ZT/Ix8zpTr1ZxDCEOMbOvCNB8o22Z2zRBId/c2wzyeWJaFYbCDL4SwVzl2OPRusRwf0xx9DEZH0oaLAfHCst7wH2d1IqzbtjupGuqUOAwGA3E70Hlv6T5eBH2/oHoXAyiro0sw3NuysxOP7/C3G1tdCYcTIR5oy4XcN1zNxOnO6FDB4+cHuDzRP/wU0ozWFeqFvDwznh7xr09cn95/8jabIsJ/QqFZq23SlFr3TGsA9BNsA5Que+srjjgn87+XjVYzKsJrt9gLdcYtqHXDu8Q8zUivtPLCtl1Z80bwFSXjK4TRkdSQgIjN6T8Rh5pVWdqbDfsk0FthXfou7O6Usu2/Q8KpI68m25E+EFIweEUxs0Yp254rvut81f5udCP6EWADZp8UQZujlUaKgTElSlvptVonKWYH1p52AlGAHnA0+s5XpSutll22VHYnz2ZL05YoWWjFfncnjZA8QX4BdJQijiHdI91BTRQ16UT0CUTYSuFyeyWlAyEmA2BI2wfc455BsmCXeqXt4WMuOgvV0v0g8kJTS2XcAwVBJxSjAZkftWHoewGC0ZmbhagfZkjJ7raAgTGqbc63bSHKgdGc+4wD1LLavAXTVfbecQFEdzeAOrzrxPix2rB2oBYhbysxftyAZhOxq7IsG+I78zAzjndMUVFdEDeYV17M/dHYULcypkgvns5Gd4naTNTuJBHcHaKK54B3I8OQaKUzpQP5tXD0J7pUnIv4aoLmGDzPTy/mz1YlpmTbdoTgPE/Pzxzu7nl5feHt2wdqrYzjxPV6oZRi1dKOyOsfFxjOm5RE3E5WAkkj9MoYg1UqilVdWIY4AmlIqIuU1gnabWDv/X7BOUrOhBhpecNHj7qdAN6ErgsiHq0eiZhcRRw6Hki1kbczYbrD9xG9VZbnZ+tMTm+gZigL8viO8s2GfPgGRViXM8vtA65snE4j2/KMdAM/mJzR5pKtK1qqVV8IQYwo1VWpZadxE3Zpj7XGgB0EaoTy2Qc2Cdylma0UlpbZ6op4CGEGSbixIM1Bd3zzcmOTineKtM4kZsMtGYtYcSBud6/thzVio4rWIfnZKrYejG7v1HYJPRrjoDdKseTDtqPWnPN41T3muBoVy1mcRAqJGIWugZxve45UQ2gWdTEMJgWLliaKNlrLVL3Zddwjpa2gYlmhuke/fIqy3TmyNHyw4gY8vWAjhSpo93QKtRmV/Wc9fj4OSoQgI0M4IsFTK9R6o7dKlEB3Sqkbr5f3DCntnlHb7Do8NE/0EyntsgQt5neVhahHaJ7W7E4e3WBMxV53HSXmEf249Op+h5xWdK8005D2ob1JgGIEccJQHOtiMa6tdkrpDM4RUmB0jtZH8lrwznRyvXZqXT6J4nuPtFopxZiRvSrpXcJt+8/pH61Zq0EGdGQrK+frRvDC0BPzMOOHM9tSkCoW0sVCbRkfo/lbxYjXfp+ntiYEn4jB04rimBAGDtMjPcJ2vdDqaro3hLottNI4zZ/j9yzpVhd7HUQsh8h71nWl7fnirVXWNTNNB0opjOPE87NpJXU/0MLegn58GxrhpZtvF/Ap7RKnRvKBIUTCMCDeEVMiTQfGwz0qidaVy/mV0+mBEBw5GzChb313MmUSA8551vVG8okWRkL09FLweOgFiZGWHcEr+fweP9yj6hnEc3v/HWlZmN58DtcP1HLF3z/C7RsoGy5OpDe/xIff/k+4vF+4f/cD1uVmc0nv7X2nShpG0jAyDAZsqb1Qt0zwHgmR4Dwhhk/tututubV2cq77YtDhfaBtZ5bliWs+05xY5neKFs+gQnIjYygkP7BsDXWCVyVvlk5o1XaiSkOlGGXLZ2QH/NamBD+CenrxrBexnCPZ6GpLF+eVlKCshVK9cQ3UNIohOLwzurp1bODwe2LqR/j0nglfL3hvxor7+89xLu166MAQE5s6aAveOXywgL28WShc8H5fx5gzTcR+dogWJuadw/WIlyMqAx8z0EXUwgh/EVIYUSEy4zXg/WAUZAKLnumt4MQTvHC5PXNZO8MYLQiJgRhmINK1MseE+EbtC2t9QmQj+cGkMHm0ClLcDusFL5GyFbwMZj/UDgLiFNd2irizeZenmWbLqc0HgZQCeVVK62xrIfpC9nmnMcMwOrRFRLJZtVSsstgpzWAiCtWO60quK7flYjESLVkWuOuMaaSWTBgHcIlcAq3OIJGUZoYwscgTt0vZ57ygamFqkiIFbzkqIYDa4Wyw30AMA8EdCX4gZ0hxROJtjxZoSGtILUQdSc0OaoP/2utWcsV5YZojy7oZ7LdUUjInUtglMMuyfJoDBROq7uAOs3caaszt87KdASn70oDOONh8LETTmoYYCWlEQiJ3xeXCMIzcbhcOh4O1uGJzya3YIVRNL4a0TqkrXR29Cb5mq56cp5aV3iFfNnpZWdfMw+MX1L4agUawJRAOd3mltxWZ77j94e8ZNi0MxMfPuH77jxm2M4f7Ey+vr9RuMOlg2BxaLZQ96lgBF8LuWXaEGIjRXiftZmqorYPzxGj51Le8cs4La81UHClEqlrkro8nG2sYCZOu1WRgss/0sEOsFnOTqWsGAXF+F4qbxdNh7w/HtEchO7aloM2R4rDvCzraGl4cwdlNU7vdjAs7uyEIbl8oeW/0oq6bbda1UNu6V53FyONhwHkLMGvOPOatOlu6BrsmRwn0NHBrlVz1UwwFOzIxeAsoDH6Pk5GRaXyk+yOrdtMJ107TDRd0B4f/yY+fi4Oya0e6gJiTwEuEcKBqo/UL0jrBD9TeuazvqeoYDmGvSBzzcG/ZNWpb3BQnan8Ft4G/4HzEq1n5SpV9XtLsRfaGoVf2lsjB4KxdSSGgBNOPiRCdw+/uF2X3V9tbi7yt1OFGGxJdHQ4TEofBm1YSdgz/SHU7gV0dIUQ0mthYfGJZz9zfHW2O2csue3KkMOEkMtwF8mqfD35mTEecN2DCzb2wlUaTBfGG6A9houPR2lEsNrdrpdQFJwPeHfFxwrsZ3eN20Y70hrRKKILoAB2SE3IrTEMiu0DLeRcWK/16MX1fN1HyNM/UUtDeSSlxu/0RNOSTdnAXgH+EYKzrxjia5m6Kw6fPy54PpD6Aj3TnWXOl9pUQzXrmxWRY4ziybRvjOJp0TB05F3qo+DhSykZQuF2vDCdP7QFZzwQPPoysy4ILgd5Nr8eauS0ZJFGfX7j77C0aFB3v6HWkPX9PvC6MhzvWr36X6XjgeJzR5Q3Xy5lhnIi7O8jtbizTDVs7GbyBp8XLLhi3wKayFRvPiMXihhRZS6EtC92Z/dHFyMHfcaJzuS5seWVdO8KZGA6UupB7obuNEJXZOcvf++h2aA4JA2YfLLYkwfYE3idbdOiAaoIWqNnRauDSCuOEgZ8dVo3tnALxNnd1Lpil1u3bZGnUdkZ8RBzWdWRPp+52zI5zBR8jPqg5xrx9b0Nd2A3GecEFIeBI0ZGLOZBqVRwJFyy61jkHvaKtouJI0x0+nBCJzCRqjvtC85Xuyi/KQdm4lSej5dCR2M0JExIDM1u/IgRO4wPOO67LB3JZrGJoAS8Tzk30ulo4FG2fNVaaLgQf8amZ+LyO5npoCrIiLtpGDkUb1N4tdlUcYxxQrWyy4nZJQjcVsImBBZB9m+gzpS3Umujqgbj7WA115kMi4kwkS6CnG8Up2oz6I84RJNJbRrczREWdEL1QW2GcjgR3YBgmdIZeG8EHvJ926tGRLrCUCz5s+F7JdWVIB5ITKqAFKhlLZnRAJvorhZEwzHifdtdCRXwjlxtNE6k5BhxaN0qx3OZSG3m9UUpBXET3N/BH6yZOuJyvPD4+8vr6ur9WtvE+nU5s2wr4fcPpWdeNlEbGcUKV/YAUE18325YKBmiupe7hWDb/DN6zrMunCtV7o2Q7Z5VFqYUFO1RVHUvrLNcz2grx9EBw1qpHqYQgrOsNBGt/r53nP/xt7r/8NZzv1MsT8eGB7kbYMkOa2FpGbwbxPX94Yj4MHMYZWjPPt+qnGXVMuxZwF8DnnD8tofpuvQzBWcsa99mmCNI7UQU3CrksqPcMITE2cIhlFlVbdL4/f8sYT/TWqC1TWMEvBN/wEj/pNx3R1BYyG6tLbgSB6CJOE9IDrSc70HrYI249rayU1ojJE9L+75NOTAFPpGNysRgdQ+jAxto2tF+NDC+JXBrbtmfWtIbDrIg+wDBGkEIuz2yLoxS1cLaW8UMCOj02AsI8dLzAuqmBuHsC/dgNGR8zxiPR3YEmvIu4NBHdQK0FcTckOoL/BYirVVUuyxk/HxEKXSz5rRRrB4KfP919nX9Dr8UErl7prXApr8yT0YxLaebnVqvkcNAk4wi46PEKqoGuntorKlb2L00RvMWUmq/f2nNvuDLtSs4WXOZ8R1TIpVG10X0w6KpuVBbWJkSXbJGEhSqFGO1u2qxVdS6x3jZUPVE6Ndsd2TPhqEhfd5vhR/+4cLx7YB5PRB8opbHVK70J0Y04ORLijPc31rwxOE90QtkWYhwILrGVbKAM6bvX1iMy0LdX0MA8nCy7xANdcVER7Th1aOmUXi3PGUW7UlDomVYtE6Zpo9KsRXa7xKpWSil2h8fCpWKMe0VpGtoQArVeePv2LXZQmPB6GAYDSIjbJS7B5nw2pGMYbYt+f39vpB7fuF6vzOPEqp1pmrgur8QQyLWyLBu9KfNOmsnrQpxPdOdxwdHKDQdE16k+8frtt5y8I64vnL/5x8x3J/qW6duGU0ebAvWbV1JMnGtjzZl5GliXzDRESgpGsVLb4tqYwYTRxke1hUcIAXFxJ42bYwo1p5piaaTGP/UUYKAR/MZ2u/B8feKyPVtiaQvUsnHJGzeXcR7D/JHNM01FAogMjPGIU0ftlYpQu33cS8PpQJDJrI/VpG6lbtRmtuDWbcNswXQmg0vBTCDm1414H3aPtu0gfIOtNpreAKFmyNnvmfSdtKcsWnrihe5ncla2NVIr1hWo6ZfH4C26wVV8gCk4fAjcLo7ezOkjzhuMtzmcTKg6UhhwMtGrZQmJFLxv+GQqjZ/1+Lk4KFFlKy+secCPjq4TIkbQLuVm7emeKZJc5PHwyCU/m9i8K0teaZoNLe86oyrODwgHul73WIVKiImwe05L7tSu5pTxldIKjonEhPCxJSw7+NdTc7b1oxoLD5SSyw4ecKQhAYXSFlxx5iCK5l12qnTp+OhxPuHDniZnhSyOSt43406dUWSkIuKoapvaLV9I0ZPCyLizJdk6tV4sItYfidxxSICOlPYdPVeKNnq94SVRq7KtOyNTVnxKSJzxbFwuT/RaTL/mKo7IEBKuJUKbOMxHhnTHtmSCd1TZJS3aGeJgljiEOAyWcbLPIy3B4I8gIMMwoGoEmW3bOB4P3G5XvHccDjPblonRdI8hBFqzRVjQvT3PG6JiovK4k7n39t4OXHNgsFefaQ9zG+NocQO+4VrhdPdAWa54hGVbGKTQS6aVjOZCCwOH04HLd98wiOP63e+BfEnNjgfXqL0j92/QcWB9+o7Tm7eU8xO9LHQa5+vCNAxc17q32fZ0+X3+YGJ7c5q5FkjeUkNtrGOHi+sOSeETSMKroiz46vB9n2v3wq1v5N7IqjQCyXlyW9jKgtAJTg1A7ZxFS4RA9CYudz3Sa8HJYG23GhwatZGLdI/Tgki1TkpNsC3OmcsmGDAjJRN2d6f4EHZij9kHnQ/45umbI+cGYhnkrQxo340AzvblFkqW2bhSN9g2iwHpu8yv7jPUcTAdcncGgBlG01huN6NR+ehxOuLxeB1sOx4E10FDp9WV0m6It7QC538RnDkord/YyhMxREIZiC6gLpG1sOVXhjgaDqkXhjDRtXJdnsml03rlnG8gnTRYVTg4wYUBrR3pg4Wn+04aguUro6g484zirI1uFY/iu+C6be+km+tGnQl6ezfUfuubDbE7ltHdlDGNtCpUb4mI0ject1loa1ZNOBcQIjF4SoRSVsvqjgJd6LWSeyf0ZtncJHxMaF9Y85X7ux9YW+gDQ5+5rs+Iv5D8xMC9AVDFUfvC0m4kBbQiLJTiuK7NIBqhEVzG1zPJVWiV67WSQuQwB4JLpjELA5EETejNBvK1GCk6RGEgoW0APClGujiWdbVsmhitCtwrSxGx2aHxvxiGtOsIK9M007syjgPjOH1qTWM0qcjHw3Bdlk8H7jAMjOOIqpKC/awQKikms7B6Z2MX7yk571xQu0l1OsM4kXMmeYc0I+Tn5cLy/ITUK3zx55lPR9y2EEJiffrA9IMvYb0YxOKnz8gPf0iRTv/wDTEIr89X5ilx3Ra+//5sCY0p0btBLqz9tt+n94ZmC9NT7UhIxqf0DvUe0oCEsDtLDLfWc8EDd8cTLgz4aSa9HPnD/p5t/Z6NleQEbWpSIleQXol4QrQZX/ACbHaTdg7vIr1VVBbEt08xJNotEsOWSYVSy65h7jY+2vWp2pXeGi56hALO9LiORP8oFJdk0rdsUjDTZu5kosHjoxKD4MW4rZfrB/KyktdELWDJqQkUalWSu2cIB/AvtgwCxsk26rdbsyWjwxISekCao+diyyOtNIqdCV3RuuF+9tL75+OgRG1Tu5YLMZwIsuKCafvGWHm9VFQvBgft4P1ICgkNM7Vcd7dNRrBDUWRA9rhbK+etSmo1Q8g4VyEkEkCx5UYXw6552wnaodbTTjVyRoVhswG/WuBVa5XaDNfvoonNUxigV6o2aq5IrHSXqdmAqUHsTq1qOrJKN/yTrzZAF08tcCuNKEaB9ntL83z+lrePv0xnQMg0rK0t5cr9Abs4mt35aYmC4kpHRfZQqbC7jfalNcYhVDWgh7aMtoBrgegdg4+4agdkrQ2vDemN3iz3RPo+rghA6LhgGshhnBjHGdldN947vB9IySjUpWQ7xIJlXIcQ9irSM88T3od/4qD8eIjGaH8veE8aEtM0faoke20gcDweiM5kL0OKOBcJwRt817ldv9rotZBzhuARHey16oXkIte20tZnlh//FndvfsDjNDEf35BvzzSEc6lMrrKVG8O3XzHPB9ZLRnojJmvp3372Ba8vZ/J6MRdS9Lsby+064F1kj8NkZ7If9HvOizS0V3qu+yFZbEao5hzzzjOkyBt3R2uO0oTSHVv5jpftTG4N8YPFJEgh+AqYKy1rRXAE8bt22BkUZm9m/Itq6QABAABJREFUnDS6mkqht0D/ZFqx1xdnfEkzhCjiO7kZvUjESEk9KCEaUrC1iEgjhAlfOq20vYNzxOg5pMgw6A6PcVZ41EZpG6VizjZtNvt0M0Mc6d2hPZke069s+ZnWCzEG0qh7dZoAw7fV0nGSQTrd7VVkUIsb7p1Wtp95RP1cHJTWliVa92ylEV3BS2H0keAdKQ1c8yvNF5IPLLkwDAdiOjD1TsuKV4f2QtSA64IQoBtnrmRLS0QcrVmVh1sJccSHRK2RbTUvrBsKjsE8qNjW2+AbkGsByXbINHa9J2h1lkcslg4ZnWfJF7Z6sxlfMGI1rjPHO/Otoladyghyo7PaDAtvtOxubQK9IW3FEbnlF56e33Oc36Jkcl0o7cKyvEf7jdIx6rszATFEEx6L8BEgHD8qBQ4eCca7pGdEEtHbxj+ow4vxHKMmu+moErwQZEBbsljSlk2DJg2354gjdgDawauo68zztFfxH5cYfDo0l+XGOM4cDgdOpxPDED8tc2IMDHuLHUJEVRmnkeA8wzDsaLaRlBKtVrZtxQfPEBOtdmJIxJiorUI32rw4oeVCCsK2FXrOdNfRbaHentHWuTsdeG1X+nbhfLmynr/l7Zu3BNdol/coQt02une43snvP5COA9JmnArr9QNbXXn87AsuT/Y7IxBdIudMo++ADquAgo+kmFA6tdwQ5/a5+D6XRfAxEUdvURl1o243cq8srVC1EFHmlHh7uqOo0rcF9htNCsMuKFdyW1i2K1BQE8KizhHcaKmJpRGcgldKy+RcLSpabORUtUIwUIcCuTZCB6jU3G3WFwTUFBtDNJ99UEfyB4gTt7xS6g3nlcknxjCQQicmW3iV2k0I3zulG1dVRGi9cEiJIY30tpJXxzROVu3rK6WeCQ6GwZtqIUd6c9xuhRBtXuuisS1tMtRxAl7ExnY/4/FzcVCCzYC1Gzx2rReCH4lqcw4fAr441npFvSP0Cc1hF416osyEMNL7hpOGFLvzuWSSEnFpn1O6fUlUCFHx0eYrQiQeEq99JZdmdifXGLzHSSJFoUWTmdS2WcRCh15BW0DF09sAOtF1xGtiTI6lVFq5kvYh9JYzkUwIlVo9a6mU1qmlU5vlIgcXTcupDoiG9MfutqrC68t3xDjixFPqxpafuOaveF3/kGGYyf0KWEpeEgNgCELzHnGNuI8HYhTSaBWzfCRlUy2Dp4PLljNuyyjzJzs6t+VicAVvCwkX7QahPpKbkNJgsqRdUN73zW6MiWGw2N1tsw31tm0Mw8jhcOB4PBJjZBxHbrd1b6sH5nm27XUwXWmKiWkc98MzfPqaXtveotuWuZSNlGxhEryn7BvxlBKtZaPNhMj1fMGTyU9fE13m5fmZ43jg7vQFa1qR4wOhYPT9lPj+6RseHz/j5fxKHEyyFWPg8vyB5ALXy5kxCetygaKM00SIka7tk12z57xLcKy67L2xrCvihSGNBNktuNG24T6kT2qCAaH0zBQj6kGdsGw3mjTWvPByO9N6NojUPmpIcSY4i8Q9+MRxLlwuTyyXQlPBDUpMM6KOrJ21fXS5GQi4V3NTqQqlqEmxYjPorghLXgC322ML0hQ00JoNZm3c5In9SKvg+kcplO6ouYBg4JiumVrU9McaTS3wkQwmna0sjGHCayIQoHmUhNYD3lvsrMNutNqEkpWSlViNIhRKJQ5WwTsxR5ln/CRd+5MePxcHpbW6NkMUVUpLtthxdqcV5xCJKMptW5mCp2rAkQz6g1GCHCDa6bnRtJI3wY920IKyltVmJqJIjUi0WWFyI8LMYfyMdWucbx/I5ZUYHgguolTScCCVznZe2LRAM0eDdxFqp5eKn0+gnka1rB9/4Pm6obriPVSEc3ulDhDkSM2NnCtb6ahvhNhsY7gvPhCzW3oS0q0K806gFrrU3dNaWbZnar+ytbjP4Ua82yM8nUe14h2UboJfmsmfkk8QzAqqNaPqUcngG87vczRRRBulLqwtggucHk27Kar2798yrWbG4Y44TMQhYqiwiguOaRy4u7vDe8fr6+sOInCcTvcMu0PF5p8YtKB20jByOt3t23IYx9Eq0SExjCNpsJbbOWfe/mDvD+cc0k2cveaMc4Hr5ZUQzAvYVmEYkkUztMKUPHm50sorl+uV6By3l/e0fMNPB3p5hWZe5IJjHg/cunI8vWFbrpzPzxwPB46nB17ef0/vjZeXm81rQ+e2bsQQGJLpQlvbiVWt0XoFJ3QxJ9bgB1s4hoEd74Pzae+ECtozfpcLBReYd8F6roXDcmO8XonekcuFTSvTeDIRUJyZ5zuiOMbhRPAT7Y3dqL7/8C3n7Vu8q0b4UaXVRqkZh1oshO+mIilKL9U27EMkxLbraNvuZW/k0hBpMOzjrrWQ4gHVCVFF1eN3l5hIsWQC8cZe2EwPmVejdjnxeGfXroX1Kdt25Sae+8Pn0AJlW2gUao2EdKJzo9NxvhJTQHti24TSgC3Qim3sxyHikhETnEwM6RfBwijmX21aEJTeM7neCGXYEUvVRg3qDTOVN4SOY0Sb7NSbwSJctYNm2rLQCJSmpNkzpMAQhXU7E6K1fbVgxvtpxnMi+hNv7g9s5Qvef/gxTRcMQWVB9ENKXGWk5I5nQrtxFUNw3G4LZbwgbjZK+j5HciTqViB1vHPkvlDbE0MqbK2y5WZD/W4tkLoVHx1eDgxxxDezIZqbpxBio7UF76wlTulgDEBVlGyuh94Rn1AJgDf7li87NKCQxgNUm44JSlOrDkQjlcpSbzuLsBNUSH5iPBw5hDtKrsRxJKSBUjJdb7aEGUZCnAnJQt4uy400DIyjzRJjSFyuL7Rd5vP4+GafUw5ox3LcxaMqdniOkWFIn7adIsI8H7hez4ToP4FvVZVSCkOyatWHgEdtG177rkOE5XbGOyH3xvW1EQXWyzPdBWgb2+2FIdhMNQzeqnpgHgeW142tVBqCq8rxYUZLZxI4vz7z/Xff8/DmkRSTtfR0S1cSx5wGEBNYh+g/SZy8318zVRsLuG6BZ63iQrN5tff7MNlef1uJ77G1IbBdr+RqrqM5TMxpYk6RlKJZHWulSWVZFub5nnl6x/3pkbvjO4TAmm+8e/yS715+zPuXn5DLBeREbZ2SP9B9tU2xM4JPTJGuAaUaxWq0ys17IfgRLwl6pdXV3oe+kXumVKNqsatUQBlTIA0OL8adLdiSplSh9WQZ3z7Ye8Lpjmyz7f1te8FJQGtApNDUFkj7OBKwLbZQCTHQsUC/2m3GbpEQGwe5YxwejCDf/ozyIBH5ZeDfA77AlAt/TVX/HRF5A/wfgD+HRdb+q6r6JJYZ+e8A/13gBvxrqvq3f+bPwBEk0Ppqtqi+kfUMWWgyYuTlhjrwMVG3ap7pesPryDCdOIz3QKNuG9fywjmfwQvJz+TV5CHH8Y4xHsjlui8ehHVRUhDmw4Ex3RFkYEonoks8Xb6itBsl25YweuwCxOM14p0jBcH3TnKNy8s3HN2XgEOTEKLnMN+xrJ7ezgZvlUbpG6UVehMqRodRjVZEhGIHrCQgWIZ2K4hsiGRKfWUcJvAOj3B/+hFbu/Ddy2+jfaVRcNjwX90VFFI8ID4jGHxA3O4SagEXFOmNXBtVVyNSh8AYbSkydiG6GXqktk4cIj54arELUYD5cKA7axORTimVw+EAznR2Io5lNXxYa50f/OCdIfuD36MLGqfT4y4YN3+9VZCGYrO3lNG/DWbrPy0/erfqo91uDMNAa81id4EwmA1ymmZEC61masn0vLBtV3q+sRQlSKPnhea82SK9N+vcdkVfTPp1vZyZDieQzvX1mdP9W7bLmWEcDUJ8PXOaJ3LeGMaB1pQhWQSCiGfbMtvazALdzb9uI4VglXRV1Om+uGngCkqnO1v8GF9XLdqh7/KkP7apPc4zwzlaNU1FqfS+0ZrnfIFpuuftXeJ0PPH4+I7T4Y5cNr7++sds9ZVt+8CLnqFGpvFhP0hfcNGsjBoCtSjCQKkm8qbZZnkcJqIfCCTiOKDNnF+53OyaVLshBxVElHG0uAbx9m/UgjFeu9I04EnEEGymHgNa1/16UZrYvPTl9j3TMCPdZGBOZMesNWPV7lImcTbCsCiMZm28U5reuK0bzgVi+AGO4c92UGKBp/8zVf3bInIC/paI/D+Afw34f6rqvy0i/ybwbwL/c+C/A/yF/c+/BPyv9//+iQ+HMPuJ7jLLfto7EXoVqhZCMEqzQXI7eIuirdoJcsSJmLZwOFDHzK1UpN6oPUNXA+oWTzpMjGlkC+POpuzkknl6eib4I1O4x0ehq9GUg5/oYpIIbQs+VGLItOKYx4hXb09gLlSnrMuF56efcv9wDzIR4szD4Y678Y7Xq7K2GxITQkW14CQQg+UX20PpFXKGMVnL2Rq2mHJGwK5lYVtfSXczdM80nkjv/qvUnPlw+4d79KhnXTIpOcQXBleZhhnySu4rZXvFyUTDaNvqOgXTlHoaXjOsr4ySoA+UdmaM9zSf0JpZt43eYBgsesI780l3lNYqp+OROM5speK9zXZVlW3bePf2C1KadnueHRT39w+ICOu6GIF8GJjG2dpUoGn7dChO04z3OwZsn/kBiHOfhO15F3a32hhiwPfCenllW15p65mgjbIuaCuk6Ujd8h6j4amlgROSBMbJDvttuXIYJwQhHmfWy4Xz8wu+V3qreIEhBesMgNIq4zRbtZsz0+GIT5FtyaCdvG3myPHeli3DtG+8R2IawXlqKXh0ny0mWi+UspoV1gdCEFxIFPEs6vApMsxHQhqhebY1s5G5OyTEVZ5fvuLt3SM5f453gYf7A1N8x2mMdH/lNX+LbNA1o9Rdn3qg1c022cETBZpXcg7IXjF2ijFOu/ESYvVEGVicY62rMUA1IJgeNMZISoGYzKaYt25ys5J373cihCPJBXpTvERinBBN3PozTS/2nDCAemoLBlppjUEaIQHY0q7kSikdJ/aclmyWUBP7r4jPrNvXOOeZ0md/toNSVX8K/HT//7OI/CPgR8C/AvzV/a/9b4D/YD8o/xXg31NTHP8NEXkQkS/37/NPf+xzl94drQmtV5qsdDKNROk7mdtZxoaPHa+dUgsiK8MQGMeZ+/s31Fo4bxdKfyX3RukrU5yBRNlgnCPTeGd5NGUhRMfteubrb7+i98BhuiOGEcSqm5wzXVcggyukJDhmxjDaMLpXatuhF1Sul2dc8syHA45EcjMhBYITPly/parH+cBHHVtI0TLCK1R11GxAAq0bjA7x+8C7f0y3W7msPyUdEqfxhwxDYhwDP3jzF8ntwtP19xDpVAyDNSabf6kKc7pHmjN4gwQcwaQ+wZB0JQuv65UaC81PiDvhXeA0DNTSiN4yhEpppDTy4ekDp9OR23LGBQU/cv/m3S6atvYypbRXUyvzNDNNMw5v8pXSOR5Gy1P3zrR6xTSWHzFZIo6aMzKNuz0xMAzjDkow//S2rJ8WO61W1lqJIVBLhclmzKdp4vL+p9TLmUIGGnlb+ezNI80rm3TbpvZqEqqcyV2ZppOJv9UqurytvLl7YCmF5fpCXlfmMZHXQgiBlKKBPJwlb3rnLdZhjzlBGx+p7h/nr7VZJILz3sDTIZlWNgUk2HhBnIKL5l/HQVNK7YYB9JXL7ca1F7JAbdby9poJLjGOQs0Lf/jTf8AwJo7HAw93M4fHmYf7N/yo/grn9YWXp+/Z+oWmK6pGZ/dqWEOPyducU4I/oGowtdwr2y0zz/dW3TVPd4HoRg7uBHml6j4CCpEhjbjQ8SGDCK4mtHZavu1Lwkjwe56897AfsCk+MsaZpX0HLht/kmV3AnXwhpnpKkiX/T1vVmNz8SUQtaGIOPOLRyVoJeffp5TnP9tB+U+cZyJ/DvgXgb8JfPHHDr+vsdYc7BD9wz/2ZT/eP/ZPHJQi8q8D/zrA6S7uLQX0ZpeZtkxtQg+7/c2NBAZQy+RAutGZ25lh8KQ07YdG4u70yOv6LdQF0UzuTwQ+J68OcZVh8nif8NUgAD5kXq/PLD/9He6OR6bpRHSj2clQSn2l9RuI43g6IP0INdo8tK707brTzjtVlfX2St3uifNbejeL3uTuSPkJ2kRVtRmVmMgZ1z+RbFoL1kK0Zni21EG9eal3WHBpV16uR+6OPySFBA4OwyPvDn+R1hovt6/o2gw1NqkFKfWO86PZF7kCpjVV2ek/Y+Iwnmgb+BpIjAx+IslIK53kPMF5cssc5omcG1/+8J/jfLnSFMZhIAyzCbtDgFKpXdnyxrZlnPP78mYAhJeXZ+7v71j3uIfWmi2gdrnP+fLKYT7u/ErZn0dPKYXj8YBqpOTN5oIpcbleSDVxnGd6bZQt44LjfHmhr1e8Vu4eH7lqIS8NuhDiyHpdcD5wvHtrYJS6Um+FXAtJ4Pr8HT4K1XvGdKD1yvvrjcP9HdM4kKRzubxyurs3rWNzeD9CczTtuDTQ8bt2NNpSJDaOO5gXhHmaTRHgjKxjJKM9SwbDA/rgcCZ9AMUIWL1TeufYBk49ctoac/zA3TSz9shty9S+UlvCe+H19sTv/sHf28lLSmkbp/mO0R94OP6A4/GB5/NXVF2JzhE04oaRVhWaGE6wNVo3SLXzO61rhevtlSnNNgf3jilGBnmLyMIFWxT2Zs6kNATAUdpiRVErZvVsyuQDfavkvjIOB4JPIIHkhMP9HVlP3LbvqH2hN+hSEW+Er2EIxCTUUqh1h7V0MbCxHRp2w5FKiAJSTVHQO/n/VweliByB/yPwP1XVV/ljAxJVVTEy5j/zQ1X/GvDXAD77ctaqnsDM4GxY3CSzG17pVSit031FnM0Jg7f4TVXHki8gjY4yxpm74yPT9Y68vuDEUcvKZfueKZzIr4LqHcMUcWRUinm/e2BZvqb2yEN9RwrT7rVtSLfB8eFwxzy9o9dAWZReGpnK5oTqhDhEZm/zm5fX98yHR1IwYosTYUgHNFezhZm5zKQq2vEeW8a0QKtK10reCkEyMjhrztXIPLVv3PIL59sLh/SOKIE4JGI88Xj4dWrtvN6+ptRMKRuzHFGxZUTvSqszKU27fKQjPpuLRiZcjIRyoBchMTDJgek0o1VNtkGku8DjZ++43l65XM/cnR7xPtJ6Jw6e5XYlDCMdeH094/C8efOW0+kB74Xz+fVTLALAtm27cDzufzyvL98xpESplRA8l8uFeZ6ptXK9XhAgb5mSC8fjEVTp2ti2leC86fS6EkNgE3h9MQgGzhktB8c4HiEEO1S94uNgCzA/wLaRYqSIp2wLzlV66Tw8PHK9LZTbRl1XTtNkXvJW2MpGVzie7gzhl8t+7SheYGuVngt5y5aZ7c3Pvi4XeisM4xEh4F3HhWD2z95Rp+B3ze3uIFIxorm1+8I0DBxPbzjevSO8/oS4BdrSUVaUCcWq1OfzT/md3/u7DOMRBOp2JcrAKZy4O7yBNNCvyjQlfJiMqOU7PWPvSxHEBaDhNDI4hxvsfbYtmVN6ZFIbF8UQeONGhpa59Qp+BJcI3hHlgHTPUislr7RVoCWqgOpmrXirzHMiDTPiTXo2yMiYHriW96zr99R8s+fHfYxnSTb+KHZu9AatG3DEx44PnU6xWaUMBoVGiH/KSfjPdFCKSMQOyf+tqv6f9g9/87GlFpEvgW/3j/8E+OU/9uW/tH/sT3z0rtQ6Ed0BFyqlLUi/UtoNY+MppYltur1lE/uwE1nIfLh+xbv2S4Qt0dWI4fM8cdNAzyutbdyWjA7dmHs36IyEEChrQU01vreimVoWPLpn4yhOEp89/jr3D29IYSZvlVe9cKtncqtICMgUEMkcBVBHp/O6fM/DKXBeG943I6wEjxYhSqD1+jE6fBfldrrfrLVSRy0bOShxNE1b6xYBColtW1jyKy/X73j38CUhmLMlhsjb+x9R28q1fMuy3Eh9oNWIj8W2ymGCZoFrMSScW/ZNslkWc82omq7AB+Xl5QOqEDHpivMj5/ML799/90kk3rTRu7ItzqpG71hutj3/7PPPePP4jmGYeD1/YMsrx+PRDol1/eTXnueZ1vQTgm1vplB1bNv2yalzPl/ozSI/PhJ5nN/nemqeYctGX3ACU/BoK7ReLXrV2WY8hADiqNq5Xq8kX42iH0ayW1i3glNlnGZy3gBh3RaLl2iN1jvXbeFwmJgkEYaBbV0otxvh4Q3jNLO+PtPV3EkxRtZs1ZPh1rpxKUuj5EKplTRNIM2yg1zAcrQTsnMsu1OzWy6ZUhaDakhEekVqRlwnzhGfPT4H2grrthKDzTtDEN5/+Jrf+b2/xzw/MPsTGip3x5lf/8Ff4rZe+d2vCkUKp8MDKRrk43Z9pbcLdbOcHx8Ni6iqxCGh0slLYamZ48EyoMxqGglhIuWO7pnbsZtzRlralSyWFqAVtBS8sxnj1iurKkdpHKZ7EoL3R47TA1N7yzMD2n7MVm8EbwmneTMwdKvN8IbOFkjiTFmAGAugdYjY7NSsuj9bSPnPsvUW4N8F/pGq/q/+2Kf+L8D/APi39//+n//Yx/8nIvK/x5Y4Lz9zPgk0hVwDc5oRtTha7zwlq5XYH4WuzjyftXZCtCVQLZ2NJ755/jHenVj2vGe/e1oXCqWt5CLUWpimE+Vi0RCnwwkvyrZ+j5ZMcJPFKWg0bL1UtHVOdw/88g9/lcP8wDwduF5XbsvvsbQP3LaV2gqSItGN0DLeKSqJtV65rk82d3Idr2rEE9TM+Wrqj75vcItWXNzM292My2c3hisigaqO6D0xHFiLLb22duaaT8Q07lthSOHAu7tfx18GlvXMub5wmAbCvlWO7o4Y7lANaJ+I7o4UK+pWWl4hJFo2eMK2rjhxhJhIbkIk0Erl+eV7kE7OG19fvqK1wsPD55zuDlTtnzSiX3zxA948viWlBLKLzNPAOFq7eb1ePwnBY0x431jXlfP5zNt3jztEwyylpRSmafpkgxT45P8+Hg/UzVQFivEce2vk7YqLgVZuhJTote9JlLb93zazqKYQWa/P1LxxOJ043j+wvDzZAZQCx+PJ3EXO2JchBLYNtlK483cGhOjFhPGAuy0EXyxypls7+BEzp9ostlYt9kFEiCHa9niYEBzaGoRg2sTbapEd0kEsTsIFIXbhfDlzrSbqd71wGA7cze8o7ca2Nl7Xi2kdK3TpqEa8F776+rc5HU8c48z96S3ewcPDO379R78JEvj+6fcJcuLu8MjjfWS5vPLhwzecr++5LR9Yy8YwDIQYaCJW8SbPLS/c2o3B31sAYAiEwZCDGhJhMM7qx4RVqkeLw7vEVjdAmY4HuvPUtrJdr1yXzOOD4uORYbxnCAEfDngFaqPWn1gUBMKWN1otNhcmIvxRhpJzgndGBPvYqaqPEHZP+p/loAT+CvDfB/6eiPyd/WP/i/2A/PdF5H8E/D7wr+6f+79i0qD/DJMH/Q//tB+gHcoGGkZbBKjsAAoLU+pdLKujOkIQWt2oi9HIS6843/ju/R9wGB5JfgSg9I1WN1prlOYpmtGecVm4n2d6cZTVEG7H8S1bXvEhk9wMGiktkoYD0yzc370lhsjpcOTh/h1D3Hi9rHzz/Tc0zGJFNdukBoe6sHtNhVs+k/yJpleSOEQbwUOTSpeGiMmMxmlimmDZAlux4bT3gvf7ckcUvKdrIDiIKZO3K+HOs+YL42DYsfPV4wTG4cRb+XWu7gO3/BPWXDh4jxehlVecqlVPnBjcgRiVKq8GMmUj+JlKI4SRUaKRrmVEe2O5XoyPOETm+QhccBKZ5onr9ZXLesOHgWE6Mo2HTyCKnLdP3uxpmljX1TS0LhgKLiRu16uxPhXWdeN0OlFrtRvJvtUOIVj8hMgnX35eVuq2UbSg3oFY7nr0gdIaKU2U9bp7ek3ZQCv0tpnuFEcaZrZt4fz+yni6Y5hG6rqABLw3+VOpGz7ajPl4mMm5cLtljseZIUXEJWrZRfjR2mPp9hrmddkdYsqyZrznEzhEnGUxtVzMg1zFGEgh2iKlNwuAs4QHC+dKB6bJ8fLyxG05s7aK1kpKM+N8x93xwnIr1K2hwYHv1FJ3qyn8zu/+p8zDkb/w536TeZyJLvL27p6if55pOrEtG8EnDscDh/EdU3zg6+9/CyeNLV/JPdN6RuJoVWPbqFF5zi8kn5j8SCvQXNtBGUac6mJaadUOO0w6OEWHwJweeTh9QdsPvfP6xOvtA7lUUrjjYXyHZujSCM4zxhNJjvS60bXQFNuidywCwptLSDF/t/j2iZpfxbpPJBL/FCrGP8vW+z8E/qTv8t/+p/x9Bf6NP+37/he+hstrZvRiGisU5xtBNrZmkbNa9/znZjncpRYLSwfSqDyf3/Pt0+/ycHoEHFkvLOWFtSzk3KgVxEGui+V3iKNsUF0nhhPDcEftZ0KYGIcj3g27ntFCoEQTMRzZ1sblfEW65zAdKOsJiZ1cK41G1WJ6SO8otZLzxlI7vW/UXXeJNPCW9RziCelKr8owThxcQPzNEG+4fSxhWtM5TkxpQnwnlUTJF1pbEJ1Z2tn8qlrpdKPQkIh+Ji4Dl+tPKGXBjwkXHblWpFaGIYDOxI/hZDGy+RfUZ7ZyxWmnt4jLK9I3tFTW7co0D9wdH9nWDOpJKfHy/MTTy5n5dE9KnnGYdkfKR4SVGKdQhNvttufmhE++bZGddo/FIdRaPsExbAlkFWDwFu96vVyIj4+27d4xeK+vL+a6oEOr3PKNaTAfdSmZXgs1b0zTYJkzxbLTW634GJnGgV4K6/WZcb5jPDzQqTtqMZBEqCWTtUBXjocDIKzrSohCColpsA39uq0s64ZW8xRLiMTeIUJMbec6ll1QvufgqtFzWi9It64pxAGSxTZ0UXB+l0c10jjxKPb1t+cPSK5EHCmYG+fNfWe5rNSqBBFTQ8CuhX3l7/79/whtjl/5pX+e43wkSODheCLEwJY3rpcXtJoxQw8H7vJnlL7Y5roJuWZyXk0CpA7BRPRPt1eD12ghVnPBoJm+2rZ721ZKNa0oFijA4fA5nz3+OkFmWi8MQ2NtHbdmeof3T1/xcJxMceIdPnRUIsP4SNOFXhy5XuzaEb+/1xTZ83D6njrZKaQY8C7RNTCKxV//rMfPhTOn98ay3LjGK8NonuAumZM7Ao6npxXpAe8NS1abzelUMypG/fAhcV6fcb4zDJ5NXyjtSu/Fqg5JpGjyjJfLe6a39zagbw6tgegnet+IYWAa7pjGB7p2lu2JZV15evkeHyakBy7XlS1nesuMoyDqCcUbhYiAeI8PQqdzvS1cbwY6KFr4uPLqrVPJRHEkDrQKtTRCtFwU3S+aagQ0pEV8daRk0QEuJXR7z3J74TgFGpZX4uWG7v5u7xK9VqbxHuVC7wHRwJBOVBfYts40WXRvywJhwHNPiib+F2zTWdUqszkFNA6c7u6JwbHlZnlCfiAvK6+XZ9BudO7gLdWxG5KrtW46yd54eX3l/u5+1w4aUxIg50wpmRDMleOc2/3gw6dKEkxwPU0z221l29bd023SMe2dkheiMy+x9sbtdmEeEt57WpVdFJ8BZ/ksbkJbZd1uiBakVYYQ6aXgDoIQac5o61F2kXyxPJtltRhe72004H2j1MIwWM54HBIvTy/kUrhertAqw5hsUSOONBiz1AfPeDzih4la90TKYBpK3aVJRu23REO00Xci/7osnJfrJz2haiWEgcN8JArcUma5ZaR2k8yJFQrOjazbC3//t/8j1rbwqz/8y9wNd4QhMQ57UqILvL4+cytXJDjCMBPHidrORGde/l72xACnBnHxntKUb64fOE0nZmlGV/fBaPW1stVKLpmGEsZI3xyHwxvuHx5NPrY1mjbkxRHjuJOiGi/nn1Lmmd4rw2hgZ+cDaKJnpdaNUgvjODOmYSd8feRZ9k83mNZBNBgPKTq0/dlb7///P1RwXdnWhbodOB1Oln4YFC9KWZXbUhAi6EhrF3CdYRx2WEDGB7uDfHQusOdve2dofUcgDQPBO3Je+XD7mvt572MQEMc0zgxhwONx6gwy0VZezt/zcv6er777inf3v8S2VsRVVG+2bcOyTqQ5S/Pb9WZOPeMQcO6ekjutFRCjvXQxVNVWrjaLqgFcRVwwlxKO0hdaN91ma1YFll6IFjqCc8Lr8i3BCVq9uRJ6o9brLqBONHW0Hojhnhgf9iCvI2kQSn3l5fLBZDH9QJw+vpGFmCZqt7v+HAeiDLS8bx6Hge12RbHZ6jdf/xRtK70Vmg54FbblRj2t5M1ufK/nZ0Tg/YdvWZaNYUifiEFut2CWkmmtmatnl3N8jHYopXziWIpapTXERN0yNzkzTJNls3vPslxgCASxmVSvnfPzE9Fb9EQXG+nEYJtSiEbPnw+0slKXha4WzWs3GmNE9q60ribV8YmSYS03ttvKcTrR1VmLmCIvLy/M44R4I9zcWiOFQEW5XK+MQ2TaeZppHAnjxHR4tByaekOI5vn2nlYqtWRETEjfS0FbpeJZW+G8LDxfF54vL1xvF5a2sU0WfRHcbDyDFLldK3VrSN5skegDwQfW643f/u3/lG1VfuOX/wUOesTHRq/dsuoDPL+ecQni6DjqI/jG9fI9Ulcj9GtDveIk77N0odfGLTvwMHWTPfWq9M2YkOu20h3EMZBkYD4emQ62DW/9lWW5sW0bUxyZ02hR0U253M7UfuMgE94NOC+0Uu3GkW3h45kY/Ix4qK2wbKthEmvedaiCYuOtiumIf9bj5+Kg9A6cVJwI67IhPBrBuFeabjw8OirKettATc/nvDDGYBBU3eit4cV0WsEbQbvWiIozFl7UT3OecRooeuG8/QEea7VjNA2bXfxx345VvHgu14Xr8h21Xfju/e+ZoyQGhmg+ZRUFNec0dOSjJlINfBq94AfdoRMOCJSq1G70Z+FquTt5ZOtCTLtgtgpaDFvfVbluGRcrM4M5I2ohs/Fy+TFOR6tsxQTW1+WFGAutjtTa8emIF5imA8GPSC/IwfP9h/d8v321zxvvGIeTMScjhOiRBg7bvsYpIQpPzx8o25m701tur2e0VRwedcIXb3/4aSw+pIGUPO+//5pxHHl5PbMsC631vZrcvbxdyTmbtCdY3k2MkVrrp5nmsizm6Y6RZV04Ho5M48T58sqyLHRpzNEWFYLw8vzMPETKtjCGYFnqrYEI0Zt4OpdKjDY/rS3jnDKmgTAeydXkKUk7rbZPs1CcUunEFDlMJ/QqXF5fWVk5Hk/cFosjnqcZWue23iy/ZZfTzdPMPE9cLy9mA402yza/cQY8jkit3eylMVi0gTizsjqbv3ZttFbR3hmHiftj5yXfCNcrfcs0KfjZYn3VNZKDpspWMuqELkLL3ViTXbmcX/gHv/U3yDnzF37lLzONIyFArqvZAL2w3S5IEkY3ke5+iBebh5ftitttkw5BNwU2QnR4AirRsmowSR+9EbvitVO1ojjGaSYmk+OZj9+cWtt2s3nw8YgXj49ifFXssPfBmVaydrR5gr/HS6BvUMWRosdLILhGVaVVc0v5MIB4smu0otRfhIPSOcFJo+QbJY2cX6/MhzuCPzGkzJY3xrEZLSc7cjZAxRADzttddis3Wq5ItDQ27QeieJZyplV2ms5OItlnZZ2NFCLIzXzJFLb15VN2uFQb+ndVlu3Klm9cbgvjKEwMTMMPOZ2+5OXlhdfbdzaLUxCiyR+cfjq8h5Dw0SCoSmIcHLnYAN+HRpCCa5NJi0pFXMNpQHozfd7uY73VZ6YxME4DwQtCI5cCfcXJQHczznvW7cJ1vdBzwvnEEEZyEdy2EacB0UCKEw/37/j+5ac8vTzxcr7ycFqZp4naPIdxYp5mXAn0Dut2oa5Gjw5i8RjbeuF4GPnovDgej5yvV8Zx4OnD96TLPuudRrZ1QRDmeQZMtuJ94nCYefnwwjgOxDRQW+VwPNCaMSx7N8jt7XYjOEfvnWVZ9jGExf2uyw2KR2pjiiP5ejaABsp1vRKdIskoNSnZ4qjURq2Z6IMJoWOySGMXmY4zpa7QKx3ZQ+cs/dDvjEgk8uazL5hPJ16+/Y7aGsfDgXXbuN1ujMPAMCQkZ1queBEjgQfh8fGB6HaKU9ozhmjm0pVgP8sJLuzBbSnRe0VLIXoP4liWyx/pN1tHm+LoDNFR1KMVeug7oKJDqLiIeatLs89bgi+9C5frK//wd/4OtWV+9Ud/Hi+gvpFrQbQjHUo2iZt4SGFkHh+oEll0odRMr6upjxs0FEkGznDeQev0kne6e8cHSzY1dOFoWmi1yGUEhiExjzPn6zMvr9/x8HgiNMWnvSDQTN0tll2rFSJNEDfsRYTiECTorje1xXBfE5sYZ7a5ioueWv+My5z/Mh5OHIcxcr09o23iep15fr5xOEaCOxDCivedefZIFC40fLTZhcqKiIW0t56pPVOaR/YNMTqyLBe0ZcaxM03JTPK9M8REclB7QZwZ6XM70y6d4wTJV1QCjkirjpIbdBiCZ/IDD6e3fPbm1/jRuwO/+5O/zz/+8T+y5D3NBqWgMabOcZ6Yx5HDPJt0pTecG8lNaf1CFIMZeM1sRcmb4dMqkaqR3BarSly3i0M92RUDnjpBkiPrCm0jF3tj1ray3exwcy5R2gkXIzVneskMMRHdyORHHucvWPM3nK8XtD1RWuXh9AZtCZlHhjly/fBMqyvSBO8Tog5FeXh8JKSBXGwu9PLyxOX6QlpnQphpY+Pu4ZHvP7wnIBAiwzDaJn8nd6/rAnRqyyZZUrt5ihhG7Xa7Ucpmm/O6Mc0zry+v+KNnmAauL6+kFFiWDUpmHMQqrqqWwucdt8szvc+E4PHB01vHi0GCW61IdGgf8HtMg7ZqBPYKHSWEZJU5itbGVjMaMV/4cODt5/FTKuU0jdTWrbWsFemNYfA4RtZtoeRCkMR4fyJNB2rdNZ5pBhfoXSyrySez4Aj06JF4MN5BLvi4MCWhXi5stwsfrmeu242tlj2wK9JbYasViR5fKiEapu56vVllVa3C0rxTltRxPb/yW//Z32XZXnn3+BkpReY0IM4xn+5prVPbYmDrOuC7zQt969TmbNZeN3KpqHjEd6o0o0MhNBEjqotFNhfscMzbhTyeqX2FPZZ4iAfe3L3Bu8atnNmKQ4Oj933MpkYFMiShJ/qISmHb1r2wAPGOUWz01lqj5UqrjtrAKxSn+FCp6WefUT8XB6VqxzuYYmK5fWdB9hdbaoQ0Mo/vWLNHSicFRwqOrd1w2hjHe1rb6P2Cqvlmc14YhsnaOgKqnuu6sNWNirWuKfldbBqJ3trmbcss9cxlecXJAKnj/JEYJoQR7Qltiqjj/vgrPB5+gyne493Ar/7gX+R2Lnz78vtoM07fuhakO46zXWzeBYZxINdOrSvOCZVK9B1PNJJ5FFDHutpdt1WLyei942TD+U7e1MYJkxIGyy+Ovu0LnY2Src1sCnk1YXJZO2HwbBGuKzwcjrw9fEFyE90HHtKJ2+vC7bbgvGXQHKd7co0MMZLGifNl3WUuZ47TicNwoPbO+2+/YRhHcq5czmdimjid7rlcrrjiWW8X1lyZxpHT8URp1dq5YCCJdV2Z54llvRFj+MSZ/FhJ5ryxrisAIgbHGMaBbdvI2drRVjPRC0vNXHLmdDzy/Xdf09YV9XbwltLwPu4A2kLwnhhHulaaVosHjn+U45O3jRgctWacCMfjA3m9gROGaaBsK25Tel4ZxyPx4Q3L7UrbFuaPIIzzlcvllW+//5p5mjkcjohzrMsGvHDQSkgTfjjgUjLvMkAD8TZH06r7MmeBNKJOkCAMOqBTo98/cG2Z72+BWh1NdwarBm7rhpduagkCvW0cjoF1W1hX8+C77nfyjkeCojXz7Xc/IdeFzx7fAUdSSCQfOM5H1qxcLq8mh6ueXhOtZlrptJKoRamNHREnoJ1GJmdzwTRtFKqpHkRoRWltpbYbr5dvOR11Tx9ojKND/ETYMk42ghxsHyB9F5brDmeWPZ4YNs20XnEKuWTLr9+D6rQZiNhpR/G2MG2VvhP4/6THz8VB2QW678zHiViUXF44hgPaxTa+fuQ4PyKlGaE4CLfckDAwTe9ss71EzrfGuuX9TlZQb1j4NEbWvNLp5FLwUVHnzZoYAuJMfFxqoWoj94WX6/fQzYblxTOmI+t6JZeN6y3z+vrKr/3SiOBotUJrzOlA9BPRV4Jz1NpBIr3N9Cz0TSE4vJgervZOdB7YMOhsZJw8Q3zg9TWzvp73pEhsU9fqLrpvtNLpMdDEIVrANUKIFFEkKyV/lEPYAH1rF7ZNaXEljA76jaF73syfkehEVYYQWHqhoxS90jjj/SOtREQmxhle1/fc3d1xGE90bby8vGfdLozTBOK4u38APCWbq6bVjfNLQV1gSBaFEJLFmTrvCP6jBKp98nx/nOe11nahsP25Xq/Ms1F53B4hrK2gWkxm4wLTENm2xvl8JrpARw1v19T4hy5QSyOlSCmWkxNjxGnESWTLGyF4m+1tnVINgFtK5vX8xOl4R6mdjiURLsvC/fHEtr7SFkdt1h7npw8cjkeOd/coQimbSWmenwnioJnfOC+eNBxxMVqEKuBcMq6i86adzRt6WwgONEXEW+tbvSeGgdlvfH73QKsOpwNfrwsrHR12CVx5xR8mDqHj/JXWlTVnLteFbct4HXE4VCtjHJjiiPed6/JEcMLhcGe59C4gvTOEAOPE07Ji2VQD+NXI7erwkux39J3gTdGAF/we9VBpVCyrHbcTfcSR843eHNu2cn/3jkbDxcIYPWE4UvVmEcvR7xi5P/ZeqZZa6rwSkofeGFLYF2C2FMRZ/EaXPQY72MEq3tF/MaIg1KQXKTDOI1k73S80Io6ZWszJMiST8FAsfnUY7wnOcm3m4R2vtyfWbaHSUO8Z3cxhjIZZInO9Xmy7unVKdvQ4UF22UDHXUWfpcCKB83ImuJmxKCF4q6piotbMtgpff/s1P337O/zgs98weU2t9L4SY9nBBco4B+hCKw6t5k02colHKYAHdbhosy8XzIqX0swwfk7lW5bvv6X2lRihVLvIg1NohZJve1B9g2huJvGNEDrVG5pOnDAFZ15W51jEvsfiCmd9wm0dFwe0V7y3eZ+4jtaVp+evOPgH0hCJw0DvjcPdiX4trNeVroVtyxzme+7v33G7XTlfnhjTgWGYGeeR89OF+e6Ol9cLx8MdXpwlJ9KJ0TiNMRigd5pGlvX2iTX5CWy7awljLNRScaMpFVrv0DJDUNa1cFtuxGCA2ctlRVplq4VpGEg+EfbQMgRUC8Mw4vzHeaONYELwlGrJg+K92em8MKcD67qyrivjdOB6fWIeZoZhYimFYR4JOVM2Cy0T4PX1lWmaOB4ObLeJNiTWbaXWRvR+74CUsmVizrghoLXhXEGjx7kIzuNHyx7SbYPtRvMBl0b8dESkmHOnnjl4x5d3M5Ic362N19YsWoQDQQ9Mx4Fh2PDyRKmVW15Z10bPjugjQxp4fHjHMHgk2jyz5IXn52eGz45o8xTttnRqlWEUk/j0zYwXwRF2eq4Xj/pMigZYHvyMj5XqKiIerwbqrr3hnC31em+oM4zf9nRlige6VGJwjMNAl8FC/Jy13GXbyHmlto3/L3X/8mtZtu3rQV9/jzHmnOsRGZE79zln4+N7bWRMBSpU+CMQNShAAYQpgBASJVxCslwDI0pIRlSQQAgJCghZQlBEwiCDjR/X+L7OOfuVmfFaa83HGKM/G4U2M++V5bttuEbaZ0qRmRGRa62INcfoo/fWfu37jOFnyK9zHTED5602wfgHwwnGdby1RO+JQQiuYYwG1P/Q649ioRQBfACXMCnxtMy6IJUzdUCuA2+NfvOtkOlk6cQ7Sgks/X7U7QJdMj4afLTEKDwlRwoHrrPW/2qB0ayqEdLAGqd5KhlIt4RwpFtt4nRT8WaQkgJcmyTWKtyumb/1t/9fvF5ecTYi0ij9inM73lica0pWqQ3MxNYH1EoeFRngnD4tnRfMCMSQdHGJf4KVBWMDf/rdn1MrfPz6G9TUN2AYjHH4KAyz43zC2AnrO8Yagh0QBiY3hmj8YZknlmAw3nGwR/LQ45wMx2V9BRzFiOL/DdynvCk18/3LX2K+9cxtxplIaSCl0LXrwLfffsfhcODzpy+8nb9wPE53P87Cb3//lzwen7mttzvMxLOtGy56np6/YV1vd5yaHo33vNN70yJ7/0kvID/Tg0KIlLxr6NxrtGXUTpdC9J5cNj7+8APPTyemEFhbppXK3jvh8EiYE6MPausE/w8+N9wBukNNnXpzKbVH4RsCEWIMtDLYt50YE9u2shyOTNPM5eUrwQasAMidmSnkvEEYPD9/w+eXLwQ/YYPRKNWdJWAwWAwuzPdWY/+JB3MnCjmMWeD+wHdDLZJjW3Fxwj8+YvYr9XZBnHCYFzKFfL2xdpimRw7hG5JPPBwayQXGqHQq0QeuXzLJPfLhw7ekdLj//QvYjpidt/NvqeXCh+fvtEREo4+NRsbYivWd0BWTyEAVsb0hbiBSMc5hndKfDDo+aJp24J2zGBuxcic3FJ1fp3fWWgnBaiNoGEI64vykpHM7kaYHVvnKxoo1A2sr1gq4SqlNd4kuUHvHdgNdWLxmL+fZE3zBhYrBUtpfhxwlhtIHh8XiosV6S3AT58sbJX9GuiFND/heGa1QzGAfhak3nE3KIMy7qi2Nw3VL3gsx7TAJxgYOiyG5I/vaqbuhVtiaYqj02KGIekPESyRFT/Se5FXubl2ntUVJI+PEel758vnKbfs7+CCE2Jhm7cRjBt4L0+R0Nr0IJW9cemEOB921mUwYHVsH03ygG896q0wBYjwwhhC98OH5O/a8cr6+YSVq9ss24iSkEFSbOwJ2OJwBbzqFrJlMp3j8YJ1CTYMn2YWJgJsXTM+08cZ22yimYQwkD94MXZSt45Zf+f7LX/Dd0z/BPB6YloVtVYDsYdaplLe3M+t25nQ8Ap3L5czlsjJ6xXqDzfD+21/w6fMnHh4e9TgmwrZt/OIXv9S6qLV4q8F05+R+7NaxQRHRDCRADxrTGVqvTjFwva4E5zhME2VaOH/9yOFw4jAfkKG4vMH4GUYRosPec5vGWJwLWGsA3eVZ5zRhURUYXHul98bD6YSbIiUXRodpWij7xnCFeUp3v7XXiRHRU9K+rZRNQ+nL4UCtjVoK3hmCdYRwxwrCfXF2DOPgTpUa1ml92jmYPRSLtRU3nNKz9o3gI3F5xJXK5etHzuXGrTf2XvB2IpiJlB5IbsKZwukwaK2w1wtTshyXhB1H5kNkuc+a9x5pLZPrSgyGdfvCx15IcUZGxXsFX4AeWY11GNPpUhVlYjvCoEundZQpy4R1BuvAdI8xE8ENjNVppJ+g3MEItlkag14bGM9wiZFvLMETXILhGdKJE5pDHkN1ETXjvZZG9txoWSn8zjiimzTL2iuOgXUdY7Nqrf867CjB0IqAGJY7/st4Rwc+ffpMcpEQJ+1i8w8mFHJd6Sh84bZvGBPuLEdhlEbeVvoEEuwd/OtxaTCMoQXLMiK7Q8OyZdBqRhjEcOBxfq+7DhrOQYjpXnwOjClQ0hM/fP4918sKZjAdUCvd3DCLZ47qkpZu8cngreWyDa55xRsdcdtqJyYHpnNYZrat85WPvH96oPdEqRlnhSUt9/G7gfcL2JVoG95bLfgPwxhOd8D3UDS2Yqyh9Eo1hmbVtzylhdkcieEAo7M7z55/pG4Z5y3OW4LVpQUpYBzb/plcn1XF6yE9HTBbZts2VUwYxYVtufDy+pH3735FDJbDsnC7XZjTgbLupKAK2hgib69v/PJP/gTnnD4UYrzPXDucDcjQaRXEYsSwzDN5z9Aa0Vly2RleF7jkHOvtKykGVZX2iVy2u8XxRBvaYTVD0PKwYZonzW/mQkr2rrPVo5rB6rz1GKzbxnKYFO+2rjw/PlOL6i5q3ZljouwZY61CNzC4mFRL2xreR7b1Si4bAszT8b5THtg4kaakuyzrEKuxH/1DmjspClw6ILVhDffAdkH2TVO7o9P6IAkc5sjDwzPr2yttv7KWzLQsmsscg8Vaah5YlzhMzzzOD1zMFf/BkreG8Tv4WQ2IRhAp1L5jrbAsM60Il9uV1jLQ77RybT7V9pNmVkcUxSoLstSqu8eW7vtmzeQKDu+OmkoIgvVDO9jS7smRgWmQW6GUyz3X6ZBbZ5mfgE2FaxS8S1gXsRPs61dutxvGRJ0M2yrWQAwJJzq2iERG3+ldqFL0ITn+8VUQ/39/GWM4hEgasISA9Zo3nP0JO94QU9Q1jV7IvsHB6cKlGqf+s5O6NCFi1f3SEm0PWLdgTNDicv+JmNypQ0hzpEtjsxs3GYymu8B5fuKbp1+y7V/Z8meMDJbFq6+mTchTIE2Rv/r9r7nuK5dLhuFxYgjR4+aJEAZVdDLE2EBK0Oqgt11jEiNhmmXbGkLHeUM+nynlLznM79j3yuiD4AfHKdBEF7Dgj/jQ1VXcLdZHRCytbnTTKaPTRyEkg9k727B6FKwbxziIzpC8YVSLDROzX9jYsFi8DVjxWKuQsxATdgSutx/xDzPOPhCSerOlFlpTAVsrhW2/8c27bzmdTrx8/ZEhwuPTNz/Lz5ZlQRBabzw9PeOd4+XrV2KaCSGwrhsi8vNYo4jQh0IMvA9I0tqUovR0PDJEz6iely8rrRla2Xk4nbjeVvZaCd5wmGfdcdSCt/5eEwuEKQK7glOqEsqNMT/HdUJ0OG8pRRtNJRdeX1+ZpsS2K0nI1MGyHGl1p5ZMmibyXiglU/adUSveGnqttFYo60pIkThNjFEwbmaaH7AhamZydGWWThGZJkSs7nisMhGIEYNiw3rWet5WMueyct137IDHwyNrgWg2WlehY94Ksgil1Du4duJ4eE8eK4w3ZLkpwb0JnqSTO3YnRA3lIx0L1OrATNxuF9p9kTZWH861FGrVa906pfaIwKhQXSc5w2gDmlKQRATvEjEEbBCsAemVvO86ztqEbdfr2ZBJkw6POHevMw+I8YjIgpVEL51gEw9HJTXd9pUQNVs5uk5lWWMx1pJroa6AM4yx48xfg1lvZ+HxGDhNlhgqPh7YdvC2cFxODLnhjUW6TiSMpgBfj2YpuzUEm+m1gukYJ5ghyulriVYCPi1Y06g14/vAGiE5Sx53OX00RDxtgzkmvJmJfsIv39F6JtePGGc5Hb9B2oIbAW8Tt63RXv+SfCcV9e5oGTbbGM0S7lnHVg2WhvGW6CwDELFYHK0nztcb1u/U0Xi93ljSV4KL9CpYsTqNhE72uPssuX6SwRBtOElXzFq4u8CHGSxHx36t3HqnjUo3juM88EGLYLUXrZHJvaDvZ6L1GCu0e3femEEZmTyuSlQKE7YP1m2n1KxH0TZ4eHjmeHzm7fyRUlceH99R8k6zwulxwTtPWg76pG+Fl5ev3LaNXz0/crlciDGBhHvdWX04ba8E7xFRRmEInlYy3mlduRiLdQFEfSwjd6wNRK87RucsrVWmlChS6KODBCWqu0FMyjDtvdOK1jqNFdw9LhaDYt5qGZql7I3rbVO/Tis0rN73NjC6qJ7W6Q3pnGPUynVdid7TZdBKvc8bg7RGiguHg8XEpLEfMeDQBp0PkE6MkjHBI3vB3AfAxrDYtJDCkbZdsOdBNZ0dYdiozZ6kjareYd82bpv/2b3ju8WnmVSOSC3I2Cm1MvobXRwynEJ7LXQZeAfBQUwzJet106XRhPtIZccMgzR1BoXJY4Jql2VoPbv3flfeCrnDaIEYZ6LXBzMMhomMXhWfWAe1obttNBttPOxtR8ThSXg7sUzfcFyecQSu1zM5XyisLMlj7a76klbv9VBL74PahL1aMDOYHf8fwh3/41gonWWePHOK2NGJwZBrByMsc2IrjVYqvWkouewVMQ1xgRASBiGEe8ecrDg1FAhs8Jge6F3jH8YL1/1CEEsMBgkOlbdrN27ImTFWLELZK9ZZWu9sZcN6R+s3lvkAJfLwuPCLbze2/sZtE3orKjTaI2uHmg1LMkQT8QLJD+3QDd0dg9PJn2EZEhGjvhaRjdZ3DaILRBMJfsHaRY+lGKTZO3uxMkbGuq4RPDHM6YThiABTrLxyI2+N0jt1+0y5Qwm8VWNetyBOm1VzOuIsQAVT6GbHhQnplrfbF5xJGB6I0jEB2rWDCA/HE9YGbpcrrVWOp3dsW6XUwukUmKeZUiuzsby9vhG8o45Gmiau1zPzfNRxuFGpRTD3mqIAU9BJCxkNiwbIGVWDzfvGfCcLGYTT6YG8qV6ilnKPkFj2fD9iobk5MeCCxwevZZmq0bG97BjpCmeJiVrUZ1NbA4RpWn5mG1prWW83Sm5MKWoMaQzED5bDwq0PmtEg/l52rA266A9t1ngfYAxayarePR4wrcO6grOM2jGTYINnMBhvO75t2uEdle22YeJMip6HaWZDqEWQEXl8CKz1xpfzm4IieuHr5cI0OZqoBxsKvQujOUYNtD3Tuo4JIzNgtBtsOqfjxJRmetPYVoqGvTUYFidoFC4L84Budb67m46xnRgt3mnJrNRKzpW9QHIP7NvOMmk8aKAnqDYKhU5DO+8heKbJYo1qhIWBsTPePLOk7/jVL/4G3iS2rTI9HcjliS+v36uR03saN2LQh6+3TqVl3iM1kmthiCX8w0rL/4DXH8VCaa1hXgJGNCoh5oyIxTm1qtkhNBnUXbNs++1KTJbcVQDvgtptrBOCDwwBaypDCqVuKhaqEW+9Tj+Eyu165pZB+oxzsA1DaYYujpfrV54eX/FbwMdAzhslZ6Zk2coZ4xZsc/SamSfPL57f8Rq0w9lLpuSqUYcRWNwRe7fmiRX8yAwyY2gZoTdFXk3+CRtOzH2n9iutrjp+JU0hs1IZ44LxE2ICvVvVb44KdFobLMukzYCuQFtrDXMMJBdZU2FfV/I2GBLYSgd0kkl6Z0QHSccsjWkEo+OV0jJihDEaJRfOEgmTNrz6uNN3ROujpncwjuPhiXW9kMtKjEceHx+4XW/4GO87OcfT4yMfX74Qk7p7jscTL1++Eu41r5J3WqtKwPHmPs6oI3DW6NydMxCduZdFFoxrtNYZXdAN2T84sop0tq1oTbEVljuxqDc9UoYQdN7ewL5fWdeV4zHciUUG77Xpsu0bMgYxRqZpYpoSX79+Zb+JlgN6Zx2DNE1MKRKTZ72tBB/QPvbAeY9xlg60oaecsO/YekIY2vWXoaDebUViRErFLTPjsmNGV6Ph4cTtcuO2vdJkaDc3HilrxVI4HTyX60b3O6WfGdVTRsM41AufPXUN1Hxg24Tr2qitKJlorPdIjb43vcxUI3hviFOl0cm3Tu9VS1nOgtNmjnWia62reGfwBpw0WhX23LneMlP6wHH5Je+enjkdElt94bb+ntre2GthmEhIMDmHd5oicFaPzzVX9RwdH3mYnnk+foe3iV4/cT5fuG5X9ryp9qFlBkp+t8ZiHUwuaNzKLMjedJGXvwY7SmMgTh7ZNFd3Xj9jvVdRk4FlmmB4yl4433bWayVmIWydTmE5Tveumz41ei/YpKqDvazsrZCicJhORBNUUdoKW86YaqnVsHfIo2NapNfMj59+TX8cGGfY6pXaCtuuWce3t0+YcaMVi0EXo7GcKCGy7yulrrRRsd0gokSgjmgXeIgCLegKNe0TLsykdNCsoxwQTuTyRqtvMBQd1slgHKMJxg2MCfQ2yKVQ28ocE3PQ7OJAQQ52VvRVSIZZItEnzPFAz569dmoz7LWr0S44bm1nYsIPodZOp9FHYUhGbMLYyCV/ItoJ477BeU+4y5+WSee33RQpt/WORzvw7v0vAEeuheXpUd0+PnC7rpyWR6JLPJxOvL18UR2Fc+RN5VZGBCOdvN9IaaLfm1vSO9IK0Qc6wr5emOZIa449n3EO9ny575BVb9tb0d2e6IK27zsx6qQVrSNB8M7RBbyfsMZQioaYvTfEGOhdSxy3243X1xe8tzyennl6fMf1/EVD8zFS9o3teibftN5ZcrkLwpyyBbA6chcSmHuGsFfsvmpEbYA/HpDWkHrBmicwDmrGxgRDfZzOB07Tkf5V+PjlR9aWKSbT8Qg7xtzwcaXXLzR5o2bPGE1ByW6BEijFcL0UzlvX0dnhgM74KZ41Bs6YO7ZuILIjZmOMK8ZoR1p6QESNmMYExIOYih0VqiUPR6+GJp0ulml64rv3/xTvH/8Jnh6eeThNxJio/crX19/yux//Duf9ByJV32+cXu9YavUYEr1qQ3eajtQ6+O7bDxzDgd+I8PL2I/t2IaQMdseHeg+UD1WciI5W+mBJkgih3yE4/+jXH8VCKSIaZMWwD7huGRcaOE8IE5M90keiVYcxr+R60+Cxa+S+81BmYvKE+41m7cABo3tqN+RWeOMj3753hPhA9AFz/ICNN/ZtU7y/ROptZbROMJ7b9U1hpdaCHdh7Hqu0DmNnO19p1RJ8ILjAPB2IYcG6gMmWNna8U1xaE21Y1SZaa3MW1zu9FsQkzYU5RwrufuxpJD9z2yxb/gxO6FiSn3U2eVSwAxcspg5kBBgTZYNowKVEH4VtPeNixxtHCJEYTtASDYUHD644u8NPmgEMe7/ghtDLynAVXMeZgnADG8jVc9kUnTYzMU0Tc1rwTq2IrQvX65UQEo8P32Cw7HnneDoxTUnp5HfYxTzPzMvM2+srMUWWOXF5PTOlievlytPTA6139n1lmmYFIgejYenrG/H0SLCOXDeFQcTENgZiFSBijbCvN6ZJ9RHzNDOGerVT0imhnyY7Ru+Ye7Jh33etAzuHSKc1UaGdceR9xRoU15d33sYLp9OJ0+mJUjLXdVNZnKj+wbq7mhYoe8Ukwdqkx24cMc24MOFDvJsXHcYFxCTEe8T1u2ok4ehQG+L1OhcMRoTn528ZxpK/fCT3jWt9o/SNPb/QWbmtb2z1RimaNQ7uSd+z+wO19sK6ZXVgc1dNoEI8sYZ1veGdAjR8XMHqWCy2MdyEG5YiMPA4r4tQuUvPEIOThFStrwebsH4huMTp+Mjz03f84tv3BGu5nF9xw2EGfHwZjPwJZ9VQ0EfEyMQ8P5PCRCsrl8vKL79N+DjRamOOBx7mRyY/Ia1QzYpxA2xGqPRukJEpXWHBTYbmkelg6h9co/44FkqESqOawSZCc5FhHd52nLU8HX/BlgU7jhznF97cF+iG0Q0le66XnQMeJGDvOUAvEe8mtmGpNLb9yvntzBQj6fjAZCem9MjFfeV1+4LzlikeyMMwWYuTTqkraU60bnURtI0tZ6I/0GWwlp3QZ5YYmNKE84E+DFhD7Squci7RB6pH9SdKHTB2jEFjIGhovuSMRQHC4LAmEVxltRdEiiouZCYGbT6UUTCm4bzH1nuMxDjdJXRDGzCsJe870XemGBhG5VomgFMHJAnRJhkNulCqpbZC71dV5TbtVlqTwa1YP9PbhVyvzH5imQ94u1CHyu73m1J9+mjctpWDU99Miomyb0zhHvEyFucgF3VyT2nidrniDZhRkVEx95xeCIYUHCVXhZ6UFRmFMTLOBHrbNU5jPI6OdwsYw+22clgW1u3KYTnQmo4+Gmvv3NLx84L4E3zDWp0c0vdFI0NjDK6XG84ZTT3cX85o0uByLmqi9E7D55cLVoaWkkrHuciQe+lgiIJmjc5xdwzig0aDxCoWMAbEDEzQILZYz7jdNAzvBTMao4suRL2z5ZU2Gi4FyIXSN768/UDwhuAPRDnx9pp5WTPbXrSZ9+Ax3ty5n013Xj5gcXfPukJtRfT7VOpApGHrhpiNLlrb7qMyaAwX6GYgxnEMjoObcH7BeHOvwRtqVe6C9Z1t/5GPX6zK0Lzj+eGA8zpFs+8bfe/YIdp0tSB2Ykp/ig0nhlSag02+8tuPf5sPH95jvOCM4eG08P7dE+dbZG9vCrE2Ewja38gbZQhNGtYJOLUpmL8OO0oQctsYEghpJk1PulWWnRgOTGnRPJdvxBDxXoPlDA+ja6zFJCwO7zq+VWwV5nkhBk/qkWAfaLxwub1ymtUv7c2ROX5DkcJl/xETJ6aRmKO/I7BENbOTGvSc16J+rSvDGboAfdCGoXZhmjxJFobsWDfRhqFJ1dpC9dhgSeHIlhXfJffgtAzhtq2sOTPFE0taMKjVMJgFMZ45PcNQDJhmByfEVVq9gbmR286BhAwVwNch5AFNwLkbZa7EUBimsW9d3S9+5pvTL0jJc7lc2bfb/Ujj6MPT9hveC2WDMI+7ubBjXNa8YEgMZ1i3F263N6QP6Hr7e584HR+Y5gXnLPt+4+npGWmDbTuzPDzT+mDyXufnL1qW8MEj0kjRkvOKMYPjsnC7vrDdLjwsM+f9iv0p0NyUcL9vmx6h3N2DLZ48DBhDb51SdKbbGEOaZz1mt4azFue0E5rmmdv1wrZvTPN8V4HcF1Sv3fjWdU7ce0+zluKEVhuX1zMxRd3hnQ7Kz2yOsq/0ting5Sd+Y9n19GNngnMY58E6nWbxUSdxLIj3YIIuUMHTq8f4fs+Wqn+ptMp1u/Lx7RM7jWEspQ1u+8qSZrydmNwTeX3l7a0iJnGcDrgWcR6sFYILeIRqFTBirM5lewI6S2N0WkfGXfjXEWtodwdNr4bgHadk+XB84JvDwjx5HZltwm2vnPfGZVRuJdPWwlVeOV8/UcvK7e0rp1PCRsvb7fd8//UvqOUTsy0s1mqkhwAjY8ysXDhXMH7j+x/+LT68e+TwpzNZDKVcdX7byn2i3GhTtEfoTccvh+aqRy84C8E5hil/cIX6o1gohzRqu2BtJLqZKc5gLXWbiF4x8BV42b5Q3JnDydFKY5SuOTNRnQPOaVjY6WRJdMI0P5LHxGEWriVRx1ferl/508Mz0U4s0yPD/tP8xfdXqrnhYyCGmdmrm6ahdS0j4643DdzWG7VoDqxJZdCxROgJZwae+PMiu1clpSQfkWbx4cAxebb9QpEzYjJCog0hbxu3tdIPyrDsY1OJVJhJYSK4I9dtZUjHSMD5mePyiJHE7faV8/pKTx3vtE5Zq8KBq1Fy+BQg2E7JYFxkCQuz/4YlTcz+A/t05fX1I1Z2Rm2crxdi0t2XsREjHWcqIRVkXMhype8V2SopJkYX8raSpoXHh3fUqlAJEXj37oneCz98/KijlN98wDC4vH0BhtJ2vL53IlrP3W4bh8OBsu/UmpmXCZFG3jMxKBkGLCElBJ2o6dNECI5tqzw/PyGjE+/enl4r3Rg1B/ZKsI7z+Y00TXgXMMby9O49l8uFl7dXBUJMiZg0amSNQ2n4HuvQ7B8GMzLLpN7sXgvresGlieiA5tmHIW8bjUHsFukTG4YQ1dPtjGAciIs6d2+GnjbihHEH+vUr1jv8fASp/DRDZnrBpYgxnlve+FouSFyI8YHT4c9xVKx0lthxPtFFxzeNsTh0velWp36ij5jhcC7ebQGGaD3JKt0ehEakiee6Dfb6hgTBVcvBnfhmPvFn757508dnjg8T3g/WWvl02ZTOc+vsZed6Xcn7FcGS5h3v/z5+ymxvltp3Ltv3fHr7NdA5Rr2np+igD6x95ri8o3XH9Vax9oZzZ/7i1/8mUY54UQBzawWfFrw8UFul1qGLo504BEcbgHV0EzG+MkQ9TH/o9UexUBoctQo+ZHCV1jfoFmMcjw/vef/8pyzHzNfrZ3BvHB4GvQ7qzt3jG2i944en7RafkhaiOyTvOcQH1trwrlME9v5C3m8cT99ipfO8fGB/+qf4/de/hR0WZzrWVVyY6aKjfcYq/syaCUaltwqo37nUG6fpActM8Aut3OsgNK1VlcIQHcM06E7iuESs82wlq7tnDEQctTRKaMQ4seaMsYXHeNI4iY0EX1nzhRiOOKLCU58PnJZ3rNuPDFNYR6E1oZTGVjaMaTpO2TLbyIwOozs8Cw+zp266Y37/8MDT9MiXl9/wQznfg9KN0aBmy/Gkiz9hMNyNbXxlMs9M8wGLY1B59+4DxngdKwxB0WoPDxouf/3Iup/5s1/9M5Ra2K5nJoT5tBAcKv7qit8qeeNwPBCDp5WCFc29fv38g4Ij3KSGzqY779Iqs/fM88QYnVJ2QFUG/Dwvrp6bkhXIgQ/EEJQPOQWM9eTWmQ8nrPecX75wuV54cI4UNQB/3TZiCISg2c4YJ7Zt5+Vy5uHxhIkBJwdury9Ib8pxPCyMaaa1HZFGE2FWDJRGhXrVH7YibkGWE2IMkjuYG84HGPpgJlclnXuQmHAivHv3njwK7fUj5zroFeb4gaeTY7+8UTIs04FgE0PgcjtzChO9aua4m6rKCq8wW+8F5weWgaXjRPAhMrsjfTzj/cTl+ltkbCwOnlzkfYg8WkO0A3pmz43rmjmvG1/ezlyuhdtlpaw7dgxCtMwm4Bjs5QvOeGqrlJKxxtNFWOsGdlBInOYPuGjJ7Yo3QpoadhcOaWG9nfnL3/57JBshafkqpG948EfW9Y3z9Yt+jElA1RIBjmobuXUqOkH3h17/UbzevwL+F8Av0IrGvywi/xNjzP8A+G8An+7/6z8vIv/K/WP++8B/HR0E/e+IyP/xD3+VgJVH8rbjotYQdFFKPD58xy8//A22cuXl8pXvP/1rGKnYPrF7wx46ZfOMptMVLkwME7BekUolF3zqTDFRR8SyYEXY1gttzppVw2JHZPKPYCqm6+xtk6pEZBFcN7ihpPToIxtVmZDRa+aPFcNMMDNT9JTbwNBVsW6E0d4YAtG9J4SID45lXrhcP3NuGoo1ZuhucCjxqHfD9Xomhok5eEQGUwrsZTDGyjw9YgkMIxxP75kOkfPtR3r+SmgdOqxbpRvAJ4xzYAe9rJRc+PzpI9ZMPJ8+cJy+xWHxMfDu6Yn1+pF8Dbyt7V4/09nzeZm1hpUsUgd2aDMrmEh0gboXasssy/JzbdI7z/n6Ri2VDx9+iUhju+58++6Z2+WVnLPW4tSiRWsVZy3SqkZa1htTcpRd2G8b1up3vA8N279dL1ijzUCL1RnoMSh5p/8E3TDanFAXtUarehNsFFK6w5ydoUuj7hlr4NsP37JtuiPyd8GaE0uuu47BiWE0RXYZ4PXLi/ppUsBY1e1aAEmEGInxRIweFyLBeWJM2BQhRBCP1IzpHeOiPphNg3VFuMM7fGAYsDtIydAKvVb2600bOtPC3/nt77gxaHVndCH6hPfpDmpOdBo5b3x++4iYgfUO57XWOKesjh69asF26qg0A8cwE0LCVsfBDZbHwNg/MtuVWRyjdM7nlVwywVluVfhy2/m0XXnJV3KujDI4RsMhWJbFkxaDxIrUlWoGex2UUpBqQTzDZbaszaaQKg/eY9rAJQhGY0fNq7v7y5fvERd49+6X+PmIv9cmrVQcF6gbZgwdQpFGNYpUzOKxJAIn4Hf/vy+UQAP+eyLy/zTGnID/hzHm/3T/vf+xiPwP/30L6z8L/JeA/zTwJ8D/2RjznxT5A8A3MVge6GVwKzecb9RdOC3fcJofeHr4hlAnDof3WBNwtuHEYS3E6DFhwYWFvVfaEMAiJlLobC1j8hnnFGjR6lCzHYavb19I7gFpwsDi5cTgzDCZSsXcVbkDS2bVIrzzRDfxOCXWbQPXGXRq3yls1KJZMi1fVqzTLnqjUtsNF2a8OeFsYgoLT6cnfv/yme8/f09Hj4khThgX8O4B5MLnry9M8QFvdlJyPBxPrNuZdfueKX5LDE86o+5n3p3+hGuN7OW3BNkx3iHuQAzfECdHGTe2vCqTMN/Yf/OXXJ9eqH3l6fDAIVlGLRgBYyK0oQ5tBy0HRvVYcQQbcMHQS2ZvXuX1bRDEMi8TMeqY4wiqbW21Mi0HSq3UeuG7P/0zLtczYwwOPnC9Xmi90Xpjio7j4cDter7XCDvBJ/K24X1gXhL7vuLwpClgLh5nNfScUmAKE36NINBLwTkHxmCDR5wqK9Kk44pDBm0MTG/YplnJXiulFTYRDscjyS+UbUO6kEtm1EaTijgh+AgI1nt6a+R9Y9+uSgw39i66MxhrcSFgrCWmRDqcdCrlcMBNSr43aWLcRxRFwIYJDhO0G2NfsbngDo+04wPuulG//J7z62e+nD9xHoatN87XK2+7sJWdeTphW6e2HZeEkNTr3emsOVO7WglTShinf3c3DM4mMAvOeGBg2VnLV9LkcWaGMWFMw4ZIMhUzIpet8vVyY0Mtj7U59gYbmWIrjErwlsPkOaXB8WRwk1AYlN4ppZN3oTUHY9Lac/d0q5NFL29fWfwnnpYnZFikrVhp1DxoZdHZejLX8xsHf2LvnUFl3XZ6E2YcplXGaGQau+0073E8qkCNf8yut4h8D3x//++LMebfBf70D3zIfwH4X4tIBv7CGPN3gf8c8H/9R33AECV5B3eglk0jC7kzYic6pwit4ehN+YcpgscRvCe6gGFGTGQ6nOhDdZ7GWIyfGc5RKrDe2Htl75kQA2Ga2EuhsWGNI4Qj7w6BL2dd9PrYsXZgREEFwkrtjiELc5g5+AOn8MDX9Qtr/0LOFeM8ImoQNM6pLKlXBa52T6dQ2wtgSf5Aik+cliMuPHPddr6+/Kh4raHu5SkdiNvMdf/E17cXGJ25OWLswEatr5R9J8Q3DocnvJ2J/plfvHvmaiyfvv972ALxtLAsT8zHmWv7zOv6lU6hFwU/fGqrOqS/+441NwwrEio+Go1IlaLWybZAPdELtHDBuUETXUiDBIJLJD+Btey1kCb1dr+dz/foUGaaFr798IHedffw/ptv6CXzdn7BGsvj0zPR6yjmNEdq3Xk4PWiNcQympAvTT7EeETgcDqSYuF7f6AOsS4S0aF02JUpVLqcNOqDQ+gAziDERfdQYT0AXwFaRNvAh4Ly/Y/IscUqslyv5egYRBS4PtX4eDgfKnUbei/4wFqZlphRtsGiXXzmbNe9E7/DzAR8Sxnm9blvHuYTBQa+YbhhpwUaQNCF5Z6wX3HyA0wlvvsPsKz98f+ZWGn/54/e8bSufL29kyez1O2ZjueYre151VC8qb3SUhh1eG0kCvRp2IIkCd5sZ9A7OCn7KDLuyNpjDO/ZcMGNn8paKpQ64DOGaG7dWEafwk2EFh2W2TulUCCEIcVLbA0aotbFujdttIxdHKaI7Xevp4rXu3C3WVD5//TWjfeUYDjjpuKEPt16ywrONsG+f+PRiAM+QQi5vhLGr6911kM5uOplGbZ6n6USaZmL6jzFwboz5c+A/C/zfgP888N82xvxXgX8N3XW+oIvov/oPfdhv+cML6/2CHwTvsePIlgvWaWct985lfeVWdi7rJx056gaCiomSO+DtAeOEzIbxOo5kBbyPGBy9qirgentlrTs+Hji5hUOacTiST1gM1i4s03vy2wr9ijVddyqusdfGqJk57Hj3JyxpIRxnxquwfnkhV12IvX1W30eyRNuxdmgHukMVwbZKNTdk/8jj6RtqNngOPB0e2bYfMG1DmBGZtHN8+IZB5np7wxnL6mGeO8IOY7DvXyiXH2jjVzw9/DlWBGcm3j3/pzBy4td//99h3wsPKTJNM609EHzk1ne2XAgSsTZxuVxwoUNcwa5Kfz5YFoG4R0rT4PEYnpI74jeqK2j8N2DsieADuTRMheAD18uFbd0RNCc6Lw8cDgtihLe3Nw6HE6NDrUoienp4ZponnDUgTY+m1hJSJG8row989EomN+p/MThiVOd2TDPWWMDiQ8L7gWEQXULuYZzS1ZldyyDXgrHKPBQcZW9IryrVMvGeNggMdJQyTROUwvX2WclAYcGmiX03hPlADNCxEO6kdmtwUUsJWKUCxZRwRj1RRgZGOn3PMC/4aaG3HTc9IcbQ6wreIi6ABbv8gmG+wP6qHeDgefrV3+TD+Suvv/37iMD3H3/DuV0xrvGbH69MNtFa5+12obSqn8d2wtFCj3rEbtqk8sHcp1fU1d6bUfKSZIJTK+WNndonjAzEGwwzuXfWYbgNQ+lW76WgCltjBs6B8wNnYETD6qCjfNZ17VzehPVW2XOmtaZeoxCwPiGmUWsnekfZdy7suEMmuolWLa473FC7o3WO1jNv14/0roMd0UOYoNsd0qA6qHclRIgW8YWQPPPhPyZ6kDHmCPxvgf+uiJyNMf9T4F9A65b/AvA/Av5r/198vn8O+OcAllNAusaDvNPJjdwyrTe2feX1+sJt27mcv95zX469DbIpZLtzDBD8wNnOtjesCUQbaWPFhwMiFjGD2jJbvhK9IZYzwQnRR33TrKcPq/Wc+IhkIcgOtjElQxuNJoXcK7kfeT//OUt4h/EnrtvK569/gYwGNqtEqVkaDR8HBNG56VzptuPYuK2Nj19/x3HWsKvIzpwWOp0hK1sGbye8NxzmiXXfOV+/YK1n24UYofc7+aYUfvvx32Orwi/f/7P0aLEk5sdf8f4/kfj4+e9zKysHf8T3gGmOtgujBUoXYvSMZul7YI7v6CZQ6hnnBuFk8RPIZonhRAzvmYxl9Be6uTHGFdsmnDvAvmOrcEgzJW9gDGleQJQvOS8LpTaWg6WPQmmeefZ8+vQDxhiWwwHvPN5bSlXyeskF3wZ7bfeFuGCdp9ZC7DpS10e/79aUOjSG7ghHb4iosVPn+XXMz92p5nkrlFapvdKcYTRALHpHd3pV0EqaPMigloYJnuV4ouXKFCd8mhnG0WrVBiIoHLg2rFFdrHHm7kuH3iGkiTQt2HtpA6mqpp3VLz4MmPQOsR7ZXiA9gFc4NNM7evD4H36PlBsmWL57/57PX8783fpbLuVCaTtpdtzqldd8RXqgdcM0f6vWUiek5AnGY4ZnZH9nYQq9Z4LTWnwenb0Kux3qVfcDGyrGPCEu0az6hYrZWIuCLGrpDKOxNOsczkeMsUyTmlY7whBDaZXRO3nzrLfO7aqwXc2YKrfAOo9xkdZ39rxjRTmhSGFJjj4CZQyst/huEatgmNKygjyMBaswnW463Riqtext0IwQfSbMN2xq4P6wXew/0kJpjAn3RfJ/KSL/OwAR+fEf+v3/GfB/uP/0d8Cv/qEP/zP+A6qkIvIvA/8ywPOHJHl7I8YDjaaCpKFQi3U98/X1R/baKPlKdBYfZ4bokwEprD1j+qpE8eGQMWPi4Y78HzgrDBmkZcKNK1VWMDNi1L1RWsW6gAzlTZ4O7wluIowruX1lmK4aCNNpo7HVK4ONEC1H94733/wJb5fvqVW3/0aglkKvGxGLm5QbaUNnmI7zCoX98eW3fD1fsKYRfMfjaSh9ZUqoy8cMQop8s3zL7Zp5O1+0ETEsziizsnVHbZ2Pn/8CjOG7d3+TQ/hA7w7rE4+P3+hR1QasiQR7oNU3ejWam5TKHA4wAr1YiI5hlRTvrDB8x82BaX5gdjOTS1Q6WzvjzI0+HvT/HZ7gJ/aykXPm4eFJn/TWMc8z5/Mbx+OJnDO39czxOPHx4+/Z9isfvvmWeZqxTkcE58OiyC7nqW3gfEKk68PIOUKaad1grc6sWxt/JqJzb9rkXXTKxAVtuNwhwDYEcunYKZLGoO4re66kKd6BHI5eK1bqnbzttSkzBt0MfEjM04NOWXmnLut7MLsPrQPW2sjbjRAMwQcFWUikOsHbQXEQgkNM1LHEqGbQn0YaBcGGA0ilrC+EwzMmKPnIusR4fOL6+1fy2yur9xAdp8MDp/QILug8PAMXAtgjczqQQiIEi/MwTZ7gLdIjrUQsOls92oWRX7jtLzQaow22yyAmi8QVJ4OUHIekqYuyN2rRZs11LdQxMBYGg+PRa/PKCykkxcqhWc3WK/0uQvuJFmWtwblBmhzWq+rDNIN3ibxl9ltmmQKlV0rXJmrpTa9za2kiupDyEwi4U3Nj7T9tmoUsg2YNLjl8aqS54LxS8v+xFkqjxaD/OfDvisi/9A/9+i/v9UuA/yLwb9//+38P/K+MMf8S2sz5p4H/+3/IV+G2Xqkl48yMuBlICJat3PC3A7nsOCxPh+/o1rKXlda1njYGlHHDtI4bT5qx9AYTtS4oGIaPRPPIu0fhVl7p43Z3swQwusB2Eaz1zOEJaZZcOuKOGDY8AuJp0qlt4/X6idPhA53OvHjm00S/nbHDaj6tCsMYgo10YxnOwCh4a5n8kRIGW+ts2yvRWtycdA7XWIwRnBec7fRReTg9MsXEu5Mh2B/58vUTzdxdyQNq7fiYMEZ4OX/P6I1vTivJPuOkkrzj+PAN3zx84GyuGHdiNK8jbGNwvq744DicJnzTsb2ORZyh+67fZyM0s2OCEtSlZZ2c8GC8uZOoVYDWasWIIddGriuHw8J60+7tPCd+/7vf8O2H9/z4ww84Osf5QAoTQwa3202PrVvm4eFBL3rpP783Ps4E71Vxa+6M2/u0DcZQm3IlgwvYWWMmA7AhqR+no5pUW/D+3myxni9fP/LdL79FDDRpmK74sMl7ehtKw7kL0IyxdIQQ7P3hCN4G2mj01ljXM3lbkaroM2OVeDSmGdMKo+yMpZKMZTOOw9OC9QFCUBjxKEgtGBwmTPhlQL4iTBivx3WOJ8L7P+XlL6/srfC6rvz2yw9sPWOM1+/NXeDljSVEjT/pg8QjKOR6DEOMUbFxtZIkYfyRVd4wvTNaQSgMj8q+bGGYTBtX+pjZi2HfK9ueWbeNLqr2tdYgJmCDJSTFBNKgD5Sw1RoylJXg7VCoczLsbYUQCSHQKpRNMYj7phbN0S0DlZgFH+lN6LVh7aLotQFmKHnKijbmdgurhSoVFxPp6Iipczx6TFQxXd3DP95CidYi/yvAv2WM+Tfuv/bPA/9lY8x/Bj0M/yXw3wQQkX/HGPO/Af4W2jH/b/3BjrfeCtA6OXct8nqLcUojzrlieAOj3eoPz3/O3jZu8Y1xXhFzU2QWGjovWyGECRpEm4hxUiiBHzgswZx4jJE6btSaWdIguAmLoRkDQxFaMQZeLo1SKyGBidrdtjYy2s7r7TccLgvRf0MbV6wXfBjYnqnNM7pn9AObCNF47YJyUo5kVrujmTrVVMxP3MQu6iCxgvFXrL3DSacJ7yw+OD68PzLGSs71Xvj2DAlgAvMUid5S84XfbX+bZXrmZBa8hXfP33Ccv6FXh2Apo+hC6yxm6Dz1y8snHh8jfobWO3kUQrIYu+AnT+uvrN3iiPS+IQZmG0gBJBSGVB1X65XgAnnPpBio+0atncfnb/j06RNTmvj06ZPucp0hpsiWM2GadaFt2v3vfTBNGpNyLtBHJ4RIbZXc1JduvSHcYb8/X08/7R5rQ1rHOY+PSU8QFsQY1TUYw37dKa1zXjcOt42Hw0zeN0rZOUyJOhrBTnQR4p3vaO87V7VBGmotiDU451i3jX3LyD3uZMQwTxrWHzVTjDainE/U2ohSKOVKysobtfYBiR7JF8g7TB67HCAs2rCoG64rSzSMgQkTnz+/ct5uen0Ox7VesM7i7KB2TXTI6DqNIiA3TS0ss0WGIfhMStN9qgpGa4xSsVSSH3eGpwbjrXREbtSRkAylOHo3OhLa2h01p+9BrplJdBwUUSto3avKA4fOmUebdDdrwESvJ7JhlIfZB7WgWuIMuYC1jikFWtP6cyuDXozee3SCHSQ/oOmu/tpg3YW9dVw0HKZO8MISHdEapGoNdN3+MUcYReT/8tNa9u97/St/4GP+ReBf/A/73D+9rDWkGNnXSm1Vu82mcV0vvF0+s21nXLTUvuFNJFjH5D2rd3c1bb/n5Dy0fm8GWMquMILeG33cFMRrF2obRHPAjUbfG27SI64ZHesce3mjtY0tv7Hur6RuiOMnr0nHJ6ih8fnyA4eUybmB0SiQId+P9BHBUkdFckPIeiyIHqmVMFnoHW9VboXotzjXjWZ2bF8R6xjDcF0zT4c/pUsnxsEvf/kNZkR+/PzK2+2F3galdB4OicVFvBder2d++OHvUaYnHk/v8T4wRKkzuV6JDyC562x8h8mDkGkyYRr0rkKnYTwuRryZWJZEaWf2faduhhQXzGKwfCW54z1/58BocX1II29qy7PWc71eFXel5yKmKSHSCDFirVdtAKLH5pyZ55kYJy7XM/MyY8f9iNYFY/S9b2pqU2BxjPcJnUBtXTvN3mGdUs3F6WJUW2fgdMGtneu6grHkXLgCMSg2bhidkBpo46WJ/p7ctRFwL5lheXt7ZYxOioEV9fE8HR8Zo+OiZZ4molP+pTUG6wwYJYOb1kkdVZzcXpF6xRwnhebuN0KIDL8oSadlGI1xy3z94Ucu245NB87bG1Wu+EnzrdumJKIxBCTj/YohqDK3FNW5pp+aN47gA4cwcfIJbzvChrUd74RgDH00egFrAzJ2hrvhWRAcznglx+u7r19XDPueSdtP75UwaqPWwV70lJf3gkQwwxGDdrgNRmN+wLgLy/TfmnCQocoYGcK+FWpWAHBisITGkizRDrrfaTTq5tiyioBDtJiw6ziyUW1FF2hZ2Le/Fpg1i5+O0N9wVrWo0it5zZSsZrax38j5jRgmehMqBecNNiSM0ycU1RCGpeyNvlX2+pWDdKWDW0PvOyIGuqM3h5hBHo3BzjCD1ispDEYz7Hkj9yvrftYpoaFHdLEdMQZ8J8bbHcLqMFZrQlAJUTFVznqGDQzbf35za1X0GrljrB5/wFG6pZtAs4brdmWrhik5rIm8vN14fbnyMD3wzeORx8MjS/qGw+mZ334fGK+fKFtWDephIhjPw+y5nS9cbmeOp2du6wohcNleKPWF0wwyeXTcvMNmOU4PzMuzAnKbwFbo1eLjAcMBayYOMVG2LzRp1EtBpFNnyxxBiEySCD4iZTBGRYZnK5XT6ZGSNQdaWuF0eriDF1RvGlOiViWR19oopeJDJPdBHRAxdyq8YVgDztH6uM9jR7gvtCklnUGngRgFkoSIDRHngg4Q5F13qF2p3tu2ar1QOp8+f+LDhw9gHduemSdtIAWntTQf/L3736lN8N7fFyP44Xe/4fnxgQ/vf8Hl9oK1hsP0SIpRd7zOEUMA0WaF+AAh4eOMMZFeCozCuFZsO+FOz9imsSUTIpiE8QYZV7CONhrff/mBC1e6ZAawjwttnCm9MmrEWcW6uftDzIoaa9porOvAB0OM93HZpoyCFCwmwEAwvmKHpRTDeiscbFBXk1isNzhrsWi2OIYNawrDGbxTc0GrjVY1AihdPTo1D9pIlB167hwmHQZwYvE2QtfxWzA/z+KH4MAoYWn0QcmNMSy1BMZQdm1cLNNUoa9UaZShHp8xBi4NfLT4qDjGUkGaNoRac9T21wDcizG4dCSajncOaqDug4eHdzijHet13/hy/YR1jskccd7j7ZFeLZiOcxYQdZfIxr4XLBN7WzkdZoJ1eAJ5vzKGu8vjhHWruNjBjnv+8kaKkVozXRrOW7oMSht6nOsWtXIK0Svz0Vp0AuIwKFthMHCTQgwKd+qKcwTntSMbHc6oCW8MYYiQnM7XBhs4+idGNUgLuOURxsaPHz9yTmeM+47jw7ecnk482Se8t9RRKFMnGKNTP0R8FJbjAyVreH7dz3TXWfcLIXqVd0WwdhCshVPiOL3Dmwd6qYye6TYixkK2WJfoop6ZQzhQ1jOX/Ma47x7EVLwp2LERCIwhtFppRaVvxgiGyr5n5jThbWA0DTu7+4Ms553R9YiYZu0m145+/m7ojXt+0hCSRnDEOFz4SeoVNKBnAu5+2rBDyzjGKqGn145zQQk8Q4+JrXVSmrmuK3up1A6tdGq+Ye0BGwN721SZ7BxzCMQYdWqntZ8pRMsys17PTClyPD5hbcDaBt7h5gkfF7z3WAPTMhGnA/M0E9ORIR7VtvS7ROuKAO75mRE80q4YfwJ7RChYKZyOzxj7V3z8+FfUcuU4Hal1Z/Q3hmT2bhAsIWiSxIjGu2yLMIRWizakjLk3XWa6OPIdMNwQhc8ISLf0Zsg3g51mxEx0Z+jSGWNHaHjrMU6wUyfNDh8s0SvNX9Qpgg+WEKFnjVxdrzeCmRAn2jC7P0jGaForH40QVc9hlHHCGEKtQq9KdLcWBWO7SjGDJo29DM57YxsDmxzL0TOlTnCdPlDHT2sY7mWy/Q8vhX8UC6VgwAV8mPFu4GzAA8d5JjpHH5aeLWXruNgJseJ7xNkjtQ/KetFMlEAb2t0VV6jthlRH6BU4MYbW3mopChFtWqCXO4Pgpws1BrCuk6vi0DSfLFhU0jXGoOaKXDKCIYYJZyH6yPQYyLfBcB7ike6ELoFcL3gLy3xgnhak7JzPF0rt98VG8M6z+BljE9bOePdEI3KYKsMbbttvud0q281gPswcl8CHx2fO797z49cvHKaFmBKIQ6olThM2DvwE3WT2XWAEnk//CT6VwehXUrA4K9gQictMxCI+0utEEKECW25c6415HozqESLBnoixUUcmZ0u0DoLFWT3ij9F1B+bVmb1uKqN6evoG4wL2jh6rbXD0kdfXV0SUHO68Yzmc6MPojuTO8WxNj+yqYbhrHEJAUIG9tZqWsN7QRBS7lu5QZ+uVQdg0/ZPvzaXSGlhPbpVtrwiO3Jr6wqsyBSQMWu1ghWIdyalG11rLvu+UrPT4x8cnWk6Imn7x0cLQ+nRvDWElulljTD6QYkSMqMjLaKYmOIfYiO07Uq5IPWKcY2wvYM8Yv+DihJgrJW94Z2itkGvFS+Jp+g5rGrhPWCPIsFgrWN9wJKydMOLpDQZF6+EG3d37QO+GvWqDw5iGu++kGRYriVEsw0XERdpd/tZlxwyl4hsjhGRJCaxRr7yzAecnPTk6hbzY25XeVtbSWa+Fh9MM4rEEohf6qHQRnPXqiY8HnSoamdFUxlfroDUhBsW1DdfZWiXvg8t18HYziPMsR0dYDHFSb9Voao1sI2BFGL1j/3CJ8o9joQRwcO/capG9NaORE/9AK5Vt3+lZCE4Nei4q04/myLvQa8Y7Qy0FYeC8wft7vUyEQsZKQazDBq9w2iF0EVq7Q3RbR6zFuIFLVYv+Tnc7IoYkERkOkYJ3E4hlXy1u9kxTIkWH9cLjHJin7zicfkl0gev6xg9f/orr5ZU5KRrOWO5haIsbDtrAeItNQS2OwxPjjHDQzu27DblecWYi+iO2z4DWl+Y54m2DAP7Oq/QTTMy6MHhLHzvRLZRa8GPCmwdu6xnXLYcUwA9KvYJT6OwxQTg8sTeL3Dbe3laut5XDcSLFI8NYUjgw20VLCQ0IRmM4w4JruGppLXPbVlprzMtPk1MebOS6vfLh/Qe+fn6jjY2Hx0cGHuMd/p59NGPgrQKSa614Z9nLzqg6DOCsYUqJNa+kn5igco9COfUhWa9TMSVr7bS1Rq4dwWoi4t6IKKVyPJy4nr/ipBFcoNdOXjccA7FC7ttdahfZ9w3phbbptScDluOjgiWMIxqDi16jv6NCrqx5Ix0WPTkBMU3EcMDGI3m7UMtOfHwHRMgbsr5iwjP28A6uX9g//5rmI1YcX6833r6+cXv9wjUPOgfmsJDSNyz9whgXhkyqG7EVa3Te/+AszlXW3anj2kW8m8EG6ujsrTNE6eIpeoy5u2aMBvtHC/SiVPZBU52IbYRYCAymw4SJFYuyWcXAlBIpJoLztGkoTGYrhEnrxLUqcGXxlmQM2QyGNKa44OyiZtDJUtqVUi7Urv2M0SwSOgNPEyHXyrp3zpdOH475GJgmwzwJ0Q8MkLPQuqOJJxqLHR07/hrwKA3qZPHO3ik6Wr/bW6XdXsn7Ru+d4+GR5ahZOx8tvVd8AF9VITGCRj+Ms1jfgM4QofWVIZXkZsIUcQQQi8+NEDutONqui9XeBl0KtQyGEaxR+x4CYhsxwSHM/I0/+af4m//kP8PT8yOtF263M8a3n+s9h/mRb9//KdYnPn76Qus71/OV21UJ2DI60jpBAl6g9KEQBdGQrrdQ25k4Bfa8IxRmnwhuIcYjkDQsXA3h7kLe76ItbzxzOjDFB1pbEbNT6xXnEsNqoNpgCXLAZkfdwc7Q3AVxnSiBxVnmNBEWx2YgF8v1fOV63miHgY8a4WDIHcsVGT3eM3wOrMMGobUCYnD3DGMrjTFVfvz0a+Zp5tPnj/TeORxOtG7BqM9FIzn93nFVXJv3ahHsvVNaQfpAjEfMYBhPxxF9ohQlELWqOgNrla7d7qbM1vq9RrnRmtCHZdsqr2+vnA4nbrcbS3Qs00JphVF3nBGCs5SSqXnFOeF2OUMrOBNpvbAsy/0YqLtN7Yybe3fc3m2SepztopNlxs90AkYG6fTMtn7F7Rk7JVhmzMiY0pCUkIe/iW+/5q/+1r/Br7//wrXd+PHtN5zXC2sT6hi0Wum20cVi0kpynV5mZCScHRjx0DzGzMokyDesiXd/U6f3weidOjrNWqBjrcH4SDDK5XRGJ51K37WO6QZpNvg5qvJ3Crioka02BoLBxcjx+MzD8T1TPPJ4/Ez0f0kv37OvHeP0usytIabepWuFYBJp9tA9wTuExL5flYvZjAoDcfccqyNnYc+VYSBEy5I6xwTJabayNsGIuytyDMY4gtOI1B96/VEslD+Nl9XaEGmMDvSACYl0J0rPoyJ4vAtKmKkdHwzcL8DeofWdKppF/GkXIWgdygHeRaKHKc5Ef2LPK5frGz0MJNyfvLUxxkIZiToytWZaAzM8++j0Nvju3Qd+9eGf5G/84m/yJ7/6ljVvXPdXLtevbNuF6tQp2ot+7L4X9q2x7zv7viLV0Ftm6pYlKRtxIOyuEt1EbgVapI8rZRT2kWmyUeuK9CN5z2y3CzFMWGM5xiem8JXX7cKbG5ymEw+HP8GamTFWbvn3DLOz96+IVeBviAY/DgTxcKe2bLeG8xWxVmede8GHwOkhwN0ds12u9FYJQXDGac1QrMYi7hEQzRUmhhWl0wzRTquBVjOfP/7AcjjQQ+J8PnM8LfcZbPQoWTv7rl7yWguHeab1pki0Wu6lj06KidYG1+uKc56YtH4yRgUZOKeRIRFDKYWclcs5hh6l93sDbNs3zucLznnW243WGs0IrVRulxd1qtfMFBT/5jjx+mX8rMKNU2QM+3NcSL+GYG2Ae5wItNUXU8IFbe6EOFNlULYLoW7Mj+9JD4/Y64YpO3JM4I/I6Ji20cNEePzAu+/+lH/97/wF/+5f/mtU/8atvHIpala8eY3j3MqFsBjCsREmpTBJb3i/aDjfCrHNtNbuUF6hSafURi6Z0io+Jm2mgDZwvMWq6pPS2717rUdfbyMpJKZlxoWE2DvFqWaN/LjIcnzH+w9/jmkRywPezuy3xqt91e+VseReMVaQbpFu2NYN7w4g6oMfcvfeNFX2Oq/AE8ygVci7UIrc4SOGEAbeGYwd1NER69Tz5AxNErZbrFU1xB96/VEslGMM8r5jEAXWNsE77aaNOnDW4qNRn4exlLpTeiZ0zzQFjocD3g3WvbH1lY7FNkU1+TvF2TJwdr/Dbo9M4ZkpHdjzxl7PBJ9wLDjfcQiuemx2tNYpdSeY47371/j85ZXXtxcutwvny0LuO+frjVvWAf+3t6/kuvLp9JXeHD98fOG33/8V+14wLqrwy1qC1Vnay3ZjpVGD+nSMcZhumGiMvOrRv1XSCGyvVy6HL3gRxBw4pcDx8I6nhzeu+0orleIHYiNpeqAPy61aei8YoFXFVvkkDPHQtIPsJahH2nu2ITrWN+AwBjEEDgerXUc/0fIKplB6J8WDytNqo4ZGtw1xERFHR0sbMrSrOXqjrleGGNxyZL2s9zogtC74YO6QhExbN1wIPDw8sNdKLoXHx0fGnQCFQEqJdV3vDRKvDZ+qsDURLZuMoaWV1ho5Z93xGEMuO8Kg9cZte6P2jSlNnG8X5iDU3NiAlm80v9BrJZcbSKPnSB6G4/PzXSMA03RUapCoe8gIjN7wTjOzznlsDAQ/EaeI9SDDMMWJ1jSJXS6vpOUEwem4Ze3YyWFcQqRi5Iq4wOH0Dc/Pj/TfWL68fWJvO5ctU7uKuPKu89J/dvonmFzGhsxOZhinkBebsL4RQiDGRWu8CL0Luexse6b2himDEireW1xwHJaIiV4VF7WybQXMADxhTtjpBG6iD4cRVYuYkRm20jpcbm88HC7qsicjUpiS5fSYKPVuxHTgrUdGIgZorXG9vuD8pBqJu1XTGbVY6sCBMLolt0LeKqNYfLA414ghYZyqnZ2DKWk+0/VIG4bcBqWrWvcPvf5oFsrRK3PwGBnQVWfQfNPQrm1IKfQ7g/C23aj3eV4ZMzE55jTTe6G2nVx2nRNFaytGdHewyo4cVnjopBTBCD44cjvTjMVypQ0LY8KZoE0D43HW4GwkRq15Xm47//q/9W/z6999z+Fhxk8BZz1pTmAG5+sP3NaPmg28wcvrjb1vLGlmSRNTmDCjQm93o6B2gK011LbpxETZWZbIHDyzMQQMu3PU2nn7+iO9VUz4jofwzDIvvH/8wOX8ytv+QnWaR1Rh0n0H1YbqJ0ak107J+z1zaBXxJRp1McGrPbILWx+MMohi6RiME5gEFyNm6A4ltw1rMsN6vDjWfsOJI6FmPjMUNEEf1N7orpLSgX1bNZ9oHNu6sTw8UWultk7eNq7bxocPH2ilsF11txdjvBPEV5ZluatvPb0L1ioZvdbKGENFXSLs+6Ze7trovf8sN9MdZmZdb/SudKF13TikSXO9See7e9NdTJwcUiA6/boxDe3ax0StO/lywxhBmo7oWWuQ7hjWIV6lW9CYPAQbEANWGqKtfPro+JoZt4o96sPHoBZEZq8TO71A23n54Xs+f3mhNVivjcvW2PZBR3ddMgx/8qu/yYP5QDBnhv2R6r5ifVMlQjN4G3GhE4aO97ba6L1Reya3XRctozP11jq884odDNrcKftOyZluKq3qAyDkjIiOMLaREdnocmGMnby9UdqZ3t94PHzLEMNte8F5CBJUPBYMzhs1Ut51PiKGnPXPpvFowWBweIIzeON1im4f9FHIWfOQQsV7p0H0IMh9Ess6g5RGEY0gXfdGcIkgfw3iQc5YkjOkoISRIpZSi6pHjSNNE6MO5ShSySWTS2VYhdkeCNq5FYsdBqmGUrsuuM1gxNKyo/RGbTuzf2UOj3pRsYFb793HDcTRpTBEmwnORAb3sbiY8M4iY/ByvfLrH36nhjxn+cWH9/zZr37J6XBgoNa4MupdUzqYrCP4gaXRpRCCx0RH7xWHIRWVxg8DTryCZ11j8okpTEoaF68WuXLjlgPH7YG9nliWI0+Hdzwd3nFev9DKyvnyPc4qHo1uGHnSoPcYeBzRz/gwE9JC6ZWWr8ze4ozHRE8ZQik719uGYQPnGNy7mt4hYunoQtLoDOnYZnDFMU0LwTgMghV0dzHaHSVmic6zt/FzrGdaTvRW2W4ryUfWfQOEfdtpTccCQ4qkGMl7ppRKjANrtTEjAilpV7W1Tu+dEAK320q58yh77z9bF+sdYLFt+nXg/nGtsTw/Y4buPKXuhKi+7xSUIpT8xBAd6ey9YMSCWHrb2TadppqXBecSzRh6q6QpcZhnyshkDzE6wmQR39nyhqBTV10Gtg/MZsBNmCVBDPSh17YpmfPHz/z6h9/xdT8z6qBssN8Go2u5yQPffPiG9w+PLMEjY2LN3BewgrE3homIzITowUzUqnVf1WU0rVW2TrvnHkdvRN8xddCmjsNQa6bsmeE7xg9yvuFDR5zupsu46i7OdEQavVTO64W+N/qjY5lnYppYDie211e874TgcUH5oz1YWnds2/aztthZq98HPM5bvLEY5xBj7trmDmgu9DB7loPB+q51VKuun9pWejOU4imlA3rk17DSP/r1R7FQWmM4BocLgvcJg5DvPuTj/IF3D9/R5vd8//mv+Pzl13fYBeR940vd2TfPPCdaK+z7oDYlVbc2GLFjhmNUS8lQc+HVnpnji0oQOwTjcF4VFKUJpWXsmLHjhBNHp4I0rJ21eOyEwyniQ+L19YXL5cJH+cS370/YGO4QV8deM1OIHB8iUgx2QMLRjaEN0V2DjcTJ4qPgnD5B+xAkNqxFkV0OvAlE0UmCLtBN41YuvO0HTsvM4iJPhyc+fQ287WfCJsxRmFIkOoOfDzhOtFpwkydMJ9J8ohvLdX3hegbXMg6VzO91o5fC5awsQ2stYZp4fDiQRKhGaANaKTg/1PbnAjIGW844Z/Fdg/ajNaQWMFZ32dtOM2qwDPMMo3F7e8GHiV06634jea1Hns9nYtTd/evLC62ryH7Pmb2odlbuO8g9q2iMOxhjvd10d3ePhK3rSu96xFpXdY9bq7RzGV1FdNHT9hWG1srCpI0OayPiBsMZjDhKLghvmDThrRLSzd6o+YaR/LPnSKSz3Xak7hyPD4yu5Y84BWxYkD7Y1w1XB4dZg+cKgqyIaI7VpQMmF9r5xtvrmU+vF374/Q+8vr7h3YynEpNSko6nI6eHR4KHKhtfXz6zjTP+ICyxI/aG9ZHRDZaJQLhbQAemD6QJXhxGoA8LRUDH+GmmEa3FRIeMrjvN4Aiu0e2NQqHfY0XWVYL1BJugB8R0mmh90FnLaGqsPD5atnqhU5kWwfkKQ8irA5kxMrhdbvRa7vyDGYggnWAszVsqldG1Nh6i43AwzBO44Ci9YkQYVbeoIkIfkdEtWJhmCEYIf7iX88exUMoQRh3EqE0BY3U2NVrD0+kRI15tcvGZPf+aGA1zUN/GWjK3a6HmoUCDBmIUV2WA0RWCK2bggtE3Ie+cLy8KOOgeVzx27ESncNdRhD4KSNVwchPKXvAm42ctHnvnCMcj8zJzulzZ9xfOr688TBOLf+AYDrQ6GMMTQqNTSJLoBaoYnWrA6USCauOAhrOO2oQincrg3AtmDKzzdOfoSYvpMQZKL6z7hXWbiCZgvWVZDmzmzOCG9RnvFx6OH+jdYGz8eU55mg/M84nWAVbyxdN7xvimDwWzIyZTh4q9vLf4VnG18nA4quFxFMq6KUbr6JmnhTkdcCMhzdAxWO+J04FiLHZkjNHsm/MeZzq9FMJyYt+uHH3Qm6IU3MMTt+uV8/XGd7/4Ba0N1vUN7x0xRi6XzvO7Z110UrrvDtUVLiLkUii16o3RO61W6v3nP7m79ffafbelrMgxBrVVtrHx/PQMEnQSx1h8XDjOE71mbpczUVSR0aUSQ+R4eKI6zZFaKxg71DHNwLtIN45hHD7N+OWISzOTaDSpt8wYBhsUKGHihKTEHTSAmM6nTz/wd3/9G657odTGeX1hmI04W2QIy2Him/fPWKPjma/nF76cvyC+MY+EO4AJheCvYFSnbMxAUHVGrx0n4JvgxKhp1ImG1mMgREfwapL03lGNQlGGvW9Muu7LnFh8jMpWcF71xc1QbcfZjveVNBkwGesLj8+R0iu5fQYBZxeNu1shzpqHrRkd8xXlTFqjTbKOqo2hYf3AB4sNHoKljY6xOsGlgjwdsvQ24J29Dy90gvU4+9dALjaGcLtWvJtwNHrXv0TeN0peCe6BJoOaM94EDsmxzIbeLW4VttboYogmMs8TxlnSIeqRXRqt7eRtp+xXZHTauFAlYCUieJwNlN3ivWhMoRnqDs1k7oR8EKi13uni+mSKKTJHtUa2esCMTM7CFBzH9A3OPfD1+oVaHPtqcEvCxyPegnWZlDLTZHFD4QSFylZfNV7ioZmC0rwDxkVCdPRdaG1QR6HWwZezEI1hmQ8UM5ieH/j2sd1ZmB1xb1g34awHW3+WbHUye9vYcufLyydu1wuhNyyWEIVoK3vdmY8eFxPeDubgmCaDCR1bC95mHIOyB4qvuJNldkdiioxq6M7ia9P0tRHc0KP9GHpDWdMJyVJbvkvtM/V+FDWol3tKk5Jg3ODl9YWnx0dC0J1r8FE1D/Dz7jDnrB3cpnXIn0LhxhgtEzRtYqzrisggbxtGNKwOwr6vGtRGQ9ij6+SNDUpqEmPxMfH49KwLm3eqTPZwjI/0ablT35s2sIbmgkNKPDy+4/j4RJxP+LtptO1ZLZH+pxvV6XSKv/8Qg+kd6eqH+Tf/3t/j7/3u73C5fY/1g9Y6e93x1nN6WO7JgMHr+czr51ecOKQdqCKsvRIPHVIhhsoYG0MmRPQeNKMRhmC8Z3SdzBleCN4So2OeJlI01N6oQ5FmgiU4hx2eulu6FZbgdS4/6DQOXh9gy2IQseT2SjKdEA+4YbSkVB25wPlyo3fVQNjhSW7gIiTr6N1Ru3beZ++VUoXBWqjeIEZPBBhD7eMOXB6IHbhgcVaJ/d5pnjZn3Wl6G+8bhn/0649joRThumVChNgDFWHfGzmv/P6Hv8c3T6q6vN0uBITJGSbvIBhan9hHwRjNWZ1i5HSaCEsizEfEWbZtZfU3druw5wtCoYw3rES6WJq9QgQflUzT+2DbtT4T7jdICoEUA9EHrLNc1yu5bBzmAyF60jTj7OFnSELNnegjYcyse2H0gNgFPz1Af6O2C14uCJ7gH7Eyge1s8kobKzF6ghWcNXgXMTwq9dw0PTZKpXdHzonxtnHcjtjosd4yLc8YVwiTNgNyfqXkjHMJsPS+83qpdDHkCvvNYFvAibrL8TvJd55CxAbHvmm4WGygG0c3nhgNKQ7iLJTRuOQL1/LEvGjAPViLNxPWFpoRJntgjImWyx1rV3E+YI3yMIM/ctt2Rh/EeSHvhTnNbKWpRKwVcs3sOZNSuofJdYHL+35f6NDmB7Dvm978xtDu/u7r7YqzjjG6RmBKRsbAou7on5uKKRGsJh5a2dVGmDStsOdNVcSoNx6BOE9Y0CmSeMB4GPc7zzttKk1p4vndt5y++UBcHnB+IdeGGQrWCGkmWae7phCRZUGiesLb+srt0yvXtfLw9Mz5b7/x5fXrnWs62LfKh28f9FoVodTC5fWK755kVTe87Z1zrRyGu3vn3/D+hNx1FsZ2gofhREc47R344S3dDowXgjeEzl0qJ1ipBDuThDtaMJCmhN0qkisjwVY7JTfmaRBiwoUJa7XJJqLTbCl5nJuR/g7XT1zeVtbzG8EFnAgBf39AOej653CtEAKcYmAbg1UGVQJjdEYZSqHyBT8JU7KEaPEIRjKtDcoOebVI9lrL+usw6+2sw7rArWRyLxoGHQZrhH174/v97zN601qhgVINU4s69mUC3kITDfkG2whWR6+MKQQ7Y9ICzTKypdV7k+K8MrqOpXWTcU6PV4J21oaIKmm7JYakTEarms00LeAMnz//SOkbaZo4Tif1s4zB63bBGocpjiFdGz4WncP2idI9t/3GrdzIxSMpMBnDMA0PDNexTvDW3IksFecauVecLcxux9jM2oXcDC/XVy7+yBwfOCyJSGJeFmiGFCZOp5nVrLy+/o7BmdYql9tOqZ0mEMzE4h442RkRVDp1D88/HBeMDPZtgyEMInk4zBBMmFhOlrDomOGtveDzCewT3UWi0ykd5zTWRc16rBSLv8MaLBC9ZeQrAVSX2HS364NVKpB3XC9nvBGsgVoqpSohu4HuIErBeU8fSuNu7R/s/GutmBC0SSEDEVhvq1KlmnbJrdGHyjSf8G5gzdDmDIM0TRpgb41tu2KGqgk8HWssdXimuGC8Hp29/2nhVXxeivp+TA9P4Cewgev1xlY3vBO8CzoeGxLGOnABiRMmREYHd4/P/b///t/l85ePHA4PnNcD5/WFuu0c54XnhyPeK6zjcr0gw7FMB6xU8tCRxL1UXFCveQsFN63YAD45TO003+geWjeULrSqugc/DM56fINWC0M6bgyWYHC+M7nIsBGXZpw4fCuUurHfGrhBSAaOg90Vas9gLHOIzIeV42liniaC8zwdFiQYTm4iT5F9KwSXmJYTbj6qfVNUUdz3xmid677Tr+oX6sNQe8H6hnfaQbdOkC60bdAHSjvfHfkGY40EmUCc1jj/wOuPY6F0juXhga2cGU3fVE3O61Gxd336WAsd4ZYzYjzOWbZcdRRxFBqGZi1vW8MZRxyZEBYsQQkk0WLWgFRLr7CNhvEd7xbEVbp0atPjgU/go2H0zhiZ6BPBG45LABt5fjwi0vjy+pHX8yvXy5mn0wPTNPFy+YogOON59/yex8dHWiu8f/4z5vmRHz81Pm2Gl9crwU28TS+8mxshBkYydH+kFkukMZkABrJvbKMhrZGcJZEYfqf3SqXTO9R1kPPCvOxIf+Dh+MiyfMfj9Av+yQ9PXG4v/N3f/Kv88OlvU4rjdgUbB86Ds4MpGAYKzNi7qMfIWh4PMykktn1TI2UBGTra6afB5MB0S6uGXC7sQXdYwajzxoVEK5kxhqoaeidvq2ZlnaGuZ6z1GLEkH2ij46eIs43D4Uiw0NtG9IY5ztTW6dLUeVMNcUqaRUTJ8mMoL6eNf1CXzKX8HB1qXW/20YtGl+gYEVIKmHs9UdBYTLT+5/HJn3arjEYvhXGvg/UeiUHHEq1XsZa7wy/KaPeRWk+YpjtOLtN6ZVvPWDN4fv4WH4JqC2L6ybyF4LG9cXv7zNt65u165tef/oJrPmNjxNbIZOAXH96TooUxeHl74fx2AyLdWYVcjKLd6O6o2VB2gw8VH3ShjnGiJUPZBHF6n1lpSBFMd6ToCGJ0HlqU4uTt/R/eYkzQiR1r8AX1QBTI5f9D3Z+F3LrubX7Q726fbnRvM+dczd7fbr7qDiRaVFA8EKEgGD0wHigpBFEQCgQ9EUWPjIiCAUEEQ2IgaOJJIQXBEMiJSWEobKJJJUWqKvW1u1trzfbtxhhPd3ce/O8x1tqf9a36ygqy64W511rvnm87nud+/s11/a4F1YDSnumkCSlIp7hKaODt/Y4QAsvmzH63Z9N4tBF+q9m8xrsNbX9Du91gnAZWlnXmeDxxfDkxnWd0I7/zjy8j6zSiO0PXGZwtxBJYx8ASgWghG8KqCWeFS5YBT+M6LIalxO89o34jDkqtNE174BwSsUySP60CkZXipaYyug65o1QL52VGFUtImYgEkY1LJKwZsxjsCO2w0nSz8AWTZU2Qizg7SAqKQ0VLUoVVK4wrKDTeFzCZkjUpKeISsB7QipANQ7tF4Xl99zkxBD49v2Uaz4ynE33Xk5FFws3hnv3WcXv7mtv9PcOwxbct4/kkUAkzcH4pjJ8mXtzKru/Y3jhM51i05bwqdr6jaRTFWgKFFCUd0vuWogam8IhBNo1Kr6yxYEPPdF4o4YwuM5/9+MCb+5/w0y//PD/90T/Gv/e3/i3+/b/11zirjyyTptjIya4MOuCdQ2VPmC1rWel8ojEaO2zR1rLOM8u0EpeCN4ZNt6NzCjSs2hCCI6aFmC14h22cIOVyoWtblnkmxMBuv8VpzTydZWstA0C6pq00Itj2DUPfsoSFzgMojIqc5xP9dsvL8xPWeZlff+cgFKmQYl1Wck5473l5eZGFTxUWF5I4eBCAR+s8VilKTrROHDp9Yzkfn/FWM4fIfr+THPUc0SVTiiDARMuZqhAa0fkpjdKKxmihqgCqZFKciUVXMIZlmk6ENdAPhuIMum/JzqNiQZlEUZmvP7zj//V3/gP+4Ovf5ePLe+ZFxOWbbmC/7Rk6jbWa55cj37x9ZF4tXly6GG2JFGKRBwCVEapNRumCGVY0Gasd3htWlXBklFEoZzBB0xeFK1k0nwWKzthGk60BZUk4TDbYUkQLSqZtHbptwUMomnlZWafMeIQlGCYbiOWJqDXC2zqzes9gt2zbWzb+Bq0a7NBhOugag/M9MHB3d8c4Lnx6eObtNx85jjPDoGl3HcVFrPWUqDiPI6dT5vwi+ec5JHJU6Jg5eI9tfA2jQ5a33/P2G3FQlqLQeDRbnO/QOlKIKDNWIazGNE4EpKkhrJGcxIFQFJhsCDGzlsxpjuhjwDpFMyY2u8wweJQqTEsh5ARG16FyRkctWVI41iVjrGRYOw9K1RwUV7B2xW80S/5Imw8Y5cgp0/pOwtYJhLjy9PhELpBSIa9HPr9TqOTYb95wf/eKOcw0rkepzLAZiOvKh4eFlyWw7FY6c0uvpbp4CZnnvDC4Hb25odGaaX2oVaTDtj2980zxBeeczOPUKtpCpwgLfP3Vr/BuYLe/5/b2Dtfd8Wd/+x9nXkb+9u//33h4PLIugXNcmN2CbzyNGXAhcJoDYV0YmoaubTFK9J2laMgGsqbYRuaWeSVkiWtwxmGtJzrNOS6YIAqGaTxTSuFwOIjCoHr8hVSj2HSDQC2yRmnD4D2kBZ0CnZKtcJofMSUJim/SrPNECrK4sdYSQri2yTGGKwZN/nslpkWWQVUqIog0W0cCisZaYhhReeH56ZEYVrrWUaqG1DhDCQpvBOCSM3itKbXKVE1zjdK1xuGdl2WtMSIq955SLGFNaOXou+31PrjwSZVtBNYxjwhWs+V0nHh4fCbNK3md2AwNN4cW32SsNbyMZz48vfDwPBEXQ9cL4ch6JcmQ2mHNitKKnBxp3hBsYHVJlphqrYRwUaFYbfGDg0VGI0YVxDcoaL6KG6Egsb0aKGmFkvGtRjuDMy1zDIxzZJkz61Mhr5qkAslGdKPQzoBqmZYsUBpbSHNgciPOFfI0k0zAa00/tPT9nq7d0DRbXt9v2PT37G8+8OHpHfMy0/U7NJrxNMM8seYNlIJvW1zvMBiMMvTOsmlbXGPJJdOtE/B7f+wZ9ZtxUObEPK2sa8QaUy+mSIoalWeskhgHqxts02K1kJIhkoJYyHwOlNQwZtHfnceZeYnEkEmx0DQKkrgd0EYyUAqEmHA60TlNRMv2q4j4Fg1KK1yjcL6Q1Igylpfpa3pjiAuoojls7qEoQnxkWSJhFVzU0zLyy69+zqbbM08zCsklGacnrJNNYGNa1jFx/rQAmhxnGlosnsfoOY8ruSy0XUvX7FAhcz59TdIGv2QMDZvmNdu+xzjFNL8whciyrsTsWNaF/+j3/ibzujKeT9wcbliWld3mni8/+wlK/ZxPH8+kAktaSTmgi6NrbjiuKyEuJBsZ5zMhF1ISIrsyQq5OCk7LClXsq9yKzRaTZ8Ky4nLB50xeZRbXti1922BIrEEy1sNaaNqGrt1IGFyRmaFVwiP0egUdSFkxLQuahvn4IppI7ZnWREpBbnBjhCY0juJ4qbSgdV1kPBOjzKFzIaRUg8EyiUxrFIqFEJMcqnEhhpllbthudyzTmc12J6aGNKG0Iq2BFEW/uS6CFSsKjN2I77wU+m4DxlQEnWaeF9nIK4V1Bm81IBIxSUJMKOvRRfPp7Vs+PTxhlKU3A60G3yl2G0OjAwrHOGfO54VpjWRlKRSUKoDMgUWo5Sla7KgYediFqGCcaLyqoy6RTmEApVBaYat0xhSFtUJhV0bMBKloiEbcXUVRrKHtHNYJuStTKPPMvM7EFLEa/FDYDwW7NQx7aZM1DlIh55acb0h2RzIt0zFyGk/M64izit2hcHs3cLhRDENTF549rd9wd/OFxFiXhpIs6VYz3yXWH4BXhtY3OCvFlrMep5UwbOuDIebIv/C/+7/8sWfUb8RBmXJimc6EcGZdMilJxOkyL8zLiFKRvuuE+iHmEmJYIQmaKStQaLQqDL4laAVkpjWwzAnrVnJUWKXRylWYgiXnmYxECWgf2TSW84QcgAFmCtpmtIvgDKnSklU8Ma4/x6k9uXi0NcJGrOl3cU2yPR0snz595Of9zxm2e4y1rHnm6fRAItAZw6a13B022NViS0HTkGKk9RMHMl8tiaWx9M2B3fYeXzTT8RNxXlmUCLpN00DjaJsGHBRGUtbM40rWsM4Lf+d3/yY/++Xv8ur+nv1mQNtA13le33/GvL6XfBZkNqyVobM9r/cdx/EjrZfKyxvNaDXOKJQtrCGxlKpTTWJfw0UMEU3CFodHcGdt12ATOC0pg8s8oi6+Xevo+w1t03I6Pom3WAUUI05rwYBpI0uZFHkZJ6YpoKx4eCOB0/EZZTT9sOH5WXMaR7q2qaQgw/l8Il1gvSFcZ5po0eN53+BsonGKkjRLkuybHBZWNzFbLZVot6HrN8SoKCniGsUaVlrjKDkyno/kDFZZjDZYI/NTby1N15KUx+UaT5IyfQUXKyVWR4mfAJaZ8fnEL775mr/xd/4Dfvnu7zKmj2wPFm22oCIxJNbzwjFEznEVgnpnySYybFs2fSuhcFEe+L7f4hoLWqrQkBLORXb9ILlRQ4u6d5CrRTWvLNOJ+Xwkp0zX9vjGEVMQ3mgFIlvf0PUbuq6hsYrWN3jfk4tmCZFpnFjnVSI0jKRAmtbgW1XJ5QaywRSH1x2t73FOdI+UQk4iiUNnWQ4Vw8Zu2A0bOifZSsu8VGZpESBL0pA0Vnms1/jGYYzMjq1Sle4kRZqGfzSgGAJendFqZQ4L8zITl0xaV9ZFwrbSHCR7xYllLGeqy0KD9uRY0Llgi8aqgreGbDSWRJwjOkWMDxhvgCQOgc6xmgXvC21faJ2jNTCViRwWyhqZ18SsIa5CixHxcCSHZxYSymxJSA61VDOdpLqtEaMF9fTx8St+5+eK8fxEUomn49cs8YTFYvNAawxq6EhrYo2W07gKxFYpWl1o3Ib7ww9589kP+Khbzg+/4jh+IKwzS8j0W0duNIsB12zolCy3SlCsiwS9ozJPx7e8ffc7DL3jzZsbDvtbjGm5ublhev6ISgspT3jbk3XE6g2bnUWVR7QqUDKN1+QQiWsCIwDkXDKqSD7RWrPV17zSmB5jewwNGSHEa2OqyBshoMfA3d09WmtO5yPn8wuf3d+zhgWrOxo/4Bqh8KRYmJaZUo6s6zPrfGLNhjVlwrSStUYZGM9nCooUVmJOlVgfiEnmUFprqdpq1bQdel4ddjQkKAtPjw9Y67m5uSetowiaUwCtiOuMtQ7bdJAzKkUKipikoqZE5vQCOVS5jcL3PV3XiiWTRIhCytJaiR3POHJZsWZD9j2sgeXhiXdv3/Grdx94//E9p/GIaQRaLNUwLGdYxoxrWvZuoFOZvW1xxnG7P7AbtpXfavG+Zbvf4JrmKsLPJdG6ht1mi/cN3nU0VihAqla+L6cnTqcjMQXatqVpWmLInMcTKWfaxgulvW3p2g6jZMmnnWhhyRCXUA0AUkU7O9ANSijoxkhsbSz1tZFgMm8bnLMi3TJacInkimEUHa4VxzfaKoYBUlwlcdM4Gt/S2AZnxBWUlRD3U8oYbav+VkTSWosB5PvefjMOylJqkp8FEvOamKbAvCysS5LAohSJ6yo/tHwUxmqUdgJ7TZY0i5neeo9rLMPGoAzomoBnmFE1uyamgrUt280B7y1967C6UFSgzxobHI1eeXeOzGskhBWlAkVltFkxypDiC5mJmI1seY1l2HmMn4nzgjbQ9Z5hsyWXzC/e/SFzOEE+gZ6Zp8LeSgVjlMF5R0kzKXtOseBcYrtx3L664/XdZ9xvXxFeXvimGGx1HLVti8ZhVYcqllIivukIS6F3mXURpQA6s2kdrIbnhwdSeiGlhaHdM7gCjYFZkfLMujxiPKTU4ZoNhQi8oJXGRMsYX7BOc7Px5FI4n2ZOYRWpUbHElFiXDM6hjGXT73Ax4Y3DGcN4PqMlX4Cbwx3ONLz/8DXGaFrvsEa4jd45Idy4ButEBN10Lc61KOV4fDmxrjPjvDKPK9patNOsc8B7x5wSyhpybipFvloqcyJnAWR0Tcthf0PfOHQJTOcZ0BirGNqeddaE5YSptCddErokrBLSUspFljlhJqZA3/bEuBKmheNjxOofkPIG42QLXjIoY2iapqLgrBy63oL3qOZAKc+cz0fefnqkFM0PP/stDtuBkMVK6m1LSIV5N1MKci2mhZAyCsfQDAzDgLMSydH3A8OwYdv3+Iqek7xzja+i/aIkZ7xrO9Gpak0IK6fznvPpTEyiQJDvWwmXMwgDoW0blNakkK6YPWOFC2uVIYfEtMwsy4xC0bQdXdfj20Y89UkOuRhFoSAmC2nfnXNoY4UalUT/WnKW5IGU6u+wkchdPUCR6tk6KzlFRfTQaQ2EMLOuUQ5i73FOsuCzKrL5+p63P0mudwv820BT//5fLaX8M0qpnwB/BbgD/l3gv1FKWZVSDfCvAH8B+AT806WUn/39DkqVC23XUohoJ+21wdCYlbBU0fFayJfAKDLOFmxTUFaw89pprFK0naHftrimmmoCxGjQJROWiVgWUrYUW/BmQOUGsiEXmNdEXzJez9xuMseQGReFKlZmMyqTysISCyVqcllJxYjm0YsQe9g0qFahLLRDz5/97f8Edzef8/T8if/oD/82zw8vWOfBaZZFaNjDZoMrhjjDNC7kmGk3BTu0YvciMZ4Xnp9Gnh/O6FTIWiJUu22PMpnWe5KSbb3B0faeoXespXBanjmGQOMcqu+Iy8zz4wPmIDdI4wwhdJSsKDowT58ETxVe0XWfEbMhlmfx9roOlQNd63BNR2c6QnwmzAGnWlrvKpi2RWvJXvZKk1GcppnGNxhVaiyC5cOHd+ScOOzEtZSKZNg0TYd1TnJ1rCXFhFUi6L9NRQTz+YVxWlnWFRVitb8K/DnFhO9aYpRMG2Nq/ksRn7L3lu0woErh4fED1lAfPgPLfGaNEe0aTJX4oCUYruSIwaGMw3eWuDrWqsNUWrPZ7lkXSU8kB5E3LTPWNELUL0rgxgXxRjuHanuhKa3SyRTT0Pc7NiFjzOcM7YA2he1uy+Fwi1Ge89ML6xpAw7jM1YEEvjIaUxC9qVaKHAPjywurcwx9j7Ueb8RgUIqI7p31DP2AUpp5EfF/37Z0XgLf5IARV5Q1GrpOFogpEddYASRaDiHrJGlSGEj0tqdtW1Dgncc0suhSSlOsoOlADlejDUorSsrkgnjxValSr1RTB8ShI/NY0WJbJ5WiAJMlrgVVKvoOAX7bAtpUCI4s/jQG/f0yyj9RRbkAf7GUclJKOeCvK6X+DeC/D/yvSyl/RSn1LwD/beCfr/98LKX8KaXUXwL+WeCf/t6vUCCFjPZgjEIXydVWIVO0YtWGMcgQnoIcptqgrVQf2olEo7FC9/FasP1rSJJ9smbSDDnVIPeCbM1tZo0ybA+rw2ChBMa4onwhc8Z4SWjLpeCdpfMtxjaSsbIWOcDJJKNZS5Wn9BrnMq4t3N3f8md++z/J65s7Hh4+8fD0xOPHB2LIOKNJWPq2p21aTMjkxbFMI8uSySR6bwhx5OHxGz6pJ3724e/y/vkdG62xjWOZZ0wzUoxBzSJaJy9ok+n3A7vNHUtKfPMxktuRxlhi6wVokU7MC2izpTFb3NBBlHFNnM9kXuj6G2zfsdn+gOOoJXCLPUafaZqO1g/M6gVixiqpqg99T/EWooj1FVraXWCz3aNLQaVIDDOn0wMpLwzDhv3+IOFvKaGdQWtP41sJn6uRtKmA84quL2xivGbdnK1lmmfKKCShEHKNNrU41+DbVqKN5aREI4ufxjfkMDKejzgrVb0uBYxD2Yr9U5qsFBFV55yBacx0O9C6wTqP9ZYc5TX1jWe3v0Vp2N28otvfgW6Z1syUVO2gBHbrBJgKrpOLe32GWNCmp2s6ej8S14VtPzAMPfvdnt3+gELTaivMAe9Zg+RYF7Jo9mPkPJ6ZprEK8CPjNBKPiWma2WwG+j5gs6gl+q5nu9nQdR05Z9Z1QRlTUyZT9XGLNVRrXa2wmZSS4PGCSLGUspTiKuIuCBzFWlonKEKtK+TZaNDyOXIMco0YsFYOu1gyKRViDGijUEpmxE5J3pCYCxI55aqNjaSc6nUiXSNKPh81tVNrTXYiyL5wQ60VM4T+eyZy/wMclEVq4VP9T1f/FOAvAv/1+v5/Gfif1oPyn6r/DvBXgf+tUkrVz/P3/hrVkJ+nSDSGXDKtkadiUR7VJFY1kYvkKXvd1rJc0gzRFVcviljmXEjHyHlcOU8zOoLJCouSp5eTxLcwnzmezjwbT9u0tI3HIViwZSMVZDYrri1Mx4akwDQi9va+kMOJErVAWktmXhKmhjalNlBSy83NHZ+/+Yx9f8P5NDK0A613LOtZ2JBWnEDaaxaVOJaJ53kEPFqJlu68OfLO/5I5Jj4+/wy6yJzAY/CmY42ZskxM8xltLSGMGA07d4tqM15TBbuBtnco3ZKTrhd/IBFIuohVUsEyZ6bJ8nJ+5rB7Zujvya2h37xm5AFDIM2WvCimIOg0ksXmhtZ1tLYD49GlpTVbBu1xOFqvWOcjUAgxMY5nun6gaTf0fa0uCjjb4JvuWjlIHEHNcy4ZZwqhhpY1bUvbrbTNJNSouiQhZZQSV9Bms6UbNlhjZduMbDwVCVVjJlQBslQzmkzjW4zWqBJqrjnYXCAr4Z6uK9omTLtHaVkSYOWQ9sbR73bsD7d0+z3GN5SkcaUwp4W4LERnaLYD2rUCGs6rAJvXzDJHufGLsESdtQz9gHWWZV55P7+jdrhstxu00WyaXn53ztH3PTlnxnFkmkbWVWbB0zRxPp/lRnZW0g2VHEKCoQs4J9pSkFlgzol5nokxSPrht+dCBSNLteZ9A+RqEEnkZZZiRmmssTS+qW6lJNEQ1blEyjXgT+Iycq6fo0RiFhaohI6Vb+fKRRxrqmRULpQkllitJXLYKHOFXKR1RV2z3ZWQ2o0WqIaW6BH5eb7/HPwTzSiVUgZpr/8U8M8Bvw88lXKVs/8K+LL++5fAL+svMyqlnpH2/OMf9/lzgTUmrMly2sdQvbOapSQKEW+0PGWINRNH1SdNYl0XtDbkbMlo4pKZHleeXmaO04ozhk3X4GzBoEAL9n5eEglY1Mwyj8xOBsDeGuZ1ot1Zii5sNgmVEnGR+E/tGkkBDAVVRnSOQo4GcixMo9BK2sbw+v6H7IdbtDF8On1gjc/sdi3vPryIdEkrlJI0QVqFTobWdKjg0SWTtOM8GczpxBKOhOUBrAjxQ1lpbS9gBh2I8UycFXNMhCVh4jecz89YrwgEuq7DGkPRilwUrW7IJULxqNJB6QCF0pre79FxgdmjF41bHCVbhqIJMZHLSlccJYExK+b2Bxjd0XhNa7zYZ2PBl4I2vm6enylpRZfENI0MuwHvekF7WVODoWZ2Q0vXSYUVoyTvNd6jMKQo0IuSA60zdF5Yj8/2RSC7mKt+UoLupf2zxkgsrxXXi9a6CqgLyzyx2+7kBnOasEykdQEN++1A0AnvDTplod8bQzaKdQ44E3Beo5XcSsYYXCuJlspKVYoxlCicRZWjkO3XmRQd7W5HNgVVAnlNqKiZp5Hj8UgIAVA0TUvfD5SSeXp6Zp4nrLWiDjCKNQQB7WrNZrPBe0/bysZbaxg24KxjXSPn84lpmoWYVPM7YkxiqZxmmqahaWR2+GsHhRXv+yXqQqpKLQeuE+fbpdJMKdekTHlQXTS+IQSJBEHhvJfXIGeRaFVoyZozKBkrGe2w1glzMstDQxZxkblySqHOW50chAApBmLOpJyFNnQZu1QzgChUrIwk6s8f0n8MzpxSSgL+U0qpA/CvAn/uT/Jx3/emlPrLwF8GaFpNVhFdBtlDOYtKEazmfDyLyLVuKEmJJYg+MloHRW4KazNKB2mpZotJGpc9Noo/OGcoygu+KUHMpc4vMxlLjoowr0w6opXBG02Klu3WMRiF7gsveWWaR4Zhg2962s5yHsVyCQmjFMpIqPq6TmjdcjvcoFLm8fjM1x9+wXH+BuMKJUfmWaGKpwwW4z2+69Cu4NpnmcOuK1l5sjUs60yOK8Y7dNcLpXtZOOcjtjToLHCRGFZYG1zuOY0i9N6Vjr7dy/ZbR6mkrBYQqjIQDCU3UBy+aVGtY2kjoTtBArNa7Nzik8GogWAzKwsWoY673mHvHboYUpZqaJxGztOINhqUYw4LRWnJY15PWCsPiBAzjfNgDON0lthV76+ovZSiVCFZsYaFdZ4w1qPIGF3wVuO1ou86hmHDy+nMNJ5QSiICQpSlYDke0ZUSpI20k40FTa7tZhTa/bpKEFpcgUQMis62ggojYVOmbRqsjajKp4kp0GiPtV40hQqavscNPShLUYaQVsZlIa7i52+8pP+hLGW7lWXC85nx+chxPDHNC/GyGa9ko1SXUPX+EdjFNJHPZ0IURN+6zBIy13UoJT+/uy42avuZ5WBUStE0vrbQhXk+M8/zrx2USim6rquxEdJSO+fxvrmCSEAqWWsdWmtijBWOLB+/ruKGW5aFsIrywAb5HDkL/DmlJFHSIUignFF419C2HZ0ThKCrhoIUI3GV2lZmjNR/15VVIPzRXDJGGwwFcmaeJ9ZVbK1aa0lpRdWHcf7e8+ofaOtdSnlSSv014D8LHJRStlaVPwC+qn/tK+CHwK+UUhbYI0udP/q5/kXgXwTY39tiuhXNhpzBdT05j3gS5lh4fFzovKVpHDrnuuwwWLVKPIHWpCrxSRQaZzEe2kYsh8ZLxZZKEVKK1bhGYdfEOBU0FuUVURVCyuiSCEHaAB0tbacpeqFpMkklnPNsmhvSOvJuecsyLhir6kwnYbDkIkCE5/GRb57ecnx64Zuvf8XzUZh72mXWeRKoxdSwWwfu+wHXGuaUeDo/y5Ow0lWUFstZZxu09ax5QelFABNKkiU3bos2CtN1eLehb3f0rhMikELmMlnVPCJDSQmnnbhMnKotUkshMY8LS9uhlVQJpSA0F6UwTUKpjCoKr61oELWAIFJCQueLkoPYyPuVaZmWlZAziH0AXWoVpmRmdDy9MLSb6wzJGM00TVhdSGEkx0QIi1QGSpIdhW9YKEkoP0YhABMl7MJSvs3QSTnVmzvTOlfbuTPrOqJLoOv2lJwgZ0qJ5AzTPNMMDrKIljFZ5qRdg7WaVBKUQsyZznmatkPZhvF0ErqQbepQXCx05ETOkaZp0U5RWFDuDYqeqBKfnr7hq3df8XgeKVlhnJFqcJnJRQ4+pRVrinIQ1ZmfaVr6psUoeDm+cDy+oJSq1WWDUoFUwQ8ySyz10HU4Z/F+x7LMEjJXK862bTHG1soWqe6UkNO0kfnehRwfQrhWjroSmtZ1JUYxb+SSWcMqCY8hoI1cN6Y6lnKWKJIlrrwcj0CiryCMrm2xyjKtE08PD7wcj8zrQtu2bDYb9tstKhdyrNKhnDBWS3KCb0SClBKnc+D55eHq4nJVVaGsFiTcP8xBqZR6BYR6SHbAP4EsaP4a8F9FNt//TeD/XD/kX6v//X+v//+/9X3zSajMWrfiTMb7PauRsrn1GZRlmR9ZxoLTQtQZXMNcolw0aFSS4PjGSAtgs+CqNsZihpY1JnIWSQMkmmLpnAMrroB1lfmLwogmLkciMM4FUqSPGrdRuEawT5QgiXZGnAtRNdLiKbFHOQ3b3Yb74Z7z6YFv3v0OTw8j07gwnqCw0jYNOa4oEusy8fw00tge22gcPb0S21XX7dgNG5xRkCU2Qix6hZyR3B/dobPFaoMpIkNxrqHvNzjf1IwY2WI2zuMbzziOzFNkSWe8CzjvpN0tGecsdKCtvjIVxUedpEVKScATWpZc4/ksMztjcDVXvFAkVMuaGsWQJYGQHaSG1grJfdN3WGt49/5rQsi4TUMpSmyga2CcTuw3A+t6WQw4kAEKqh7+xhrariU9HmXYVLd+VhusNjRNy+HmlpJWmV8qIV7O5yMpjkznI7thw/k8QZJRTS6FxjuMki15iAvGaZw2FCQIy2iDd0IWUihCjLRK4wy0bS9IMd8CBqMcG5NIcUHpQtP22K6hdA3gUbpF9wdod4z5PS/HkRgCm80W1RpiXOtMtru2uMY6OttgjCZr8I2nZMk0SjXet20us8EsMR7OMQwD+/2BmCIgr681hq7rrrEQl6UKwLqutRKL1F8u5nzGNY0semIkJ6ne+q5lu9uz3d0QQ2Ca5kreEpH9NE2El8A4nliWSSpR53GNF+PJMnMeTxcdFX5qUeoZXQrjNHE6nVgqYOUiJUo5Yq8RtEUiLpDRndb6uvhx3nG42deRg7TwRYF1DcZ8/1H4J6koPwf+5Tqn1MD/qZTyryul/jbwV5RS/3PgbwD/Uv37/xLwf1RK/R7wAPylv98XKFmxrIq+taQs84qcC31TaG46VIKf/d4D4xF2G0tjC9oZsoG0RqwyGF0oS6Lb9Wgt1HCzKpZplnY9FdZlpfXQZmiKBVdQQ8I6TZgjy5TIRfyrWUEpkTlozKIwHXLTuV4WFs3A3bbly/2duGPqtg0gqULfN+z2PduNJ0WDVwM/uP0Jne7ROjJ0HU55qVSsx1vLdruhbTsRZ2PwStP1HX3bYa2vWSYZKJILnhOxROYo8atOWYyypLRSgBwDUYxkKApt52kacYosYWGtdr5lkUMu5wyKOmPSpBiZa74M8O2WM8arhvNyiE7rIouknFBFtq4hhmv1V1TBuhayUHGWMNJ6yUZ6+/59xan1+OvsyvD0/ImUZ6ZJValNQRshc5NyXe5JZWmtwzvH8TSRKXjnabueYbul61rJIDeaZZ1RSpNjZF5GlulJrHU1j2foW0oMDMOAt/I78E1LWFfZjtYKKKVIWBIxaIZhwPq2xkpEjJJWV7mGstmA6dBrxiWPTSvGgnId+LZSxr1Uv4Bxlv1+h0LcJkZLe6gUNeju25lfMaBcwWotsGNl0K2mc5acEs5KAuS6rJW+JRte677Vql6iMeR1DSI8bzx91+OcZZ6XGqERa4UoS6E1rteZ4fVA1gUM+Nax2W4AxTZGaYOR1rfkQmhXchIJlyyAEqXIrmGaZXQi36/FKM08nqoeMiH5SA7ftTTeSxQImXWdMTVPnZKvG3HrxPWlta7XgUGpwrpGgTInWEvEmu8/o/4kW++/Cfz5v8f7/wD4T/893j8D/7W/3+f97luKhdOxoNYXhtaSWBjLI9r07JuGN/cbpinyzS9EztL2jm4LWWVscdiscdQZXSzYRhD2qYhbJ4dMOK+QCwaDVeLeUUrM+5cZps6JEhI2QVgTrjVCSWkyyjgO/RtuNp9xf3jNYXvPpt3ircU6OYC8rZnjdTnhvEEZRUyRN9uVL3c3zPOPRa1gHQqpzhrnsU6L86HKYTQibi5knHX4KvSNKcqsbhGnUlEFHyKLkizrnAuucXKIgFSSOdM0IkWJMTKeT0yzRCc03tP5RnK8a0SCIMJEH3c+nwkpiTQkJcIa0NYI3LZ4rBKitLJGBupGhOHOOlSNNdBGKrsUAkWvrHklpZmyKp6nUSKKU6LkBZFseMbpSFhX2q7FupYQC95rvPOElKTlpd48WUKxnDW0jSfXhYvrOpq2p+TM+XSSSAAKrbcsi2xyFZZNtxXdrtGsa2Do5LAIy0LrO7KyZJWIVY2hL6MBFNoIVd06afFkXOIwKYGvEJEExgzorqNoi1JOyuGqMYRMPL/w/P4D03lEAX3TY5Qi1G2zu7bQkhm/rgvLtDI7Q0mZ8TQybDfc3t7Rei9g4yIPvVSJ8qHO4rQxhDBLdVdnhjEl0IrNZksz9Gw2O6yWCr5t2/r6Szb96XTi8fGReZ7rPa+wvqHfDbRdR1GK48uL/D6UMEVzCqJFNTK2uB1ayTuvkQ6hPlidc2w2G8K8QorkGMCourGuh3tF5TXKkPICRVOUJq5itQ1ruGa4t01DrJ93WRaWeSGGREq5LrTk+izuHwEeJUXjy5bllGF5j/EF4w3Ti6K/yXSt56c/eoU3hofHM8aJ7zMVoR3vrKVzljXDy3GmxDrA1RkdFmxc2VpFMR6MgGnP84SLFms9rfP4rWHT98zLzDoHYjZkl2h6xbBp+ezwJV/uf8xnt5+x2ezwvhWJiRX5Q9t4hs1Qn2gyn8ypVOCwwjnFsHfEYSWGKnvQYp9y1tF1/bUqtcZcNWsxJlC6bhUNJsumsfGJ2LXXGVHbUIfikZAEFixfpy6tUHXjKMJb772MN9q2Vi3ylnKm1I1xKYU1BB6fnlhjoGka+l6Ew9ZaUimslcbkTYNzLc4117mmVADlSumR7bYlWYPKjmU5kuIkdBsUu/0Boy3TdMYZzTD0KC0PNJQwARqr5YaPUcg1iC6Rgmj/lPAVu2GLazqO55HztNK0A+N4JsbMbttjlbSCzhRSXGlcx3ma2AweA0znke0wQC6M41lYphpCWOnaXpYVWuO8PIC6tqXtOrQ2NO0Ge7iBfodyG8qsRK2BQum2Snt+XbeXIjw8PPL2/QcZ/cwTT0+fyCUxDDKrC0HVhUnVGdfNcyoZbRIkA0ldfeyllCrM1tdW9TK3BInPWNa1VoiBYbvlvmtp2lZSUKuIHIXoS7U8zLXW4tCBq5BfKUkBQMF8nnien6qET+aQSktn0ziLry19qdemCMolVbHzHqs0ZeD6+b13Ve9YN+fjSIyF8TSy1Nlt4z1d18n4R+mrfKnUn3MNgWWeWepiSeA7Fu+8cET/UcjM0UrRuRatocSRwTrMMKANhBBpvWXben74W6+x3TNPL88YYylrJMwz+WBpty1dtig08zRhFFiXSBvwjSNnQ8ESswHj6TtL5xs2XU/fDzRNjzKKNc3Mc2BdE0VnslppnefN4Ufc7l8xDB2utWQdBQ+XNDlHUBk1ZihZqjzfYXXNdvGS31OykIWmaaakFYOmVGp3qe4PpbWEjdU2yZhcB+iyFCkqC+ew9fS6u2bFKL6tNI6nMy/HF87nM95buq4TXF2WGFe5ZKkbaVU3phrr7K/dWCDbzP1+TyoZX+eYpSimaeQ8jrRdy2azZdtv8F7AtAIukUMyF0GcpQu5HshZEdbENJ7RJdB42aoOm51g1mKkbRtJbyzS5jZdi0IRY0JpJY6QmNDGyO+sVi8pFbxvaJsBb1uBOUfBdT08PDJNM+u8Z7/t8c6wTgvOKJZKqBmaDnJimWZ2w4AxmnEaAUuwWtw4KclycFnAi03w4kG22tF2HWp3INseZRvMdk8OkuNOmVCu5bsUBqUMrunY3dyx5MQyTzzMM+M8MY1nHp+eoSBdSyOzw8Y1QKFrWlzT0DYOYzMvxwdA9JzWieT5gp3LlYt50UQaY7i7u6sH0IrSQl6KIXA+nphH6TratiWTq61RVCamivWtkwykFAIxrAJJXlZ5mCWZGyoFMSzXa0rcPJGwXpZEVsjzObPdbOqCh6umMueMokj0szbija+HptaWeV0oyoCxtF1P6xyuypVAgRbFjLYWn9PlXTLzbRqctlc1wR/39htxUCqUoKZUJGdH1+zotxusFpr5PIPfaPqu5bN7hS1wOp1JUTRQy5pYkmJjHYfeEkwLRejZdzceZRrhXRqL1R3GdFhnaVwjc44qQ3Deg5G5TQqypfv0/J6UAvthh/difYox1k1wuQ7WY1hY57EeQIbdAL5Vgkfr+2sb7KzGGFjXi+RDgLXTvKIUdF0L1f0gLV65zgYBrPNoLS9biCKpmOb5KnlwTm7UYd0QY8Z1jn6zpawiwQgpMi4LKYbaaq5Vm5YJo2yFvRfdo/GOfruhzdIC5Vq5LYvMrYyxONPTtXupsp0jpXzd0Eq8gtCLpmkmp0COKzEE1pQwtqExLY3v2A6bSnUKlFSY55ltPzDPo/j1q8VujbWSM5ZEZg7i0uo6Sz/0jGum73qMczwdR8F7GV9f34ZxmmsGtEaVBZVBF8W6LAyNxRmFVk7YljGCkwouriPKN8QMi5nYbvesSqEqkUaE0EUePq7Bbl6TmoOI5FUSUtJ5JsYjZtOLp1uJtKUUhWp6Xv/4T/Pmt37K6eE9fzdNnNIL+UPh48cPhLiy2+1ZsmFZFnJMdE3HbrOlUAhxQWnNWh+cn3/2A5q+Z1pG8T4rgzOOxntenh9Y10jbthwOB5ED1cP09PxSt9kyr1Zas1ZQDSVftYvOOcIioW3GWFASsXK5DrXWdXYqB/R4PnM+C6RZaO6ydW7bhqaRzqhrVd20X3iiQnv67vWvtQZj0TViQxsBrSi+/dhcveCxmg8uMiVjhPCFlnFAyfJ66bqg+76334iDspTMEkasSnjboVKDy45SZtIM53khp0LbdpRk6Nst6wxzfqazHYPe03DD0N3Qbfoq/jY03os43Ph6wYI1HmscxhqRLaSEMZqu69lsNljv5EBZY81VKUyz5EPHKNGmWscqg6iBVqVcrXEpJbzzsoixlhg06yxPL6Wgaby0vSn92oUwL2L2N0ZwUFpVYkpM0jpXaYjPhcZTN/XpCidYaqb1xWnhnKVtPdbUUCVnSEYRZ/HLXi4cmfckFFyH85eK4TITWteFtc6jstIss1QO1rprJkzf9QCkNP9am3eRjiglgV/LdBQZj3a4dmDb+jpLS3KTk7G26uFChKKwpiXEhNaJdZ3wTSvzxVKwKqFLROuMdYZhkA4honl4/MQ4jqxr4u7unjev73FWEhgNC9tes+sETlGqIiClQtO2dJ0sBFRUON/gtSdHSfMLYSWEuVZLim7oMdZg24auH9BO4mUpSRY1YWJ5OvP84RMQ6V462rsVt72j6KZu8EVehrEMrz7ni5/8WR6en5n9wqv7N2iosArpGuZpolBY15WUJUxLG6ngpmnm6emJQ1Es81h1xgLyNTWmdVkWjPG8vBxRtbq6aCe9b3DOCFQ5J6x1rItQ+y8zyxiDQEYoQglXMM/iN79cO7JA6eQ+1IrNZpBuo2SKMrRtW+EU4ia63FO2ztK/e49N0ySpi/oSM1soMaCql9t6jzNSKGhrZDZMqdehsD61lhk6zsrg4/I/VUr2fW+/IQdl4TQ+4xS4dkOOlvmUiEUI2+u0cDy+sN/v2G73NK5n20FeYDfseXP3JcPmQN9u6G2P0QqlZR7mXYcycvPHmsonm69cBa6iRKAKh6ktSikZ6xT77ZZhaMlZ2pWUEsuyyCyxXljLsqAuUoWUWNREYwzbtsMZQ0SqFtt+G68aQ5IhtBHiigzfC0YZQo0gvVR7vm3IqqC0wVp/tZyl6iZQStH3fXUfFE7HsxB6lNiIl0k0iEuKpCJtTFFyweYo4wJjDaYO1nOUpY2qXlxZCMk2O6R8vaaUBuc9XddfD9VL6uHl5rxo8IDr0zuHCGSsdyxRlmyNcWhlaLyVNq1I/vrFK7yuAnDWFNK6ymEaJJPZ21ztjgXnDdY5YpSDdlkT6xL48OEDzporxFfcWi3aaGIIMnawnlIUyxrlJkuJkDLeaXzTYoqWpZVW5CIzS7l+FdY1KG1p9jdw/4biBzivpPWEsS0hBL76+JbWKO7SjSx/mi2qbb5zo16wX5b7z36Lz+6/kkz5VRYTzhmMlpYzDVuWsDLNE42Vh6tSipQDx+NzFVBL+uNclmsFpar1M+fCsnz7gO37nmHoOR5PnM9n5mViXsbrcsVozdBv2O33zPPEy/GZaTpVudEG56wsRJW6VoSpUubXauv03mOcKFpiLlWKVLioB787SwWuDqPLTNRamcsarWT2H2PtuBJxWTmnjFbfdmIXnWRWKxkpIKyxsOiqBJHv9RIs931vvxEHJShyFH6kzBoUyxxY1plpOYrhP0Y+LglLz+Fwh9l2dO09N9s7Xu1e0zQtly2iMao+1ZxESisJ0CJ/OxcydZN8ubmvRv9URIKjCm3n2GwlNCplyVA+j+fqfY3X4XKMkZLTFQs2TxOn85nDbi8jBOeqbcpXi5c8jbVBFhLVahVjZFommU1NE9M00/cdbWpZ11WkPcagcJIFXWSTl/NyffJKi/LdC0/iS2OIrDFgjQR45XpBKZtxdYkkXlmp8GMST25MkRRtpUNbipIsmhAixho22w1d30kwWpa8oktlehn0p5QF2JCLeHDrEmJdF3onn3vXd3Tecp6eWObA3eGeFI/4ppUM5unIxt9inCdGkQY5ayWIzDY419PYkXE6S5UdG4xxtI2Q1WMMHE8nGu9x3qFIOCtibeul8vZtR1hnclpwOpHzijCgLVRAc8wJY/zVtwyIg0Yp+rYlF7Bug1IDqndM4yfC8Yl5nohTYDGFpU9s4GrVvTx55OaVf/NtTzvs5EHZeKZ5JKSVmEW7Omw2DFWlIB9SZ8HZslaAzPF8xjoLRtP3PY1vhe7tbX0ArteW9rI08b7Be8fLizi3lFI47aRS3/Q4Jznp59OZ9x8/oJTiiy++4NXdLfsK1VDm21nmeTxzPB5xxjNsNmy3W/q2/XbBV7uwaZq4RHZcrh+ZhxdJjUwJMFglD0LlmnqtSoFymWeq+vS6fP2UklgctUaVTDaFrBRLqItNpUR//I/CMscYw647yIBVOxni50AKkqaoyegAujjCGUKj2QwHNvst+81e3CQFQlikGsxK3C9VUyV6OyjY2mY4vK85KfUJWEqRgXJKko9dxdPfbsQUIa7XA2EcR8ZxrE8uS4rSGimlWNaFT89PpJx5PL6wGQYJj69Px8ty5vJ1LxfINM/XjVzjvUgxjkdOJ+Eh5pw5Ho/CUew6Yoy8vLwQgmypla5t8hKuguhL1WOqZUshHE+vrfyOqhdXK10rQLm4jJbWyDlPSKH+Xc0SA+fzyLpGdoc9280GrZFY3lpBXOacuspLZE5U6phBWqEYZ6noyAxDL/PNKDo5+Z5Dfdi1jNNEiIGYEq46UcZxomkE8yU640zrPI1dGedZZpeqYKqwXWlYF8nyNqrQ3/YSJpYSrusl6GueieskRJ8cmM4n7m5uCPNCcEGgBAoihWSVfH0q8UojPMp+oOhEWk+UYmhuD/i5IX/4wKfHj6gc2AyDRKk6mVNfyQwxUnQELeDc2zef8dUvf48wTUzjyPH0jLWWw+GWfhhom5bQy2F3Hs/ye66giGmcaPeOza6nbTuZ333Hv922Lbe3t9cO4Ns5YGEYBl69esVS0W1933N/d48ykqFuzES6U2QUT0/PfPz4yDwtvHp1x719RaMbXNPgm0aywYvMqpd1Jb8cr86Yyx+hD0nnIR2fx3vB65WqyZXscaHJK2RGKqMp2WbnnOr1Ktt40X1+a/e8aCxb38qIoia9FsQ4pdU/JD3o/x9vxiiaFtYpsi4TZUpoJUgr7WUG5kuDshssLRZH53q27QbvBFYbYqzcQcE3FYrceNaI/IACKosfV4suSyoemYflLBkTMse5vD+zrkLjTllIKs45uralpMy6LBRnxQNbxb2ZIg4UY1mWheenJ2IIzG17BQdImJW0oSiuT+5xHMkl442VNqlWrCkl2k423MuykFLk5cWgtSFEeTKWRcC0pUjVZvy3PwfIk9cW+XrWOHzdeHNp44tcVJeZqaqym5wSXV16rUHmVNoYdruB25vXOOdlxlz1dTlG2fI3jSxwiniUldIY51inEasVbbul8bBtrQR6pRWj5ME2DAMpZawprMvC6XiibYxo/rSuB6O0XMZIvpKq21VvDMc4czyeOc8z52mREUPKpCiE69ZpXh1kSdMPHUZrwjKC1aQQab1nHi8PXRl5zMuC9Q6VJKtpnUS4vtvtZWttG4b9a5TfQ50z5ylxfnpHySvewW478Pu/8zu8fvWKZR6x64Jt5KCUeVoiT2dsbyg4dvevePPFl3z9y1/gG0+XBsZx5OnpiVIym35D4z3b3Q5jDdM0MY5nVJbfz+l4ZLvdiNa33hOXAzGliLWmUn9KBUroa+fi/YZpGjG1Gm27Fu9b2gro2G537A43HI8vPD488vj0ifGriZfxzP3dPfvtnt1mz5vXO2KMPD8/k6q19FLNAlcrodYy0nBVz3u5Zud5ppDphoGu7VijMA4u1fw4TpzPp/px9mq5lBmsqT9vgEW6raM2tG1T7+dMDLEWT/57z6jfiIOyAOcyM5eRtkSa0FAWMHbForCuRTU9xu/o247ONXijyQSWVV7kUpBKyds6g8lCQtei8L/oqtq2Wr7qoBocpcjFc/lFyxIiEIIsSnSV7YeQ8CmSYpAXsMgiSJwOUpmllKQ6Q2GQBVJRIsmYZ5mpbDbiWliWRZIl6+B7mWfGcQIKvmlFZNy0xBxZ40oqibZpKdkxTSOJVDVm36LtS8kym60VnaCpuJJVxH6Xr0/zlIQ9mOtDQusErBhjWUPgPI7ytC0yK1NWUUyVL6GqrU22xufTEUqm6wbWdZY/i1TFoGh8y+Z1i7OaYWjZD47WWubzhHEQ15V92ZCXkXURF0iIs4xOTEsxVg4sa8S5ss74zsrsztRRQ4E1BkJaCFksdzlJNSoPkcymb+i7BtsI1zDOZ5xKrGcBQugiW/62EdYkNrNeDtl2hzWavnVkMjFnbvY7cqm/k909qD1FGexg8FPg6cMD2ig++/wLfvHLX/F8emEcz/DywLbdolwvdaWxqDmReEYPd3jbsNnecBr/NvOy4E2DPzQi4M6R0/TM6Ww4z1MF8Ca6rsFtLemTkHqM1eQis9qQ45Uibq2r4yNBrm23O7FdVseOPKAj3ntKKXzzzVtyzmy3W3b7PV3fc/fqXq6R04nHpwdOx2ess9zs9zSuQyldJWORpllYllkyz2uXJgtOjbUeYwu5lKtIXZJWkxRC12s3o7VEQsS41oWoHLjn87nSjXRt5y+zdLH6phQrvWjh8Um6n3meKVnspG3bfe8Z9RtxUOacmaeFNUVwARpZKsRlIU6KvnTovsGaBneJBSiZZQ2IZdteaSCXQ+PXBsNG44yEb12Gw9M0Vs9nueq7LnO+ywUkli+ZP0n7ngnrQlxX8b/WJUDjvVS9rSeGyDmerlvkFGSBcgWKFiG+yFJh/bXv95JLLW2FbF0NiLSjkXa7bVug0LbNdanUdR2bzZYQAsfj8ddE4ULz/jYjJudc/bf516QTgtfyyKEXGaeJppH2+3Q+c57OVfeWMU2LsYHzeMY6R5sz0zgynqU9vL/7TA7+ZeZ0PjIvI13T0jjP7X7H0HeylW881hte/2DHdrsjhcCHt7/P+eNbQnqm6zbonJjUGdv0zEIbJsWC0XLIxxhR2uL7HrMkihLJStt4rO1QO4FhiHYwoXLisO1pm47et6zzSmtbynqW18hoXk4i87Lak80GVQqtN1jfkLUm5MIcYTtscd6Tgmb/6oA1rs68HTkGUniClFDKMC0TQ9/zwx/8kHl+4XQeMd2GPkwSbVF0zY1PhLePbH64A+9p+q1sbUthXmdc49nvtzhneX5+4enpE/PbmabpuDncyAMvSVV+EYVfFBbaa5q2IackFVQRH7cszZxoL624lmTmJ3nhF5hGSlJQLPPM48On+juVyqxtHbs3n1GQwy6mlbIk6fZqa+2c5WKHvRQkQkR35BIISxBos5YMKlO7h5KqqiRmkWDV7yWEVA9b0YHO88TLi65nSrkWEa4i2C7FwDLJjN37jrYRMpSpERl/3NtvxEFZUiFPUIpBWYOxFtdoFlZCzEwls8XjbCMLkVo56O84HC6bMgE25LoBVmgFjbP0/YD3/lpFdV37bSW3fCuGvXweuWgCVIlBLkI4STFWUbjF1gGysUZE4lkkEtM0StCYFew+FR5xqXTlwMw0dahutPiXjRbCtrViCyxFNq9aqWtmiyoiBelaz9D3+EbyQrz3vLy8XJFaFzQWSJtzmYVardkMA8fTmePxhYJUu+J0ySCgOqZxRClZGuQiTD/f18WT1tequimyiJqqNOT0cpRqQivm+VuYsGD6FZDxTi67NWYiwHnE+46h7/nyR3+O+f5L1mViGo98/PobvGl4fnkAMiZnuqaReVoVDqew1shpRdEGtGXoG1JRWCO/eyHXBLTK7LcDw9AKtzCsmOJFNmY9KE/XO1KJWOfY3XyOcg7XtDTeQZhJyyzi57anaTym7Qml0HQd6hJPUGfAiszt7T0Pzx9Zppnz+UhOgV/8/Bf8xDfcf/lDLsu3y1ucZggreI+1jpvDDV1jMBiejs+slZzTNh3zODItZ9mXl70I8ouMfy7XtnRchbRojHbCziyKYei5vb1lXWW8cz6frwVH0wxYazmdXnh8+oRWlu12S9N4pmni08ePPD1+QFvRr37WvOHm5kDbtjw8PvL86ZMkYGpzHWdd7r+cq+ytFLwvWGeYxpkYIm3X1YNZDvdxlKWuQUlmT9cKtzLECkgRDamxohU9n09XeVtKHmMs0zTy8PDAsiz0wyCa30aKDglM87TNPwKttzaG7XBgXQI5JmFFFrE1FaPRvsE2HmsVRheM0jjT4iqZ5tJGXmABwPUQhFIPEnfV8omx3l///qXSKgVykmGX95aca4RBkeUNIG14PfQu2/BchbhLmJhOJ/r6ZNaI7MI6B87gjCFFOcjjulC8RLk2Tjb2vvHiazX2qskMIZBTlAPhO5KgIuZ0QliJcWWa5GcYBoEZyDxTmIrS4ov7YFylCnh+eWE8iz7UOMuyrjwfj5zPo2zU62szh5W+76+2O+cFhFFyYF6OFAZKUWKXzFJxPz2+Y41JgBLOkQvEUtBWfl/H05GuH9huWzabg3jkteXD0wONhTSPUCLb3Ybd5s/w9PLM/vQFHz++Ja8nslZ4Z8gpCtwgq7rAM6RcKEUzzQHTaLxuIGecb7B+wClN18hBOCYoMeEbQ9GO/e0t2+0tfbsjIuzHL378Eza7Q9UoZsJ5QlPohw1GQZ7PFB1pbIMybdX1JWFzxlL1hyPOeebxTEiBeZo5nyZ+FNa6HzKgMuvxmXB6ZHi1AyfbXAi8uTkQtgPTPJKVLFycM7x+dcDbP8svfvEzXs4nxmnisD9graqLCvl7Iax4Xx8cWh5aYY1V8+homo4YV5kFVp+7QnE8nnh8/MTxJNX97e0rtNaM44xxjttX96KqWFd+/qtfUrTipz/9KX3f8/x0ZB7PRBK+kdjgXDKd7rBGsawTp9ORsC5oI13MbrelbRsBdpAZ5zOPD4+sIdD1LXNYcGdXF6GaRrV1Vi0OrhgiKaZq7hD25cVqO2y2HG5uaLqepmp+oaCdxbfff0jCb8hBaazh9Q8+J6fCy/HM8enIaTxR9ILD0TXVuF5nfk3b1pQ5i7GqQkMvLazABi4leYzr9SC8aP0ukqBQyTgyTBatpW/kKZRzrG1tf10M5Sz2wYsVjCL0ohQj3XZDYw2Nc1htJOR+mTlNZw7NnpvNBlWjCD59+oQms9vvMNpxd3cv7ZBzYATXFmOQOV2KqMrYu4i3L620NrJkSCmx2+0AiFFa6nmeRHtnNc8vL1JVKpnlhFVaHFurblUXPqok+kYExcoY1hCvAmLv/RXAevk9Ckk8fSttmmaenp6JYUU7g3GWTdOhc2ZIMl/dDD2bvmN7uKXfbKX6raMGY/c8P7xnfH5kPD3Secvru1vevLqD1x23n33Oy8NH4jpd9YTj+cj6ciSVTCyQ8mUBBq3bSFdhZKHnjKGtABLnLd56Nrc3NK1nt9vwxZc/Rhm5EUuNud3ub+gHyTRaw0psBnwjlV7TDfIwG894p8CI6UApyaSOa8A7z8t55OHDe1xjCYtogl/fvqJpGnJYMW2kpMLDu1+xPn3ipvkS54Q2vxl6lt1QK/MG29xdnS9xXXnz2WvaruV3f/8PJVY2LOx2h+v4oW2l+o5rYAqBGAK7/Q7nxPRwPD5TrrEPmXUdaNuWZZl5eTlyPp9RGJqmv4rVrdVshh7rNoQQeHp6JKXEV199xePjI/f393VDLaMhsZkWUiqczlEUAmGt4AquCD/tLfHDO5wRGpFzlpvbG7nVSq4zYqkCS4EQQ6Wtm7qQ0fS9F65BWOm7Xu5nra9LK6UkCkJV/SSAt5Z8yXn/Y95+Iw7KUqDfbRi6De04o5tvCJ8i07hI22nlhzfWMgwbNtsNm2FTBaXmut3KOX1H8qMrkNRe53iXC0wWN+Y6t7sMgGOMpFLYtB5THOsqF7zQfuRQySnX1lisees8SntRimjxGoH29n1H01imZeTl5Yl1HlHWs64L43niszdv2O8OGKMZNgOn85lpnumGjrZxuM3A8fmF83iUeZn3OO+rO6hcRwxr3Yovy8WhI9VD0zSV7TcKFGBdCSWzpIDS4I1nDYGQI2WBnDLTeCLnzDKvdL0gyk5JBNqX8cRms7nOfpd1IYRzFRcXnp4feT4eOT69iCOjb3F3nn3TcHt3w2a3wzrLZr+nHwa22y3OSuSmMZZpmdndHPAqoPNKWk6cnr8hrkdcdwdkjGtw3R5tCy4EsJ45BJ6fTszryrxE1hB4fjkzrzP3d3dX5Jt1Gt00+Kal37S0zUDfbek3Pbd3t2wPNyyLLCfkIRDYH/ZoI2FXYukTMEU7bKAfMNbjDq+J8YwqM0p3KFrpiIwiriIpeno+cppeeHx6Yp5m8m0hpMg6HgXMOy9MpyO5KKZlpV9ngYskIXU/PDxRSsJ+N6ZBK3aHHf1mAG351a9+wePjJ5wbRdifhMLe1IfcPM8iJl9nvvjiS/aHHeN4qrxJsaUej48oJZnex+OJlAo3h1vO5yOn4zOgRQvZN4QY5HCuFPVPnx74+c9/zps3b7DOM40TfdcR1wVUEl+1a2i7DmtbrOmv79vtdhhveXl+YZ5nAZu0LfTI+xbJFbfOYqyTz5PlAbDGiNJCrOqHgaaVtEtxPElwWC5ZjAXWVtpUQDsnMRQF0h+BlPzRt9+MgxJFTpaYNJtuIB/uUAoenwwmW5xp6fue25tbbg937A87aWmN0J+lGgxXEIPMKETN33Xddw7SCxRAZhjOuauG8eK0WcIqEIamqWiqQFhX5mURUHABoxWpGvU3fU9GwBbeuetCZrfb45ytm7ZAiCsvpxfG8wnnWmIpPDw9cbPfUVThzWdvWJaF4/GZ4/GZvuuAyDQeRWC83V+1Y/M8s8xL5SKmK2PyooEMYa3SJnFdNI0QXBoFO7MVPWAuLCkwzTOtkyf0OHY8PT1eQbjeOfa7HWuQ5c7F+jbPsxCjT2fZjs8r87rwoWoFHz8946xjs9mhjOfV6y+5u31F1wn+K+RCWBem8wtm2DIMG7kJZoezhY3XeKMYnwrh/In3n37O7iYKNs04tNMkZdGtw6eI77fo4yTuqRQFqzaeqge4vt4i9pSlhnE03U4YA22lXCtpKedJQr7a3mG04jRO9JuNINfahmG3x7Ydyrd1828pxmJ1Q1EFhZUrWhua/p7jwx/y8PYtBc37bz5yfjmhtGacFj5+esA3PeNRsGXPL894a5i+GXn88I7b/ZZ5PvFyPNO2PUrBmta6jJyYppG2E8fKZ5/dVxeTr9vvTlJDq//ZWs8WwanNy8ISAz++u+PmZs/Hj+95fhYtboyJ4/HI8XgiRlkKxbTyy1/+jHEcefPmS16//oxh6Pn06QMhhDp7Vmw2wn18fHwEZejalmWeCVBpWhCdQinP/nCoWuAgPnwntt3X9/fX5dNFetY0kc32gLWG83hiDQlMxhbNpt9RFIQs6ZO6bs2dkSRG7xtCWCW8EJmlO2OucRcxVpLX95+TvxkHpVJQtKcUi6ew6w+Y0lJyR15Wtt0Nn735jFc3r9ludnSdx1qDNhrnLjDaS2Rm/rWZ5UXUfTkcL4ipS8j7xWVztTxdEVQzsar713UlLIvEGxjR2s3TRNe29IMMvbu2vc47lTGcTyeUUtzf3jP0A9oaPjx84u3bb4gpc5xGgjX86Me/dZ0f+sYxfjjz+PiI0xpnHaHCOTKPAq8tinkOnE6SC2OMEdmOUtcMkm9nsRXMWgEVOSU5JAHrG3zf0vcbjDIsy0xKkd1uX+e3zXW26ZxjWxdRMieVdmez2dbXTxGzjAKMFXfHGgPjLETqkmXcMU5iiXNa8bIunE5Haae6hvMYeHk+4q2l7fZsD5G8nvj49me8f/+Ju8/+DK7pJZq4URTTk4qWzHC3oNVjtZiupFTY7XY453h8fGQcx+uDtms39P0Gq8Wa19a4C7lTZERj7AW4IFtipcUeaV2D7weUbynGIZH3F+NhqPeahJcVCkVZfvXuHX/rb/0tfv7VV4zjRIkrw9CjPyliWnl4/EjbtbL4UQq324seN0588/aRr999jdOezz//nKKErbrf33B3d8dXX/2Kx8fHqom09P2WH/7whzw8fBSg7dWtouiHgfv7Vxhn+frdWz4+PPDVV2/RCkJcUcpCkaXK6TQTY77qH4UrEPG+kSoPUYRstwdiShyPTyyLXIObzUYefK7DNy3aVkYCGVVgs9mz2W64ORzw3nM+j8QYGEeZ4w6DaEU/fnpHybli0Qq7/Q2fffY5bb/h+PLM8+MTVmkWJ2OQfjNUpqmgCWMd9VA39aXK26ZpQhfY7XYMwyAb+fO3DqU/7u034qBMOXE8PVG6HTiDNZbtsIHGE88ze3/g1c0bbvc3lXmXCdMsN6bWxFUI0KpAqS2phKWXa1iRMYbBDdc2/DKjMcZibaltu5jqQ4qs80IMK7FWbSoXrKqE37rYWOeFbd9z2B04n09CvNnuaPuG41FiWTe7LUPXEVPi888+Z7vZMM0THz89EJaF0+lE2zSEdeF8PvH08CDLokrw8Q7O05nzw4ndbkfbdhijUKpgrBEbX71RZLYqs9rvOnfatiNnsYldntYxF5oiI43jNHI+n8k5yezXOVIuuKaBUjifxIF0cc+E79jMUnVE3N4c2G43/PCLLxjPE+M8kmsaX8oT6yLtoDWemAoqJ1Ja6PqZaZpl/lJkBowyNMOG4XDDvBrc8Aq/fUUz9LR9DyqRUMQkoF3ddcQibZbYL0UGQh0PXNmEuVTJi2Podxz2h+us+unhBZRkQDvzrVMpBaG573ZbNIr1fMbkjGoHscUiQm2xE33rslEYjDc0d695XlbOy4l+20ORh+JpeqKYyPP4xKv7O37wxQ/YH24YhqH+nva8+/CWl3Hi9d1ABoxx3N7tiEkONqUU0yhMABHOiw70/v5eMHjHI+s80/cDv/3TP80SFp6eXthv93jXcD4esdWaag3s9y2Hwx2H4w3v37/j4eHTVX9otKHrN1jr6vw7yrwPy3Zzw+tXXQV2rPIwtA1d1zNsBjKF8/lESYlhkIdrySIWT0nAFtvdDu888zSyzhPkwsePn7DW8vkXX3JzuGe7v8X4hnlZeHn7NcMwMK+CK7xkhCu4ziWbppLjU6F1LSorrLKEtQJ8Y6CUyDyu/1980D/69htxUC7Lyq9++RWb4RHXCAqq9x3Wi9Vw2w90bUPjDOR6o6aMNuLfTFFmD433YnCvMaQXl4ouoIr8Ki7b8IsI21oBDRRkWJxiqIekbOCnZRbpTEpQ7Y5N1/LZmzc1q1hoLhnY7LZopRnPZ7bbLTc3NwxDf/W8ruvK8/MzOWf2m4Ey9Czzwkt1WozThDfSXpcMa7okTBq06RjHiXmuUiZVmOeVUzxKy2RlhhOTtPrDMGCMZbfbA2JTszHjG9GcpVRYZplXdl3LZjOQa8rfpXUX2G/AfUfakYqQzsdxZJ4l8ySsK1opAYAYzf6w5dYcaqULfdexLgHnZMYbQsIZRSoZ5x0xJaySClpXxQFqYJ02fPGjP03WjmbY4dsO4wTCsawLp3EkFmRhYxvWtXAeR87jSRwaIBIVY4DCOJ1oO9HOdV17lZFd1AQxSRKjdwJeUMjoZVkXvHWS0JjFwKC9OHQKAnkwyldq+XcsicCPfvpn+Sf+C/9l/sa/83/l6fmBpvUoynVx2HcbXr9+w+3tLdZK29p1fdXGbvgzf+rPsd1sRPOqFSEsPD5+Yp5ntHI0jdCSJHVR0fiG3XbL8Whl9msNh/0tawg8PD4wzRN96VCl0A/CKT2fz9eOShahhsPhFmu9UIm0pmlaPv/8cw6HA5ckxstY64Llk5TDtXZxcn8tS0WvKYXxDV3X1i4o8Pj4wPPzMzGunM4nurbFW8fpeOJnf/AzjHf89Le/ZLvbo5VmOh5Bn2it4cs3X/B0fOb4cgRgXmaUEmtmG2SUdDqdrnLAGL+NNDmdjjw9fbrO91XR9MPwvWfUb8RBmVPh/ftPHNtPsvrXmsY5DrsNb26+BJdE9a8KVhkyQhYZhg1N669LmsuSwTVy0YYQaJzHdbZaHKU9BDBkVAZV8V4hyIY5h0vUplgNW+eYUsK2LZthqF934P7+npwzHz5+ZEkR28ocsKTM/f0r2ra95hh//PjxujgyRvBncgGtIiuaRpquZZ6marfkal0E5CKZx+uN7aoTqOSKJ6uD+mmaZD5jRetmjEOpwtPT81UeZa3koEiVpWXJUVv0eZ6uLqfzeZQ5U9fSNV6+tyJWSgDnLd0i3vbTSUAhlyXDpZK/qBDWNTDNEzHW0CqtKTHjGlOxdE4gsaXQdltiipJ5M9xy90Xm6eWE63u0cxRtCKmQEX1kioElrGjbMc6Jx8cz87yS81R/57qivER8v9/fXMcuF2mZUvD0/MzmsMNcvPFGtLJ393d0viHMs4AadjtoWpSxUDSKDHEh5QXlLVpLmNjlwCxJFBY/+slP0L9Q14Xjfn+4Ysa6rqv4MoG0pJgIqzBDh6bDG8McEtP5xBxkVLQsC94J9OR0OrEsi2T3GIk2UaVgjOM0nknlgdN0pGkczmlSltcmRxndXBgFF52v1oph2PLmzWdwvdf66yhLXlt77T4uJop5nq+miovczntRUVxm/Y9PT9IF1q7kcDjw9v1bvvnmK6CID15VXJoxjOeR/f6GOYx8ev8NOWU2mx3OGVrjeI6JVCIP5xeenl44n2Xk1feicRWLac31aVusNdUdt8faRpZeJaPtPwIVpUKRQiLYhFeGJRROyzNhnDi0d4z9SPooM8jD/oB3Mj9y1uKMr9a1SAwBSmHTD6Dg+XQUjzfyjA/zTK4VWlgXYkxXX2ioSxypaMS2tUyTCHeRyMztdlsrEGlpx3FkWkQ+c9jvCXElp0LXdaSU+OUvf8lhv8caI21QChz2B25udnz6FFmDhIDFksjzVJH+Al9dF0HwN3XuGFaxk03jhB4Mt7f3NI1Ig+ZFWqG29egqTfrw4SMpJrpOnpTPzy+14oyM48jpdGK73Yoral6EejRPQp4phZfjszgpyo4UG+Z5JucCp2Nt/02d7XJ9Ui/LcmVtXg7LEANtI9XwZvOtMwOQjWV1F4UUefz0UInqGwoFYxvazYFBWbKCt998w5s3r8klX+kwIQTGaeE8nWXrXV1TufrLlUpXVcCyjvzE/Dbee9mMpkRJAk821tK1ArmYlkDRmnA+s9/t6Q4H3v/u7+N0YXuzozQOTAdYcjhR0ozxv04tv1SV8/nIu7c/5/n5A5u+I2bJsBnHkWEYeP36zVXKdj6f0NoQkddEtLBiT2ybjpTLVQcs45FYr4/2iuRba3b20A8cbm744oc/5O27rznXg2xdFqZ5xhjLZtjSNA2bzfYqOIdvNcY5y0EaUiIXrnrathWqj9b62jUYo9nv90zTVEEtArm4BNCdp1EeVnUsJoCSwM3NDa/uXqPubnl6euTrr79mmReKEsH+0/OJ8Tzz6v6G4/GZuCSenp+xjRMSkjUYFC8vz5zPp6uZpG079vvddaYu3zOczxMhJqxxdL2nb1pKXQJ939tvxEFpFfRkGitb7DHN6Naz2exprSOGiRhWycZoG6HNtB40AlPgWyLyRV+mlIzUU0yc5uVadV1mkTGuv2ZZzFnI2G3b4ZpGEudejlzyR5SSTa1uGqw1dXkUBOSAkGmWZQYUj4+PV73mw+OjDPtzZlwk4+Pl+Mx4nuvyIWKdvvIbQW6CpmuvVY/CYIyvRv+Icw0xJEKQzORpnFmWhca3NG2HNa0IfHOm72Qm9OnhE6fzmUsQ/XeVAFLFRPqup2lk9rjdCBRhHCdOHx+ulkhTnRXzdOZCXrrc6BIDsFx1qhc7prXSlhlraRexjMV1IcRvD9ZXr17x9OmBaRzphxanQJuak9MKjCEsK0+PT+z3ezmEZ5FETfPC08sz4/mE1dTq0V6XXd57co589vkrttsduWTmZeTTpwdZPDQNh5uDkHG04ebmhk+fPjLNIz/68Y+hQEiR1jfiXdIaSIASOG9J8s3y6zdbKZlf/vz3WaaTzHODUMWdtXz4+PF6vQJX9YKAhgPb7UbE83A1X6Dg9vYVS1Ud5Mw1T3u/31fE30IuCmUdr+5fgVKkDPOysukGXr26Y1lmHh4eyBkOhxtubm5QSvH09FQ906KJ1caitKYbxDQRYiIGyU6SDmSuEjx/hVys61pttjKGuGh/c4icpxmjFb5paHeHap9ceXh4oGSBw+z3d9h7xzgn9vs9+92OEBY+PT6xrBNtM+BcTz98S+F/fnrk5uaWzz//QQXOqPo7jVxoRJfubLvdYp2vM8mCto5xndhs999/Rv3/drT9x/tmrOHV5zc0O4Oxns0cWNaI8z1rFVovIRNjZrkRO17Jmbn6tb+LLEs58/LyIlGrOUAqlPwtNaWUXKEA+ju2KgEENE2Dd4KHaptGDu1ppGTBjy1mxTrLdJIN8Zdf/hAQrNo6z1hj+fjpI8/Por30dRN+Op2wxrDZbGqlFX9tTqo0NI2u1aDMdOTmzjw/P9cMmI6+7zkc9lX3JtCIa+72Gnn/7iPON3R9h9UNIYVKv05YY+n7Hue8MCZrVnMuma6VediwGWT8kIUaHUKQ3J2XF8IaiSnR1JCnru9pGpGTxBjxymODtPXhuoWXynwNgRALy7yidXUozRMxrey2W8ZxwlnPbr9nHEe6s7hDvPdYY1mmM6sxfP7F53z48BHrRuZxZBzP4h55euLd+w+8vBzRyuCsQlfArsS7Wqxt2W235JQIIbLGhZQD+/2OeV5o6gy2cb62kmdu7+5oakxu4yy7/Q5te4rWKFYusW1hjjSiD+e780mlFHd3t/zOf/jv8PzyxKv71xWP1tDVtnVZlqsio2k8F2yfuKwGnL/FaMM4nnl5eWKZE961nIJUf7mIU+10OtF1Pafzmd1uR9O2nM8jv/zlLzCt49Xr1/S+5+ZwqILyhXmSB1XjpQMa+i1GOx4eHwGu8jrXNXgnD2pn7VVlIfPsb7mScIk0znw3WC4EOXhPp1PF0lnGSSpqGQXNeNdKFpJxTHPg9vVnshVfVm52B1Iu2KbF+55td+DNqzuO52dSXrg53LCuK+fzKHHgGpZF5vm77Q5jDH3fV3RbS9P2xJgqHT7SZkdcpu89o/6+B6USDcS/DTT17//VUso/o5T6PwD/eeC5/tX/Vinl31fSW/1vgP8SMNb3/3vfe1A2ls/+zBegCzEqzBrp5ozJts7rIhqL15pyiQxQsum+0IJKSUzTLNTmeZbALGdpm7YSSdSVqBPCUqMSvkXRA1AUISSeXt7RdC0GTdtIFrDAAhZub++w1vH8/MTDwwPrNBPTIlaoxuMbyzQtOCuwWLRmiTJDuzyJjcngFefzWVqfbVdblMLxeLrKLE6nI84Z1jlgtUYDHz98YK1P9RgCfdfQtp6iFFpp1nnl+PxMUooPD58YvFRLm82GN9vNNW70tC4Yp2m7AaMNm83AssioYbMfoAiQtrGWrml5fHyWgbvz3N7fs66L5IyniMqFrm3om5bGWNhI6ywBU1KhZgohJN6++4ZxPFJiwHnHsN/z/PLMvIyUFPn09Imma2hSYl7kMAqpcH54kSpLZf7gD34fnQ0fHz5yPJ34/d/9XX71s58J3DbI520bJzG4+SINa5imhHUNKUdySdwcbjHGsKwLrrEojSQgpkABfCN/V1lDOwwiOncFFWeUlmA4ZRWma2BdKE6Jx/vimU8ru74VuVRYsFYzjWdykIxwU2DhLFAXo1GqAFmWIxa0Mmyuc8BLvIaQ6ZsKvy1Z/PjTNLEsM6/ve5mlNy2/+OUvef3mDdY7ttvtNevoZz/7Q7bbLfv9gZyzKC9aOajatmW7FT3k7e0Nzsu82xkJ1LvI6S6z3Uu7fuku/qjcTjoYwe3d3d+L8UNJoB218yhFpEJN07CGwDwtdE3PsNkQwiqGiUVgLInEw/E92gS++eZrvv76V9zdv6pQFdlDNNVdp5VlnldiCldAxjzPnEZRkHjv2Wwqnf0/BnnQAvzFUspJKeWAv66U+jfq//c/LKX81T/y9/+LwJ+uf/4zwD9f//nHvumK+09rxlWlfCmFwTR4PBaxLyqlSFG2dEqpujBJLEtgHM9CxalMyq7vaSos4vJ2gVF4764v4mXwvK7CQ8wZlhg4PpxxygqwVElL2VQiegiR00kkNSlHjC54A09PZ+ZpxvvuKuzt+p7GNcRVZqEvLy8iVRoGgQcbxXie+fTp8eoS+m6oWCmgtGFaAudJqo+QEsZ5XAWjytO2QeVC21SIbCnc7PaM5zNPT08AVwahUjBsNrWCltYppkhve7bbjQz2jSWlgrGWbSrc3t6xLAtPz09M47lmkEtUq64EmBgjm41IsKZprFIRLyzIkgkp8vJy5Je/+EOmaWa721KQdm3oWrabga/fvmWaVw6HQ+XVy1IkhJXj6cjzaeLd20fisvL1u6/45de/4vHTRzIJ6yy+bVBWYXVNpjQFp2sl0TWM85nT6QWlNX03cDydmNaFeMzENXDY72UDXrmH59MZ17yw6XuOL0/45nilVSmzoIylLIkYR7Id8bvPUNoS4xlTJmJa+Mlv/Rbn6VRDyDQxJ7y3FK2Z1ogxhd5YdCzsdgdCWHl+eeB8nvHuga7rrrq/eZpoO1FS+Npyt21bDQAChFjmmYfHJ/phYNjIw3Ge5usG+8c/+vEVEq1UYZ5mtFEcj0eWdWG73dF4zzTJLNs5dz1ILtfm6XQS6k/OFTn4rS34u7SqUgrb7e5qMb4G11W1wcXwcan6lmXBaMMyz5yPkkfftZ6HTxMfPj2ijOZ8OvOLkumHDd575mWm7RsUhXUZKSkwbPf0fc/DwwMhLvS9dGTPz098+PSR9+/fs9vt6NqOT58+sD/c/MMdlEVq6lP9T1f/fF8Szz8F/Cv14/4fSqmDUurzUso3f9wH5Jg5fnjB5IJxjrIqwmllcuCHDmUdKQYJSa+b5HEcq25KTP/TPFaoblM3XA2X0PRLtGmp5J22ba5zlRgjp5McspuhY7vdYUZNXCI3t/vrnM05T993PD09ycE8nWm7nnEZJUN7nCg5s9vvybGwroHWN1ilMM6ha3VyudAus5uwnghBFg/WaXEzhJV1lTjPlDLzupKycCxvb+Qpfz6PHI8vNI3n1as34v5ZFpJaUCXRek/b9iy77fVrOW8p5UJJEjGy0IouOc0yJljXlTevP6PvO/HX16wga6Vq/vTwwPF4lLAoUsX2e5zboJSwPpvGskxVsG5F8rSmhf2m5eaw5Zu37xjHM7/3B7/Dx8cP7Ic9rWt5fn5mM3xkvz8AVbpDJFX2Z4qK9998w7qceXp54nR8xnlL71s0Gm01tm2wRkOWsY61Tiz0qnA+H+V1UIZlDmz3O6ZlJudI3h/oWllcPT49yra3H4hxpe8HpnlhffdeuhDv6g3veHp45un4hHOWNz+IHF7/FiUuxOVIWkduDjf8hT//Fwgh0PV9xZgVnPUo5bh/dc8l8tU5w/PxmVd3n/PFFw3We8bTiRiCQHNvh6sNz1jDOI0si6DSnp9fKBSaruGHP/kxlMLT4ycZq/QDbTdUYo/ndDqR0oqxlnN9zYdhYNgMpJjYbreAFAXGGE61OPHeX7FpKcWrtfXCl3TOc4lxOBwOV6TghacwTdO1gr3MyS+hfcsiXUrXtuQ0M88j8zRSSiLFmcN2SzsM2M8MjTNoY8WJllIVvBfaVtw3vkbnllI4HUUVcDjc8IMf/JDDzR3A9cGy2+8rYvAf4qAEUEoZ4N8F/hTwz5VS/p9Kqf8O8L9QSv1PgH8T+B+XUhbgS+CX3/nwX9X3ffNHPudfBv4yQNMYHn/xFpSl7waiVozzhGWkbTqsEt1VTplpXui6gZyqFs0L4qxtxeaotWEaz0Bhuz1Qshxal0WDYNN8BXcuzLO0I03TEFPL49MLYV15ff8KA8znkXVRxJR4eFA0ja+4tSAggZCvlJ51XdBW6OTLurDbbdlsBuZ5ZpwmjNbXLeCllc8Z1rDyxeefX/VlznnWNdQLTwz8S1ivbpNxmjieXvDe8+rVa9ncr5M8FLRUiRLnCnt7w7qKsP18lijSvusY+kPVwWlhf6bIsqxoZ4nLzNv37whrRGlFX7NLmrbBG1e/D8Pz8zN9L3nsl8PXGPkZ+67j6/NbQlh5/fqWZZ1RqqHt3hBT5qc/+iEpZU7jmU8PD4zjyNv3D4zTwvPxmXcf3tE4sZ/5vuE8TZzrFnoa60Nxs+G1b0BBSiuHG3GslAJxjSL5yomUIylmSoaQEu8/fCAuKw/dI2/evObhUZZV87Tw8eMDqhTWkAlRhP/bUcYmDx8fcK2rxJ0GheJ8HsVOuATWZeLh6YF/7M8rxvOJdRolC2jTM5gNKmdQhmkaCTEhGeSeUH3dbSdZ3EOMpLxCzozVuJBz5jyeUWpk2284vbygCzyfjxSj6ZqBTb/FWI2tZKXGy3LIGitwl1yqNTfRNAalBp6enipjckEraJuGD+/f83u/+zvsD3uapqXr+mpIkNf3PI68PL8wDD3DMLDfyyJEgNENDw8PPD4+8u7dO3a7Ha9eCXXo+VmiLC6qkJQKXdvR+Jan5wfmWZQobdthGovTwpB9enyi7TeUAkO/IYWV3rcsIaJTIYwT03niw7sPvHv3npACr16/4s2bN9zd3fHll5/z9u17/vAPfoG1hmE7SDXZdWw3W9rWXZdRf+wZeBnC/gkPzAPwrwL/PeAT8BbwwL8I/H4p5X+mlPrXgf9lKeWv14/5N4H/USnl//3Hfd6+t+XP/akboja4WrMWCrt2y4/e/IiN25AM7DY77vY37HfSbjSt0FGUUiLArW2r9/4af3k+n6t0Ql11iPIjS3W5ruKRVkrkD/O8sNkMEkeQL6luNSBefasRlNYYyTlWSVBldRvZ1K25AE1lDjrP8zUQq5TC09OjxKje3lXRd2VtKn3dRl80ak8vL6QkmrMCxJSr9EUWWW3b0PXtVd6koS4DNjy9HBmn89Um+fLyTK560b7v2VbqkHOOECWALITAy9ORp8cX+q7jh7/1JW/evKpP/fWqmTyfz3zzzTdXPV3b9lWiBON54uuvhIr923/qRzSN/GwAp/NZcoGaVlBcdV5VlMF6T1hXji+nyoesvMyc+fDhgWkaOdzcyBa8JKhUmfF05PHhgZvbW+7v7mXRsAYChXWZpeJeCyEszIuEt4VFfl8PTw9VIG1YQuB1ZQpI4JjjcLjBN62ME8JM27Zshg2bfrjOhj/7/DXb7QFjLTf7Hb6RQ75EGTmkFLA6I9GpilIuC7G2ZrtzpY/LP7lWW85JtfPVV1+RUuL17b3kSskNxnE6453n9vaWlBPv3r1Da81Pf/unHGvm0v3dPaWkaklc6bqOm9s7Pn74wB/8wR+itaJpO1EhxMjh5paX5xdCmK/s1qbtmKdJOrt1lXTG2kZ3XcfLywvWOnIuhDXy6eETWml2ux3Wmas2tW1bTkeBacjXk7HJsq5QRL/84eNb3rz5jOenJ9GTdj3TPFGyzJFTTnz69MCyLlgjB/S6RKz1bLYb7u9u2e12LMvC+w/vufvObiHzbZaONdJ9zuvKf+6f/Ev/binlH/97nVH/QFvvUsqTUuqvAf9kKeV/Vd+9KKX+98D/oP73V8APv/NhP6jv++M/by6MSyTajMPQaDnIYn3q+I3n5v5OWoOmkziIbsA3jkKqHs+Mq/AH8XCLwBlKrQK/9XzK1m+qEZ3mKueRvBAZKh9PRza7HdM8MfQD82khrDKnnGeZeYSwcp7OHA4bfNPinSfGcP0exInzwu3tzbV6a5tWhuFG8+79J3m/VdW25ip6Kl7FvZd2Z5rm6wbdNy2Hww5rBeaQS8JaTUyBXFsrXVMOjYXb24MEoKVE0zienp5Yy8ywEQKTbNkblNacp7Fua/dYU2qmTqzQDGmHlFZY5zkcburo4kTXdYQQeHg446xiGhfWZeHN529q9EWpFsuMd453795ye3OgH3pikplf4xuB6bpC1x5o2q5a6OQ6GPqOGEQJ0DaOm+1B1A/zglOKtrbE+71EbYQQwNoKxkgyv658UdEohivEZBgEGDuOMylEcpYFgG88XS/taDO0tEoUAtthw7Yf2O337LY7Qa6VQNtYVCqUFGi8I+hM53qMEpfKPEkuuXWeti42bEosdQx0sVvOs6gaUsq1tV64vb0lLBJRm1JCGcOXX37BPh34+c9/zh/+/Inddo+zDt84Pn38RK7dzTfffM3Lywtd35Nj4eHTMynDdrvjiy++wHnP49MLbT+gKLwcj1hreP3qC4bNXiRV80ROWSRDbcsvfvELmqbh9evXnM9nTqcTIQR+/OOf0DYDYOr8UjFPM85qDjc3HF+eJV+pJFKuI59RHuaH3Q0vL0dSSHx6/4Hf/73fY42R7f5APwwYW3kNSnP/5g0pp3rfxbq1t5zPp5q7JLZdraSavbu943A4EKLwLWMS8v3xdBY62Pe8/Um23q+AUA/JDvgngH/2MnesW+7/CvAf1g/514D/rlLqryBLnOfvm08CFK3IvqBVJANLqFgBl7DOsL078PrmtSwfGkG7+8bUQbTYy/b7fd2yRlKiwjLkSXw6nQhrwFvHZjuwLis5wyXMXQbRcrCI7EH8vWFZyCFV/7WQvFMMGF2I60wMC8aU6oO2jGHiw/sPvH7zhmHY4JtILopcBMZryGKvzFJ1vnnzmmmaBP+lFN7KkmmNiTALGkwhEpdLxrFE39Y8nLzinEHlTM72OuwPSQ5r4wwWew2sF3yV57PPPrtuK4/nM61vZQwxTRg0KazopmHoB8n/zonj8YWu6xGmsIwOFJau2/D0dLxWwuN4pm07PnySJUSi1EracIkE7vuen/zkxxKqZgwhxbqUksOsoETeMp6BRNsOrKcJrSWOY7PtmZeZG7dnnibG0wsAu93m+nDRWhOD5IA33pOdJRaJ3QvrQue3DK87hs0WpcVJk0piXWacluWhtkbkO6scYLZaQru2ZbPdsC4rRossrfENx/PI2w/vuL25pd9saIYdjWuwcZYIA2PZbvdCnjKaFJO4igCrDa1reTk+gxMeY6ryIWsdXSucyNTOnMeJYbPj4ekT7z9+IPx/2juTGNmqMo7/vjvWrVtz9VzNY04ITkDQQHRhWLEwrlhITCSGxJiwwMTEQExMXLoRNXGhiUYXximYaIgJQWANDuADJfAeSECG97pfd023pjscF+dU0yLYDA+qX73zSyp977kn3eer6vruPed83/+bpbhuQMkLWV3fZDTUdbn3ugPCsMTG1janTz1Hd5DgRzHbl57AMRtkflBidW2VZDyiVC5RrpSJ44hX/z0hT3Nefe0Mq6uKZqtJHEfUqjGjZEShFNVqlcFgwEsvvUSz2aTR0Lvoe3t7bKyvU6mERFHAMBlSDiP63S77e10cB0qRzzgZMtjf14pSecEgGbPdyShHJar1BoLihhs/yZndc0RxBddx2djYwHV0fagim/L6mVdRud7tHg/6rK1vEJUa9AZdHYOZz/AD/T9RiBbKGM/GFAheEPL62R0cR2g0G+/PUQKbwM/NOqUD/EYp9YCIPGKcqABPAl81/f+IDg06jQ4P+vI7+Bvga+FNlKLIC2bTnHilzuWdy9he61Cu6LuJY3bS8jxjNp0Z5WnFKEkYTca4rmvW0DwTFDslzXRaoHgu+70e49GIaq1GlhX0+wOzqJyZsCGfPNf5rnvndgmDED/waTTaOk1rqkOPxrMpYeCz1lwhSzNGoyGqgFqtiYjLYJigFAd1SxzHRYnDOM1A6SwWzw3w/RLDpI/vuOReTqWu6ys7fkDFKDEXCjMNShHHfHmzHNcTk59d4HlKL+SHIXmeHkopFBMErjdx5pUc5+MqR2Vm0xnzQk8q12ujga9DLCamxEOn0yGOK6TZ+GDnHxRZpguRpWlGHGspsDwvaJgCVFFcBuMsQYGjldwdM6VOkoQoKmuJLM8nDCJ0jeqCqFQG9GenKwP6DLMEHKFRadHv9wmDgKhUMjdIvfFQmPIZSoHveQz6epe7VCqxurp6sBlRq+p6NJPpVE8Xi5Q8n1LMdFYQCHGpTNQoU45j6rXagTSX4zr4TS0NliQJ/d4O+3td+v0B7dYWUdzCD2OKPCMrPGaTIUWe4VcqqCKnmJcrKHT8ZJbl+J5PKwwYDnvMUl0NsVSq6SDwPCUZpbz2ysvUmy2m3XOooiAIS7SaLbrdIeK5jKcTzpzbYWu7Q2ttnTAsMRz22Ox02OpcQpblnDl75iCLrRxHZLmpWuj4tFvr5NmUrU4HVSiS4ZDpdMJgqMWf8zQjLsfM0hmbG1vA61qPNC8YDhOazSZR5PDSyy8ijlCrNogrWqFpks54+bVX2NzYJHAiwnJAtRkSxzHDQcJo/DL7ewnr12wTBB55kVGOYuJqE8fTZR6KIqfZqNPd22eYJETlKoN+Dz+IGPT7PP/8C2x1tihHFQolbG+fIM0zhklCe2WVNEupup7Oefd92u02+/t6TfX/8U52vU8C179F+y1v018Bdx31ew/jOEJci/BcIM+ZjAtajTWuufQqttY2qNTqOFKYfNE3ChrNQxI8x6FQBZ6pgqgUeg0q1Vp3iNDv9djZ3dVisb6vZdSyFNd1UOiwGdcT5nWO55H8891NreTcRZmyq57jM52N6fW09qJSOUoJzVaTNM2pVrQidp7rao+j0YhytQIu+K6H6wW4uKYejeAoYbXdwgscxukMNwiMSEeG7/kH4R+lqGzSrRRFDoN+Qrkc4bq6qNM8vq1eDw9S9yqV+GDddDqdMR7rzauiUPqJKNdlOz3XR4hBFFme0qq19a5hkhyEZBUq1U5fKZCCsOQShi6+/8bUJc3GtFcabHW2meUFvqDX6MJAPzWmGa55io/jmNBkcriOC8oEMs9yfD9kMknMTq3LYDCgP+xrYYtSib2dXe34zdpwp9M5+PzmZXWn0ynVmtYv1QLDGasrbQbDAef2d3BEl2wQAb8U0ait4TseW9sdfaNRCs8sg/i+j6BjB11XJyz0un2949zv4wcBV155NX7kgydMJ0MCX/ACDy+oks+mFKKjLP15NhNK1+TxPVSh5fGiSpnZRNdSHwyG9HpdUuM4q5WIMPSZpVpHURDG4ynnds8yy1NOXHE5fhiSZhnd/TN0tjqks5kJ7ynY2dllfX2NdnuF6TRnPBmR5RlRKaVeb9JotphORuRFRD6bUonL9Hs96o0m00lKv9vFcTy2T3RQWc7KyhqDwUCnuzo6rbFSqdJo1Dl58ilUPuAjH/sog+GAU/96QX8n+z3CsEyaZrRbLQCKPKNaLXPlFVfTaNTpdrs6OSItcD2fMArodru02zGuK5RKHkURkSQTarUW+719MoRKrc4sK6jVY5rNpn4oKFeoNdr4vk8Q+IyShMlkZLLedObTUY7yXW3mfFCIyA6QALuLHsuHyArW3mXnYrP5Qrf3UqXU6ltdOBaOEkBE/vJ2O07LiLV3+bnYbF5me52ju1gsFsvFjXWUFovFcgTHyVH+eNED+JCx9i4/F5vNS2vvsVmjtFgsluPKcXqitFgslmPJwh2liNwqIs+KyGkRuWfR4zlfiMhPReSsiDx9qK0lIg+JyCnzs2naRUR+YN6DkyJyw+JG/t4QkUtE5FER+aeI/ENE7jbtS2mziJRE5HER+bux99um/XIReczY9WsRCUx7aM5Pm+uXLdSA94iIuCLyhNF0WHp75yzUUZpsnx+iNSyvBW4XkWsXOabzyM+AW9/Udg/wsFLqaozikmk/rOH5FbSG54VGBnxdKXUtcBNwl/ksl9XmuU7rJ4DrgFtF5CbgO8B9SqmrgH3gTtP/TmDftN9n+l2I3A08c+h82e3VzDXbFvECbgYePHR+L3DvIsd0nu27DHj60PmzwKY53gSeNcc/Am5/q34X6gv4PVoXYOltBsrA39DaBruAZ9oP/r+BB4GbzbFn+smix/4u7dxG3+xuAR5Apy8vrb2HX4ueer+dduWysq7eEAh5HVg3x0v1Pphp1vXAYyyxzWYa+iRwFngIeB7oKqUy0+WwTQf2mus9oP2hDvj98z3gG+gsTNDjX2Z7D1i0o7xoUfpWu3QhByJSAe4HvqaU6h++tmw2K6VypdR16CetTwHXLHZEHxwi8jngrFLqr4seyyJYtKN819qVFzhnRGQTwPw8a9qX4n0QXVPpfuAXSqnfmealthm0TivwKHrq2RCRudnZHvgAAAE8SURBVNjMYZsO7DXX62jx6wuFTwOfF5EXgV+hp9/fZ3nt/S8W7Sj/DFxtds4C4AtoPctl5Q/AHeb4DvQ63rz9S2Yn+CbegYbnccPokv4EeEYp9d1Dl5bSZhFZFa34j7yh0/oM2mHeZrq92d75+3Ab8Ih5wr4gUErdq5TaVkpdhv6ePqKU+iJLau//sOhFUrR25XPo9Z1vLno859GuX6LrBKXotZs70Ws0DwOngD8BLdNX0Lv/zwNPATcuevzvwd7PoKfVJ9H6pE+az3YpbQY+Djxh7H0a+JZpvwJ4HK3H+lsgNO0lc37aXL9i0Ta8D9s/CzxwsdirlLKZORaLxXIUi556WywWy7HHOkqLxWI5AusoLRaL5Qiso7RYLJYjsI7SYrFYjsA6SovFYjkC6ygtFovlCKyjtFgsliP4D/7p8wqDRSLMAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "img_path = 'inputs/cat1.jpeg'\n", + "#img_path = 'inputs/catdog.png'\n", + "crop_size = 480\n", + "padding = [0.0] * 3\n", + "image = Image.open(img_path)\n", + "image = np.array(image)\n", + "transform = transforms.Compose(\n", + " [\n", + " transforms.ToTensor(),\n", + " transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]),\n", + " ]\n", + ")\n", + "image = transform(image).unsqueeze(0)\n", + "img = image[0].permute(1,2,0)\n", + "img = img * 0.5 + 0.5\n", + "plt.imshow(img)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "dd8e70b1", + "metadata": {}, + "outputs": [], + "source": [ + "args.label_src = 'plant,grass,cat,stone,other'" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "6682f086", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "** Input label value: plant,grass,cat,stone,other **\n", + "** MultiEvalModule parallel_forward phase: ['plant', 'grass', 'cat', 'stone', 'other'] **\n", + "** MultiEvalModule forward phase: ['plant', 'grass', 'cat', 'stone', 'other'] **\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/rranftl/anaconda3/envs/lseg_releases/lib/python3.9/site-packages/torch/nn/functional.py:3609: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "labels = []\n", + "print('** Input label value: {} **'.format(args.label_src))\n", + "lines = args.label_src.split(',')\n", + "for line in lines:\n", + " label = line\n", + " labels.append(label)\n", + "\n", + "with torch.no_grad():\n", + " outputs = evaluator.parallel_forward(image, labels) #evaluator.forward(image, labels) #parallel_forward\n", + " #outputs = model(image,labels)\n", + " predicts = [\n", + " torch.max(output, 1)[1].cpu().numpy() \n", + " for output in outputs\n", + " ]\n", + " \n", + "predict = predicts[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "85e8cc6c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATAAAADnCAYAAACZtwrQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAEAAElEQVR4nOz9269kW5beh/3GmHOutSJi752Z51Kn7t3NZku8iuJFVNPUhbZoC6YeBEmAnyxAj/6D/OAHwS8GDBgQIAOSAMOALcOCLVkmdSUpUk129b2qTp2TJ3NfImKtOecYfhhzRZ4mjCZ0yoBBoKKQVZW59469Yq05xxzjG9/3DXF3fvH6xesXr1+8/kl86f+/L+AXr1+8fvH6xeubvn4RwH7x+sXrF69/Yl+/CGC/eP3i9YvXP7GvXwSwX7x+8frF65/Y1y8C2C9ev3j94vVP7Cv/UV/87l/5C+5mJBFQARFEhL1z6TgGSFJkUlwdSw5JSZrw5IiAqgAg+/sALiDmuIG2ivUKZhQDsY57Q6yjOIKj7pgKLoA5DjgggDoIBt7jdwCq8UVVwczovQNGr8a2VbYG3QwzQ70BSkoJ1fiZlDIpKbkUSinkopTxJ5VMyfEn58I0FXLKJE2oJpIKvTXW68rlfOX9y5nLurKunet5xd3JRdAMngwRMBxXBzXMG1NJHJdCzpBUcQczo3XYmnCtL3Ra3B1pJBJKBhPcQCSBFZIIqo4mA00UyagIGUVIWHda6/Ru1K1Ta2e9Gr3F78OFMiWmRUmzkGZFC6g4SQXrAp5xE3oFENTAY2Fg45n8oXXjjrVYG26CWfwuccFdkOSggjm4W3wvkFwx75h7PK+szMdEOSUOh4nDw8z9/YFpVlLKCBmzCm7xc4Bophxm9KCoCpIVUVivVx7fP/L01YXHt1fe/+yJ/mwkS6gIDaPSkZ7js7nSx/2RLnhvqBtFBcHi8wKSBHBEgSykJKQU6xcEcaXjoAlBkZJIWUiLMB0S06EwHYRlyUyzkqeEq457Bb0Z27Wxnp3tbFyfGtfHyuXdlXrt0IHuaFLAMRyTsXuTgICI43RUnePrO+4+vePhoxN3pzn2uDndeuwhh22rvPvxC9tXLZ5tclRiP5oY3gx3xxFSUg6LsixGTk4WoaghamhJkEDVMWu0CrUJ163ROpAPzPcn/vP/3X8n3yiA7UHHcfBYeDoeTEQJQUWQFNFCNAJT3JR4iEj8HOMSZPyf5BIL0QzcwQ1xMO9oLLX9AiII+giWGILhEj+mMr4zLhF3H5slvrb/XTU2RyqZZE7qY5e5454Ao/UN6XGVKXVSymxbJ5fGNCVqSeQslFJoxWjFmCfHzJiKxwPKsbitx0Ps1mOhdaO3jpnhbuAJdwEf93d8EIex+YRmY4OPYH0LYNUwE0Q0gpuAkrAu4z0d6wANI5HoqAiTAGK4Q3Ogd8yEVp26Nepm8acyfh5S9hHUAfEI7oxF746ZxIFiI5l3wXAQB/rteX94Lh6Pexw0Pv6jCTRprK0smBs6vj4uG+8NaaCW8O5071gX+qpUNfRSKTkBE9NslNIpuWDWmfJCOk5M00yZC8e7hbIUDGfdVq7XK9MyU/J7Uk5U2zinFTane2zi1BJIoteOjYN1X4iCxEYfT0tVUVVSUkQMjR9F8yh7XEE0AoDF/XCPwIQ7Mr6GRLDTJKhCGuvYu2HmWDVoYNVptWPd4veWjHfFMDqGRf6BIHHdEuFMhLg2iUO7LIkyF9KUyVNBVWitk0zpPWHduK6VVhs2nq+MTMLVcTF0zmPPx29r7lQH1UxPiqSxxzUj4ph0XBPkWONaEiVnmJxU7I+MT39kAJMRJCI4+B6ZYlk56O3vRPCCCHAi8ZTER0a0L2LGw96/N7IdcaF3xa2NyO2I72eUozgusWBEPE5oA5db1PoQ8G6bBZwIsD4WxP61lDIlj+uo8X3Gyjgzx8bcc7wIrm5G650yZdw0AgWCoGMTN5gUvGPiEUSMWGQWC6vVRu+GqsT/7glpGvdtZLkp67j+yLz6OG3NEmaGdeK0lrF43InLlRGQHafhnuIKJbKoXJwkjnenrY43oTej1UTdnO3aadXpXcbpmVB1RD2yOJHxWPdNpyMQKz4+qxCB2Wnx7ORDAPv6s9kzcVFICKoKKkiOVWk44uCS4mNZpzejN0G64d2xBtetIpfO5XElv0883608vLnjdLewHDrLYWE5Lix3C/P9gbv7UwSqKZOnTMe5rlfKZYKsdDe6dK52pRwL7Wpcn1fs3HHXWNMIXm2suA9BJoJVBHkkgo2IkTLkIrF21eLQHxFlX7qtx/HsHWQa91mFlDNTEXJWJI1MSeJE6b3hrdM2p26d3sbh4PaHsl6zHvd3vKdIBLCU4u+qQlIhHyamQ2E+TMxLYZoLAkxzwc3o3di2xnqtlLngNY6nOHxGVSSj+sJxHQewO20kP+SEpgTUWOsqSIprEFMQyOYYik8d+XkCmGuKjdR7RFMfpYGO0wHHvSOj0NuzJRfHR/Tdcy73OJXcIiygDcFQjJSFOWW8OrU2em2IS2zwEZqMyOqQcVqxv9Ee7CLA+VhT++kG3B6mGWNB7T+bMU/0to2UXBFJsdAkEVsrsicXocsoXzzywL0QwJUsQnfHk5FTnJJ1g9YlspytxallNt47FlBKAkXRLHstjIuTI5WLg8ENd40TvhsqjpMjsI5sTlWxcXNFNDYbHVFjmmfKsVFKjiy3GplEa9AqrOtG24y2EcHLYxOKQMpRMkoai80FPJ53b4J7ik2IIqLYSLZVcpQfHpnoh2MsXiULqEb2pfH+ooJphxxrR8SQ5Jg4vUXa6HiURB2kgl2BK7TWaM+wnq9sl5WX1ydO9zOvP4bD6wOH1wfuHl6xnA4spwmdItPtZsgEJKPZyrYVLuvM8eGA5o1t68zHzPrYOD9XtucN35TUE+Zx9IGTStybCMY+7p1F8C8p7h8RoJMImAIaC7t1fJTGmiaypD1PIqky5SgvoUdJLYaooklJUyG1htOorWNN8O74KB1xi4CFRbIxDknNPoKZRkAtSjnN3L95zcdvXnP3sJDndNtn5katnZRn1vPKfIyqRd1uAQvVuDaRcW8YBz10NzaPgwhREqBquCh5XAcqpENCB0RAzoj+0UXiH/3VEZDEEx6ABOiHBU7EidvKdIlMZz+BbmXDCCpfz4Qgsg6VOBGyClOZgcy2brRLpfV+SwETsUEjbY8MyG9lZQRJ20tVl1sGFQe7juRRUR2VY4LkjUk6mxh0wUdZJiNzjIyPWyocAdvprSEqtJYQ38AERZHcUUv0prgHttQsNnDKmTJ1vBqlKMtcyDPkLHiJU7xLj+xzHBCu4DJK3R4LTlMEEPMP1xs3n7HIfdybCJDHU2K5a+SDI9LwFhhYb2Ai9OZsa6dtcfr7CPhR/kRQ1XE9wp5l7eVgBHz3uG5uJco4cnxfRvsijp/LJSE7njmydUkCbmgSfD+RJZ6TKqQZelcwQyyyL9uMlBTPsFWgxSl+fm5c2ntWO5JPwoM/QEnoIaGLkpeMTrH0szu1tRvMkCZlOirTBl3Ai2HZ0VIod5nzV4mXn12xddTYdDQpmuK4EzeSCipRRmeNgzi5jwPAb2UbFuWpJkFFI6vO4OZ4N8QkSkLPiCvKwNdGQPHxnjkPfDbDJn0shj725j8CH0lAPZo0Mh+Uook8F5bDgWU+MM8L0zwzL1Hvuju1VlQbzRtlTiyHaRxmjIAVycX+K7vt4ULj0DbHXdlqo3fIWckIqQkmMvZpjs+TAldz/ZCEfKMAllIsGHTPorgFgh3fAh24ht7KPvNI/+EPYx8iestb1CEBSSFLIomjEhnDcc5YguvmkY2ZBTC9g29uo8Tihs3xtWflLqOkAcl7/T+CGDoCbWeaNW5eL7Q2B/hoHgFvb1H4vqETqN3K39777WSLYCngE8VB1MZ1xCIvJZPSicNpobUaNz4rZVZyUToNU6MZdHpgCVjAJLHOY3F43Gt2ENwHojHOFrMezyoZKXWWk3J6bUzHhicj2YEmnepCpXOpwmV12uZ4j3JVRhND0sgQ9WuliO+LSW7PIrbAh+DE1wJ9PBO9/V1zIueEpnFS3zaV3+63IuP5Gp4czYZnx2hIgeIDn6tR2po4qFGmhDeFK3g3nI6IYQksBRpoYhjG5o1ZCtaNXmtk/a3RvVOt0XUlL0bCSNmhODoZOgtWEiaFKw3OjpJQlJSMpI6YoVJIyRGUJERQGzjTDqHIKPWRsbb3ZEEYB3VsuK1WLivMmnGVwKt2jNgjSGqCnJRcBJ8cqYplpa6xaWRH2Pfy0kFRVJQ0nklZCvO0MJVCyokyTZSSxoEJOWdqrVz7lalEiSkIrQ3M77YP9QYDOdCtBdSjFrifZTptwB0KHnfQJWESAU8y5JJAFP/H8CT+aAzMbCCOcgPEPmRTe3nGbSPvQCbjQ+8RJSJxwoHWO0mVrEoSJ6uTBUrSccI3xA3EKFlQybTaIisCrPsowz4sBtgP+xFo+7h9orTNSXmc7CMCiyjTouhklMNEKRrg+NrY1kZrHXHF+qjhuwXm41/LDEY5Z753YK6RUSCkKUoGScJxnoASW32k/e4O6vReaaMEr7aianFyiWHieBpr1ASy4L3jOTIsusYmxfAkiHlgLZNCSuiUme4b+a4znzLJDKtGIdFbQsRo1iOLIzZSQAMDUM8gWW4HlY3MTp04Tfd1sS/VlKNrSGwSVRmRV3HvpKRoTvHe6gNPYgQ3PuCKYqjUeOfY/ZhXXH00ixzNgqdxkKrQFyFVp106oNCN5Thx9+rAclpwgdYr63Ula8a7s1063ZzrZaVeXriuT1wuF9brE729kLIxTY6rUSbBJqc2I+WNZXNSV9bc6Vcj++heMw4672jKJDcgurDeRyNER4YlHoelxWc3dwxD1Ug58Lk9u+7qaHFSHqWl9VtTBxPa2sE1gHFxjBoJw94RvW3h0aFG8Doy6UnQuZCmgiYfla0gKZHmCSW6w4WMiNC6Mx8WltZwznBdxwGqgA1IoQecYAPXY+C5kUEgJNQSqnJrDuWUSHmHVZSc88jQ/+gI9o/NwMx9RFD9sNj5EJ8k8HT2aLYf0oLGg9IApM3AiHLImtGTkHMAe7eUmz1zif/qvdNao1tH3EmqlJLx7LTWcIssZMel9gQN/YCTuQnWdqA4FpFmZ1oyeS7Mp8zxFN0iF7icO+dzi7JqNXp3ejXa2qOMFg0sEB/tdOg0qgild1rqJAlKRUqZPAk5JUTSKMvSAFSdy3bmWi9Ya6jH4oj7bKCBIwbAymh9C9I9MDD1eHqqJFFcOtNUmJYJtPPZx3fk8g4OUA6wSOKL9++Z5zckO3B9/y5u9+gW7U9VFVKOk15ltN/F9vbGwLsGHiqCaSxOIYFFO1xUR+YKENlHGnSFgE4bmqZY2ObjEmKZWzeyZLx2zDsuHSkKyYOWk30gkx5dsClBF7p2iittQAblpKRTYDsuTq3GeqnQz4gqtTaua+N6vrBdHjEqrV/psg4sspOLIyVhbvTs+OYkd/w+Mo4uBk9GWvvoLMZmEN1R234L0DCgFxG69cg1O4jrCADRb+1dEMmkeWY6JtIiLHOiFCWn6AIiincZGWRnXRuXi7GuTt1qJBF79173QMINFokEYGSya8VKHgdDiuZWioCSUyYlxnU1EplSo9M7TxM2R3XULbrrQhpNqw/VR9yW+J1usSc1gY59a+aIZDRl5jmRi5Jzie4kjWY/D4ifCLDRPpRovi9MZ8Dm8XBgr8v1VtaAjNPmA04VMSzazV2ExuATeXStvAefy3rwTqx13HpgVwPbm0qhlERtjW2ttzL1dh3sXUSIQnVsDIgsZVbyDMe7zHJXWI6ZZSnkAusKl0vn5aVxedrYavBsRBvbZniL36GRM+MWgKoP4LIPYF0QNCVyzkxlGoDpwKy8j86QR2vdQXrcJ3OjyUYA9AkZjRSRyGxTEryN+juPLEZkYBgFnZSclfvTzLdf/Qpv3rziJy+/g4linnn3/jwKtYTUiniAmDknQEglTsEI9un2rEUFJEXaH2nUKMtlcJ3GoeYR3GQvG3VgPAPzcrfRgfx62W/YALOtV7oIJPDsaNHA/4rh2ZBsiEKeBS+jadDaAKednAzJiXJXYIr71K2zriuCcLms1LVyfb5yebmyXs/gG2ly0BVLFdONLi24jeNeSFYmVXpcGj1B0wGbmEETkmp0GZHABcVRH0nAOABGwkFrDiYkEboZfdBlNCtJg194OBXmU2EuOXh8exOpRzajmgM39ejQ0uO9VZTuTrd6C44QgVNSbCQzpzUjiVK3yuQLmqJklNF86TskoRLJB1GyRpYczaqW4vrla6WwD35lq5U+KpXeO5hiFo0ry53sgfc6iiYj52lwKiOLMwK2+cYBzEbLWCVF5sSHSO6jgSK3bAfc9nImHoyPTAWi5kcJwNsEt0ozYlOSx+kcp5V1x/rgTJnFye6Q0g7CQZkmpmVmOTSu5411W+n9A64SQS1HZiaGWY8HmROlzKTZmRbnMMO8ZI7HzLQkDnfC4WrMy8qTKueXjlLArxg18DSEXKLk9RanixFgcE45+GSq0AvZ4gRPA+AweixEjUDnA+zt1rBuVK80VkwbWQrmGdUSAO4oKy2PklwCb0tJmO8mcoHTpJwOE8eDcjzM/PK3fplPrm/4vZ/9Dg/3J/rTj3lHoq6gzEALEFrimWlSVA3NOtrdwXiCNEqA4C85gwg6CJjdomM6Eizw0a5H4hl8AFBvP+sOvTdarRGkemzq6GiBF6J8z7Hx05YiOMwDl0qJZGCpk8h0oBRI00SZhZTBfaP1zPXqrFulrcb6snF9uVJfLrRtw1Mjz408AanCtNFTvzV9Jpkpc2brFyZtCMHTy93hYlAFv0awEjestQFjpPhe8ZFtAGaRfXSndwarUWkuyBTkVZ0HBIGQSUyao5xPRnFj21aQNmg9SpuUKStsFrzVG/QT+KzvYPHIAn1HhATQ+IzV2si047kHYaMjQK+xf3rv1N649pVuG1BxGhCdccNG99OQHllps3gPN4uDtyu9GjIrMmVUMjlF0EqjmggqVgqI4GuZ3P/gADYagAOcl+iO7UCgKGIOFmWH4yMLIQKOjODlN0pYnEojzXaiRHQBaZ2eQaRFttcb1lvwXHqPh76TZQEkFtdcCnpYOCwTL5eJy3Vl3Vq0and+1t4dSwoN2tbZ1srxbkFSJk2JaSlMh8LhNCMpc2+ZspxxP9PlQnNjIlGmNMrVEUAGcN5bpzWjdWOtFbTTXWluo8PWcY2FEeVFxxxqa9S2cd3OrHVj7SudiqdOStyCS+DgUXZKUtKkwcERcDWkxL9NQw3xw8++w7fmE1ISclH+xp/9X7DWL/hbf+dvUh5P/PbLF1zUaanRCniXaGykCCzxO+M6NUXZbDfiZRA0y1QCV2T0edhb5yMARY1JlFCGuX7oXHswyM2cvjnS08jabP+Ykal1C2TVuXHEAhdxkgQUgEbg8Dk6kkkyUuKeSInnc71c2Vaj1hcu58bleWN7XKF1xDuaKtOdxGaWOGRFLRpNMlHyxLQUklfqFnCGI+TqcAdUY3PDroo2wXrgnIoPJUKUco7RWjRjrEdJ1QU6hktiOR4oh0wp04BgQkFQOyQZDQEF0VAa9B5YmOZCPiR6b8yesA2a1dF42VnxUa6qg0vHswRWmiUw62RsfaP2lWud0A2aJqapAHHQ7IoW7xZdR/bmQ2DeNiqSvanlQMpRVmrXW4syCAIdPBoI0xRql73b7T5I8bQPjZ5vEsDogUu5j5Teub2hd0PSDtBF+Sb+obaNjv4os25p7Eiv8dGBiJLrWitVbDDwO72ueN8wD2wojzb1jikIU9Avxmab5ol5mTmvB9Z147KubFunb0bdH57spErhct443E3ceXTbpknJS3Bq5unENB05HF6h8jbiabeQcuQZJwJPa4Gr9FppTfFro/bK1leoikmOrkp1dOp4KuQUtBBzp5lzXl+4bBfO68rWN5rVKJHQ0W4PLtoHPCNHMBlnuwTZKJoDCIsm7u4y1M7r0xv+mT/+l/hoes1n80ccP/5Vfvmzv8z3/8v/B//ef/jvMn3yGdvzlxQvXOtG6x0fRFdNHxo0qtEh0mDbxv1eCmXKI3MkDqQWp2wcfL63U25lBQO4lkHDsT7W0J65ebD6nR0zicCKpshWJWRqWXYqiY1SZgDDDp4hiSOZyCLU6DTaGjyx62Xl8tR4eXehX4yiylyie2fV4veVwPRUJpa8cDi84u50IBWhMfMij4hVtrqRJoOD4GdBN7DNRy/VB7YXny14UhYNh+p0F7DY8NYbjtME0lpZ7sqgRsTG37YWmOSAH3rvEfhHy1hTRrSSS8Imgi9XdOCQe5rFkJTFM5MkSA5ZUp4zWnRUUuPa3ai9BmF4i+ThAyk2Egsbn00kAqqb3bDreL42uuSBkOaccTrexxEnnWYNTcKyzEyTMpdMyomkidY3DKX3n4eJv8t8biTQ/XSVGz9qB4B9ZCY+FiTukaG5x0nqzq5VFB0sayR4RL1Hp7HXoBS0ivcap7BDTlHCpBRdnpTTAMMDI0KUlODuGEBguWZaM2rduLZO3eKmM1Jib/D8uHK6y5xaSE1ECillDuVIyjPlqHz6cQabyek9rbX4nZJorVNbZV03Xp4FPzdUO84ohxgPK0gRXBHmlsmzIDkWcK/G2la2VqnWaNbiEJDQ+JGEJIFJpMETUpORkcSJl/Y0GxAT5jwzSyJNM/P8iu8s3+FVuuftb/0Bn19/g8PDa/78n/7rfO8Hv85/9p/8+0z1/8xvfnnmpR95ul5Y+wsqQc6UUSqS9u5gZGhlTsyHTM4Jd4sSrhOFRJdROsreYIySUGMt2I5xyq03hqfQN94aQObjuUeDJjruimWDEr+Pr3kQyOicyjgk0yAhp9EFW23Dtk69wPPTSnus8NyChKuhtUVBa+g/xYIMN+U73tx/zKvXr5mXiTxlPMHL/Mjb9DPq9ki/XJEsXHMnSmxoo+HBkAWJBqUjJSHNBSs9GkJdsS2eZXfBXFjPjcu0kafCnKagZYzqJrBQHY2NAax7woY8LcBAC7yuSPCTNEVg82Dvu4baI5eg76ShrdRZKVO+YZmB7YY0yc3RHnpiM+O6XahW6YOTaUN5UHd1iAoqPZADN6TLaASGLtKGugRNlJKYSmGaJo7HKbC+ATolnREzrG3fPIBh+8MILZaPtigj+9lTslFgjp8ZJ7GMzsrAsdSiRCCB3rgw42axM+xHZjlY8a12ZATgqzfU02C5D1mLBnAa5W2AqaLOPGVS8uCY5UYvSq1C3Tqtd6w767Xy9u0LeUqUOTMfFF0mkELSgqTM4aB89EaZ5wPPl+cbt6n3TqsFvQpbrwHuY1jvtFpZaydNK7mDbIZunXIwsiVSGeKoarQW6XQXi9IwJTR1UubGRt/1dDqyIWoQUANs1fGYGs1DrvTJ4TPSVfjB6Zd4/3tf8ntvf4Pndz/Dtw3PiePf+X/za//MX+Pf+Lf+V7z55FP+D//R/56nszHNyk/fn4MGgN0WvOkOxifypCHLWSY0SXzeFvKuvWmyax1t5xuJBv1DPRL6+K4gq8pOKYnP4YE/DKB4lCUWZRDdb2JvH+VKGhmGqMbaGtpcdIDKo5lkfSg8rhttrUFpsIL3Tk+OZqU3oa7AJBxOC6fDKx6OH/NwesP9wz15KrgYSz4BibYm+vol162hd2Brx7eEbFH+oE7WhNDJWVjuZmRJiERJdnkx5LFTrw2rRFMkKdU7tW10y0w5D4lskLh732ViO8u9082iUz/2jQIFoWehHPKg+gCDNK6qaImKQzNoFnQaxgtDOdCsQmt0j67/LkPawfiQs1kkHr2HsDYANUQgF6WNA9x6izJWBBtqEkGZpwPLIUD7qRSW5ciS8y1mtNaoVvlQ832TADYcA3Yg1xkn3gDmHQIvEMHToHF/CGWjhDTEjOTx4dKtFB5ArUMasoEAFyUKxcGTCR2h0y7G1S9ojjZ/KYnkinoOXhEeHYtuX7u6kFEE0z/BnJDW6MN94fLcePf2yrwU7o8J7iZ6E3qKlvXhGCCsloLOsK7XkSaX6Bxh5HKOgCMh3N6a01ulDHxKRSi5I1MnpYYmcM/07CTNSM+IhcQqSuJwoNBplOWj9HUfUhAZtJbY8ZGJpQnrG71X3hwe+P78XU5b5vd+6+9T+0bWzOlwj3tn+6rz3/7f/iP+xNM7/vq/8r/EDf63/96/y+EwMV0LlhSVdON/uUfhLiqUSTgcMjkLiJEURJRaLTBND3Gv3h5/HHC3uKTRaYv3Ff6/Fwc6MLBo5CBRwrp1cpMg8HYfLHXQG/C8NyB295GNZopVpW/QrsHZ6htREVgDd9oaJV5NjhYjVSEx8XD6mPv7j3j1+iNOxxPzvNB8I7BVY7t0trWBXVCvSL1y3QbEURVpgpqhKTEdEof7iXTKQT8QZ65Ouwjnpysvzy1yAc1MS4bUBw0htmftK70q2w0Di6CtmqJZNiR2XaKJBh4qiiWhXXDyrSHhSUhZyFPI11yBJGjO45kZrVeo0f29sQ7w4U4SFAtRp3cfh8yosCD4fAY+SIyqQvaQbLlZkH5zJpfMcZ45zDMlT5SUmedpHDjR/czeb1rabxbAvEUGJBHCwv9lBAf5oMAne3RfBr7lBAAawtbIvEIzxa0kgp3xDmodXOiW8B5k11ZjK1Qfm6Ma16vgsn7IPDosk6IlKAuMDE7M8EG/sN5HIJTR1fRRpgWucn5ZeXlaeHmoXNdOKU6jo0umpInT6YCkHJ9Ribpdoh18uV5JKUi1vXe6d7rBNCemBSQZ81I4nIT5lGDyYIebkorQa6Jriu6bRzCQkilTdNA0BT8H350tCIY8gEc3t3ssnjs9UizjT51/6oe/wlc/+inb5cw0Lagn1q1yXA6oQmvKj/7u3+JwuOOv/yv/Nu+e3vN/+n/++/y0FiZxuobMy3rHepSoOQnHY6YUJ5VxNnXIOQVXzqONbs5wLJHglEZkHj8wmj0jbY8AN6KbQ4jCY104bThWDFudJlCBTejFkCnhadcGjvfr0bEKkDiFAqsbrRq9NawPMq44Qo/StAltCyZ/b+FuMenM8XjPw+vXfPTxJ9zf3UUzoCc6ndYrd6cLz5cVPHOXO/fpgXf+yIuc6Y/D0CB1ymFmfijkU2I6zuQ5kXLmQKJuxuFV4W5z1ssawdWDYN290i0hJvS+BWcrmC4B6KdgwieEzYczx8h0yAo5smhy/IzqB8xYsw7aTGTCLopKBLmhkwkgHT5wxiyqi14bXht6KwsZ0rQez8qimWU2dK5pHMSmdN/oRNc8T8phmbg/nbg/HiPb1DyCr44OcmVbb2n7Nwhgve7hNzyPPPAJT/pBKiSB4oqNRa/jQw9aRBIjpejIBNXIyDvDe+in0siZzAJHKZap2egJkjptg0aKDmJvPLUrrRrt3qlzp8xhv7FLfswivfXWwfpN3Nq70Xd+mNoofRrPz1fev3vhsDyyTKdBFAUkMZUEyzxoD43z9QyqWDW6R5pbfQsbEYXlJNwdJ0iVskwsx8x88uAZFR9duxSiaVWsC7sOb0/vU8pMpQQXKyV28mHrPQTlLZQFYgkXpVsliXM/P/Ddh+/RvrpQz09M0wJ05nmm9UY3o5RMKQfWywu//7f/Cw539/yb/9q/zfunn/A7/8V/RNeMEcRZ64I3AwvaSp6UnEO6j0jwtSR0cK1VdguVtmOn8MHSSEeXcqgRrO+8ImKB7/CL7A2fyDTcnD3n7zUcGaREVpVUsbShbjf9KMQBZhZeWN59ZHGGZEFL4LmjLYcpNG9YT7AJ1hOqM1lnDss9d6cH7u9P1L6hNd34TfO0cFiW0YI1FoRJFt7pI1+1J+q7jSygRSlLYj4WyqGQijIdDmjKGM5SQyGwXmC7NqwJkNHsuDZskJ53x4tgwzd67yRPcZBJ9PxkQDGSnTQpdA2sMfkNakk5B0VmZPU3PeoA31urSHW6x8Gqoxw3i0Ogbw1xR1vc491pBYnKKBGUIAbW5QRPMEUaHxWROssycX934u7uxOEwMc9lPENwb7QeIu8p/zwZWN/bwCGwDeuYOEP3jl6sOh+nZYc+Olk4KcGUh6tECh6XDleDNE5NSTI0fhYfuiu9RXnVNTqQNXdWrWhS+tpZN8MeV1ozpmljmjNlygF2Z7l1uiJgWZSgPTCEZm24H6TIItW5XK68ff+E5sTp8EB5NbF6iJ/xBkVCxzdN+Hamt5Vtq9R6iS6SNzQ5dw8T81QQjDwl5qMzHSBNBKuYaASIBnAbzajAblwVciKlQkl683RKqpQcKU+3itWNutYR+6NFnzwxKUxT4/7ugevlGjo3KUzzgooza46DZmRz93cPWJn4/B/+TT6a4G/8jX+H/+r3/it+8+UrNgzp0AcbOiOURcKVQncZVyy21jt168HxGfhXBJ4hPfaAITohhbk7nbi7v6O2ytPTM9frOXDSYSa3EzF2TzcYMIYryVLgcQbSeuClth+sI2PoFrYs5gNO8yjH1IPAnDJWMvW80bZwMrHqrGtnnpTDckcpE/PxwHI4RrAFyjxRrQ2WeArSZZlgBlfoani/cP/RCenwYmfsug1zToccJdt8mqAo82FBFMxmynkiL5Vp2+g1PL6E4GEaHU1GyXn0Nwl6kA+liQDFSVLI04He4nm0aXR6h4g4pfQ1svduWpCHCHtw8q6dzWOvpBJVQkp243H12tjWDfceDb5u9LYTtyM5Gec+IaFKN+NF15FZuVNK4uH+yOnuyOF4YF7KTeAfHWWhX+veI/zmAWxf7IF1xUXFagzyRGCoEse12PAWCvA2Y6jvLVQnlU7ORP2cwhRPI1sNAW8URtCdWkML1UuibjXayBn6ZvSSWLcoC+yp0opRp47oGjymEt2OnXPkg7y3lxCI4UVDmSUOhEfX+XzmCzWWaWIuM5MZrV+RAnlOSA5A1FtjvV7Y1kqtWyTcCvOinJZ7StYAbmcjz51cDNPAMZLsFi4MqRA3FwYf2F9m1/wFeJtSiQ7U3o1VYS4pnEYlDpeFRJbEp8c3TDjXlxcmmeK9JXzWUtaQXQuklFi3lWNJ0A988d/91/zg1/86/+b/7N/hf/Mf/K95ryvWhGShOSwpMZfEnFNwtTxwqm5GrUarHpnOoE5ETI4SJf4pYIPT3YHXrx94/fBAyZnL9cr79+94++4dl8s2RPc7whpUAPN+K/vxIEl77ZEtZcdyAakB7qc94AlIwBLRCHHmYwkbHZvZVqEdN85PK+tLo1WhujNloSyF4/HEMkeA6RYibwYAva4r58sL3RrLlMh6oFXoKXSmRTJy7fgZLt5xq/RrolfI9xnNE8vdgWmabphgWWbKdWW9TqzXDau7EmUEKG0hsicFSXV/DXB+dwjGMlaN46IDI8s3Anfv/cYLa7XTWjRI4t6OPdiFvgUpu1snuYYo3QkwvhltC/2tWL8x7sFvbH0jqrIIYtHN9gEjWDfmaeJwPHC8W5iPC2VOcVCXhKTR8OkdN8LNJWx+v1kA8yGaDvLp6CKNzpHdQmO+0U3UO5gP47UA7EQ6KXt0YvKO7dgHUiMMP4Ox+DujDBS27tG1TEEtsGR4MfK6sa3Q+/CrSkLrnW3d6A1QCSF4EkQz7sEKtqExTJ7RNAKYRKlr28b17Pze5z/h4eM3fCs71254U0rLlKXQ1gt1W7leLtRqN9pASjDdFZYlMedMTiA5jAlThtV6NCZG9brzq3TIcW7gsNdwr7RMGfYqEKxtGSVqKTkaAVQg6AJLUiYmvvvR9zjWmVqvWOvM83FoKxkq/0K3RmsNFeF8uaCaOB2OvP2N/5J/6c/9T/lbf/c/4f/6D/5TPM1I2siaWErhMAU/SVDwRA8MnNacVu0D18//kSNTg6t0WGZevTpxd38KXHAq3J8W7u8OzIeZP/jJTzifr/wh1v7oVo/GdwS2bsE52+LeSgpgsEs831x0bNoIhpoj8E554X4+seR7lJn1Wnn66oUvv3jk3eOZ5ajMD5lyLLx69YplOYILrXZqrdTmnM8Xnp+feXl5Yl2fwCslz7EfmpDniXyXSR8pvr1A32iXjm2Nfmm02pl9BBa4uUFMi5PnxLRU8iWzXiutbtE59igp2ffhEEC7xdrPuoTcr0cCQZ4CG5NpaKA89qHKwG6dbVu5XjdezmvI+obhoUoK+VEb+1eUYlO4e/QgbLOFm2q+aftsrOXg5Zn7jbzqQ/scBgidU55YlplXr+44HDNlCfOBPGVyznFgAd3D2LSNIP6NA5jKEH4yDPJUQlzrISHx4T91i2U3DlAjqaN0dqJvTsGczsnDTG1Y8ojskHwsWlWhFGgtygRJSs+QGlhWWm0UhTxnyrQMBm+kxNdr5XJeeXm+cnnZyCSQGiCmpDiVNUDk4I/FhadstGb4auiU+OrxS16d5nhfT3FTu2Ft5Xo5c10DgzOzKEnmzFIyyxzi15JTuHOmsISRTcFyAMfutA59C0CbHL7rvRnWApTfJNrHuSbEUwClKVokmpRShJQX3CsJ5/505BP5lO/Mf4z2WMNxYTiCeOsowtYqp9MD18cnihrJQXJm3TpSrsyP72h/8A/51//Ff4O//aP/lp9YA4juUBaSBi0F36kfTm2wtRZSngLS9NbIsR7iXryjxbh/tfD64Z7DHNq+aSpMU2hQuxlPz8+s10atRjhFRbZMCqPIOOUdJ4fZVND6RiM2urW5gAyzv2jXwySFUg7c3b3m4XTHsdxR9ICSWD/d+PKjr/ji8S3X7YXj64WPPv6E46vXLMcpsJ/WqetKbY2n9y+8e/uet1+95by+o6TMNEVmmJOMS3IkT9zdGbae+Gqr2NZ5eX9BDgWZL/gkzIdjiOXpSMrRrdPEhMZhZtFVd+uRAZmRUooNKwKmFMmEWDNDUjIlLMZxxDO3SDd4mxH/jHmZmabCssy8nC9croP4bSGeHz210FNKDbrEtoXeskGSRsmhaSwlMc+FnGOdejIgdMrdHbd2c504Tgfujg+cTice7h+Y55lpnkgpGhu7eqNuA8NkV+B8wwBmvuvgiJbo4KDQo2MU9eruVADorp8cbgmjs8SQVYj6EIdG9hM6y/394792/x8tGlrlLlQJgbGNrmYqR6YyU8pMyYWSMykVQLleVt5+9Y6vvnzP5emK9YqQMQ8tmajT1xZSnbQjAuE6OpXCVJTL+kT1C8fpEECqVNYOa9uoW/wJpngE4vD2SnEiTsGZcoabqEK1ML5xFDOl98D91AYFYF9gPQS0rV3wydC5AD06RIO+IpLjwrmQirLkhPeNw33h9XSk1S9IueCaY9HvAK0K56cXSl5o20vwzhCOh0xtG49Pzunt5/zKn/hz/NV/+q/wf/yv/2OOXhARSopBJaopvO93HSMVzUayDzZLYRNkN4Ktm9AsHEWmaWJeFuZJA7cshZSUj5Pzcn3gfHnh2S6DpU8EQJUbH9E/QDjBGesaVkPDwNLMyLnEvZewhDloYTkeefP6Na8f3nB/emCZDlCFy8uVw93M3dPC0/kZneH169fMy4xJp/fKtkpkLJczj49PPL77kvPLI8/rE0kSx4MzTwti0alvAibhObY5dEI0XF869rPLIAUH/jRJIU9lHLDhSZ9LAzNaV5xM3Rhcyj6kWrEjdy1pbPqOSgHJA8aJoBqcPh/UkuEI4aFKgPCku78/kqbE4/OZdV1vrr7WAmNrttK2YZrQwxZoyiBTaDfJStKZw7JwPJ6YpgWXjeaVtV3xXgcGLsxpYp4XTscTy3IYBOESjQWNpoJZo1ml2kbtG7X+PETWUdVhfhNnh2/RICqK3PAJZ3SLJJTxqpA6N3fJ3jvZBP+aulx2EHuw/APs/bBAdTDvVYd2LDltTAKa55mcSgxpKIWSJ5TMujXm48x8yLx7+8jz23PwdXw4i6bIwMxAG2Gt4rBMmcNh4XCaScm4bG853H0MOTpYW7UwuyNIfPY1ADkY0j70ZX5jrndz1q1x3WoAsh42zOaRfSbiBG1DF6iDJFUtBLPiU7y/R8NcU1gauQVrvIkz50IRoa2Jy/OZ1IM8mGTYkoxOnnujtyt3p9e0ZQmP+Xrl+fmRaZk5P5/5gz/4EX/sW5/xr/9L/xb/2d//m7zbXkg5uqEp6rU4cBySMcrazlSEKhaMe4lWeBASOwmlNnh+ubC2jYf5nmlKLPN+6hrTJLx588DWNn6SvmC7ROlkNgRJLqOL3EHKMKQMAJnNSCU8zCjD3lqCkqIEWXNaMofjwutXH3H/5oF5mUk9cXm5MB0S+ZQ41SO5JO6PJxagdJBm1Hrlcn7m+fmJp6f3PD695fnyjkt9xiwaQKfjK+Y8xawBV1x7uFUwEiDL9AYvX11xSeRlZj50rGS8gRUfA01s4Lgha7Ot0sVD8N76bY9kIXiNSsjKfUiovGOyW5EP3ubAJEMBoyHfGjijpoTSOaYMMvOS4eXq1FZhZNGtd9rWGaJNspdgy5uTmCj5yDIdOR5O3B3umOeFlMPeyam4b4M64yR1ck4cDgemKVMOOTiUOoTgW6VZpfVKH7zGnyuAiSc+DMiIjmRwE5UkLaaJDMM6CPa8WWRoRsK1Q+54cmgh4BUF6dGh7LsgdLB4d0+kOG4DGxMfzrBJsGSUEpquZZ45LHfM08y8zOQUtX+rTpnz8DOaKemR58cnrusa5WrKtwDpPdGrxGgz4vecjgeW+8ImzrVfKUkBY6srdfNo0fcoA1vrbHVDE3QLj7LIShtoxbyxbht1i+lANrppZgFGZ8roiRg0wS2jScgibC0wvbjOcI4IJnUwm6dcIBWuV+deD3xr+R53deJZnKJxovXeSYM8FACucr6+0HpnSkKZM3Uzti0GjLgZ73/vH/Ltf/av8D//C/8q/8F//u9jhxzdoDFVR6JfQ3VDcg/zye5hWuRBjIxsksjWxl/O5wtv37/n408+YplKaPDSAJKHYd/xfuZjf+D5fI3WvIMxhnlUx3pIkNq4VzYUHj6gCwxIgd0ETzq4folE0cTh9czDJyeWUvAOaQ4jQrKRauJhXjh0441M2LlTLzUoatcL68sL7XLl8vgV5/WZq52prZK2K7U3TodXJDJWhbZtbEGIwzVRrWPVMJSndytenllOJySDaANPTMsU1kM5rHUkBfbi6phGgBbfHWD3wSkxgATJuFhQlobsr/cWz4o+hO/phifqrnQQwmTAEstyIKWMpsRlvXK5XII31wTfBHoffZEW5PESkrXjfMf96RWH5UBOJbqdJEqemedCSkaTSvMVvJEkbLvLksNFNkTOoW7pjd4qvW5YC9F83uVN3ySA+Y5DwM1tM3QSHywuJKwxBxt3b9FCq05Khm5D0JqJQSTK7T1DkB0YRh+2GyJ5SIX0pr9MEmzpFtGBMmrvZZm4f7hjng+UXBCUrcZYJvEUNslmmFTyGpmO3D5ZdKvMx4SbLpS0cDzcczqG3U50vozaVtZ1Y73ayJYMazEMY2sr2QKYVY3RaegGUjGLqTnNNAZQmQZDmdA5umt0nDa9dfFcA99Qhrp/4B/X1oLK0hKUFFITP5K88HD6lEN5Q3t/DW5U7cNTKg8TxnguyzLTrHG6v+fy8ky9rBzmiZwydbtyPW88/uxLDv/w7/HX/vn/MX/z7/1feJfTsJaJ8j7YCYakjmgPXpVAGlOCbAju9w7gfp9x5+3TO754/5b702dYb9B7bEIauiilCgcv6EGDm7DLs3pYx7jFEJK6dbath7A4NfbBJuYR0MJSIsqHIFgSPlNzoiwTh3kON1qgr8r9JfF9PuZ1n/g0H3jlE1vt9Cys140vWuLvnt/z+eM7Hi8vVIuxYiGWqVh6QVOmyAwNqoSN+cZGF6N6EDutC1jjRZ55f3ckzw+0vpH7BA55Sjd+IhilpDAKGGLqoChpYIRjMIyNzmLtV6Qp7lfMhN7DNCBL0JbKPIXEToipXgwGvYW9c1KQKXNkitFt1XheL/St4a3h1W9DVVKS8GlzjXF1MqM+IYRdSMqFKeUgJAvRVBi65aQ6dLRBu7Fhnn+b3NVGl7S2EPfbzxHARNpg4dqeFIHnW5BiDHhwH/La26ioaLXXOixakqKpD1vgCGhGhxS2zbFUx5KzNrrDjvrQy43f7Rrf6RpjqqYlc7w7cDo9RLnkwrqFKLrVxroe2NogIeo1DNrQQVuIsmrrlYB1oiWeZWbOx5iPV4hAtLXYNGuL0WPrSt+E7XKlERTp1IYAmgpagSDjbS3wr/BFSlgLxrra2Gcd6rp3coLRPNDBW6Mj5YxKQrUgMqE6o2kh9YmHhxNrmyj6gNsTNIvOGEGBySXj3m7KgTItuAlpPpIx6nph85WphAfUVq98+Xu/wQ9/+K/yl//0r/Of/Pf/L+pBhytHZHR98ANJY8JOD/b+TraM5xflrmFj0QrWKz95+2MOJ+H1fdAUou0cBoUHmZHJmHon6dhIo1wBpbfO5bKx1RDo17px3qK5M+xN2f31d4uXDreBE94dqw3PE+LKfc18fDlxaDPf4kB22L66UiZjUh2d2xN3orwv7/jNy2/ympmvamUdmJdZ8PvAmdMp5jNuTt+U2jc2r1y64VtGTJDmWGv87He/pBwy80Mij5au1VBrIA2XhozGSRcl5nna4H7ZoKlEECtpeMqPzl3zCF61Nc7bsKQ5wzxlDkvmMI9qxYLonVLQO7w1puIoha0sbMmoVqPrWz04Xz0hqYczyFx4mTam0tBhccRw38Uk3D0YWbMb05TBlVZDJuZtcAoJmkddK5fzhev5yrrWob30bx7AgBsr+sNrKO+daHNLWKbEN+uemYJ0vCqWnF62MQHa0W0wuZPEeLPBh4IdbhskWQ+PJO8Gmm5WLTK6LCSCgV9yiIwPQTDUNbO1SrnO5HkK48N5DrfUwYVRVUzCqVNaIWdlmU/QC94nrE1IL8TkskRdX0L1vznbJch865VxCncQG0lOdFJ8N6szAZnCT60bvYe9ddsa1BaBCY3SOg92epEPQzyIYQoh8lZSmpjmE/N8ZEoLs01k7vno/iO4RtdnqyBace348B4XCkkT5iu9hz+WlsLaKpoStW2068rrh1fonKnVuf7Wb/LP/4m/yN/58X/D++Jca6Vro/crTdYYphHtsQCdW4yQ21qU1zsFrBSJ+QMZSl4w73z59CWaXjPNMwxnzzIwSM3CNDLPnBIqAfbHaR1rbVsb5/OFy3rlWCfWvnLtL4hu4PWGQUYzyeg9GOR9rVwvF6Qar86Zb32VObwFP1ee28p2OQeMIYL1cAj+6M0bPvvB9/hVvfAPfnbH7z9fQKIrPRT24M7atiiBtkxbO23NrGNAsA1yakzNc9wS5/eN918+8/F8hybluq5srqTmCIluGp54A3BXCRhFRrckZkbuA0UUlRlMw5NubazrCusQ2vdOs8qlbphmJMPheGLRKbz4aqe3Gg0pKbgbU1LmrFRJbNcNWxmDciu1C90L1q+k/h6xgj+kyKh9WItPidRDghfJioSCZMzerFujDjrRjrddzyvXl4312lg3p67bwON+jgAGDC7YPgnIv8aQlq/xfgY4IhIbdmj3ugWXRpNRNDyyTAx1DTZ6jnJp10VGAIxF4fsDsw+eXlEthPZRVXDbwDdEo4UcOqrBwSoffOjDCif4MCkJnsbD3hJzmTgdTpQ0BZXhCr3o8KhS2lnZLuGL36uzreGv3s2QHOzvSsisck7hUOs5Fq8FFmQmtBbuln0Db1FuTiWy0G1gFgrRwVVBtdzuf2SxQtLMko9M+UCWTBtzAcICL4SyVsNg0bIjvobuTZxSpvCPSonT3ZGtFdyd0hK2bWFpNOxt3n3x+3z/z/1p/vwv/3n+09/+L7jgbN5Gl8jondBAbka7dtoV6lWwlqKCJNjcU0mBlxRHk5NVWV8uPOXM1GqMAyuKSaeUwmE54FlpPfq2ENKqEBAHMO8d5peFZbtyXQtb3Xg6K5u9IBr2MexjrCyaCVvdWGvj1fvO3bXy7XfG9P6R576FSNxiO+SktNZCLG6dn33+Oc3g+9/9Ab/y+of8/vkLNgwbOJtZZH5iDHPAFt3ulELyJI2bT3wfeLKHmeHlcqbWmWQ5Dq8K2QAL1nzvFnMSPHFzPkDCA66UGH1WghKhkiMzNufYnHWbuV6unM/nGEbCwMmSYtqR1DkcD2BOXRt9G7bUhIRP05h2VDLznFlbAPp7DGhthWvj6f3IvLzT2pFaZ9a5UiZlmktYSGkLvHRMM4DA84IIXW9E2+vlGrKqa2XbKn2ttO3/BwHsw4gsGYHLRlkpmO1DKvbv/SDpBqfXCBhti4GyfZyMcYDHGPqs3LAS3e1xRlNg19cFoz66UmWYniGOSMWoNNtwQodn1tlZOWENE2OarI2R78ONMuUPXdTDfIjSqym9Cv0q0IS1blxfOtcXp14q66XTrh2r22gGBEcuvPyjoRETw4dubovrtgbN4iS2zekmeG1YZ5SNgQkW0Xjoo5N2G4gQZwLmgqaZeTqhUyFbLPTtfIHzlWadZGG2aCZkAdNokojGBjA3GkYpB86XK7YG1tTrhic4zRMv5/fUL8785T/xL/Lf/M7f5qVeo8Sqsam2zajXjl2NejXqhWiItJgPoCWhWbEswxffIdcI7A3ePb5nWWcOh8Q0Z2rLnE4npmkKDKVMgXn1gcUmpUxTWK3gIfW5W9i2mfPlmaRw3oTWXzjXM0GidBZRkoRzSX678lG78is24a1SzTgy04pQppm6hmxKtYNN9LqybVfevf2K16++FV3H9QUEmu+a0AEOd8PUiKECwe+TZJGBJGg7Pjf2UCEkdtd1JR8EaVG6fCi6UiS37YOFkA6IRpOEZ9hxYZkWSp4+EH0xkmbMTtStcr6eef/0yGW7YtaYUmGZl2jaSEPniaUU+rVR142+1bF3dt4hwZLXwcAnxp5p9sA/qTy9PNK98nKemaeZZSnMS2I5HZgOiTIJZRbMM721W9YlCrXWGEpcG+v1yrZe2bagbvTrFo2UnzeAfT2I3V4j+xJxnD5O3f0WDgawDKpr88h4thaA/0g11B21aMGH/Ifo8omQU0Y6ICmkN0Nrqeq4TIHPYZgWunXqdaWrBU61XbjUC1u/cO1nmoVHeRgg7jMeA2Ccpolao12dpoKSadWp14aXxOXaeXy8cH1prOvG9XoNALkPA76dkiEC2rHqkNINHDXT0YaO6dK9ExSI3gKjaTUoKOpIiZIpOmvh9po0xcRlaUhq6HblvL1wWO5ZdGHOmbSCXyqSMvVypadMJQwdlykDOSb7kKnmtG3l/bt3nA7HcBYtCd82DscFJ+OaOZ5e8+4nP+bTP/VP8Zd++Gf5v//m3+JltbDAXo16qfRLBLK+efit1TGmbTh/du8xvacL2oP71FRo7si20XpFfEZsQs1pecNykGx3nWirYR8URgEx8Uh0eN/njMjMznUqa+J8TrTewr9eFuZyzyEduL8KvyYLf1zvoHfOdSVpAQ2Om7XOcTlg3Zhyx1rDcnClni4Xzu2FQ1qYSkbPF7w3ujCGdghIZzVQr+BOw2kejhGawkbKPbIwnZxpCUtu9ZCnaVcoAhqVws5tA0N9d8SNfTJPMzkXNMVkrZzT4AiOLD1lHJiZmXtmeVVY1+BUxQBdwSu4CFkDlBeCn9Yb8dw05lzG4WNBQl/CvTZcU4Or2THMG08vzzw+PZKyMR9n7u7vuOsHll6Y58JUlbJqcP9yJA8ijLmcRl031utKuw7M+XqlX531+nN44u98hq/rtgdEO3o8Pry4RgY1vhawgI9ycPDHLIzlrFlkBJuSk6CdAPNtTC7WkT4bWBJMnKSJfYK3jIk2Ppw7W9/Quk/52biuG+fLC5frM+fLey7rmW3bwHKw48dHEJFosacAqK/rmXt/CB/+rWFWKW0OPOjauFyjxOrdBwt/CI67hdOlDBKvhoKfIdbuFmJhr3azdrHBjQMgJW6TyVNgfX0zUKipjgELgQ2GL7ryIs8ccugd5wSpFbTH6Ks6wNGcg27ysp6ZvTLZgmsnl4m7hzfcP0BrlbIcMTY8C9dWeb684O0er5VvffYJ9vgzfv1P/hV+88c/5g8e/x7X1bieO+0q9C30cTvQqvt0KR0UiqbYcBjozQaXLTJGPFwg0gZZJ+YpYRXa2hGLmQfXurHWjonhfYEeNixB+lLYybKDXnA63DPlQqfhF+dO75iP93wsM385f8oPp4/Jnni2FUXJWnCiw120kDSTxPEWiVQ0EIycw1zzId+TLeESv9/NYnq3aGRXN0b33ondtYxBeTAH8eAilkOmLAmdCEK1jk6dfOjemoXddAx0icCUchzCNiyrDcOlU+YghCYPXM7Gmkq2kEpibjXIsNbDaXVMRcomlFSYpsIZx9crlmLA9OGU8JRJU1hQb2sjFxlDdMdz9dHxbD4O8rimdb2EbDCPGOEEruedySd2FY+14ffXwjW5WTTLLudKvza2nysDk336NEOV92GG3T5xOR7UXttHdAhnCWGX5TJIjWYpgHzp1O4xeTlFhpay0sXD5lfDVBALLIudWT1OGXen9s5ar8gqOBM5h37qcr5y3c5c12fW7ZlaL9TaSGqhRdcytJ12I+YKxrpdeTk/48sJWrDoU75Eedo31q1GmTVoFL2HeV+0UEfoHpllJ6Yo+XBE6C38zNx9UDbGjEmiQ5ayRHBiZLXjHnqDtsbvZVvRbmN8/ZlH/wptjix3fC99QnKovUX7fr2yrZ1Xr4+8ur/j8at3WK0shzvIM606p/s7lpzIZeJ6eUFwXh7fIjyFXGk+cLk25sdnPvrhr/I/+qf/Er/3xVt+9vTbWBMmXZCc6O1Klig9kZjAI7L7+ocdjxH2x7Rol7fWoG94r6xZKFPhtByRlGhri64XwmVtPF8q5+szp7uFw2GOASzLQkqJA0GdcIu5n+5QtHA83LOkifs085Ee+asP/zS/dPedEN9bC9pLmTCHaV5wJ1x4hTBOnPLgWiWwRCOcgb/70feYfntmW59IqUT2LPvgPhkDM8YBL4pkhRRDirs7jDIxJUWLUZaC5j0W7zPOAWLmYxgIjrJS4nsEbjY2W6vkrkhOJE8RsHLQmlSCX6galIbsCfMN64r1jNWACalxMGLCbBPrVphawXpFu3CcC/MxkSbotSD0cI4YeLSZhsX6FhJD9Tk+r23UrphH53zHzGWwCiJshFtMq0FYbT1wYasCPdO3Gh5p3zSA3fRTI2PB42FETNpLyg/Ba/dj2n+WGw+oD2O2DhILo+fwJd9Jqr2HXCjGp0vMjNvHeg1TtJQkiEjSaX3lsgqeGt1nkgbnaV1Xal1jTJl1JDU89THCyod/WfjJ+7jw3pQulfdPX+DUsLsRG6z9znmNqUG9Vrx94CSJD4cEYWSN4YRvYzxYlAExhMFbyIRE9iB/s/YLPhhOcyMjmBpR3cTv39aglvTmN/xpW1deri/kux9w9+0DpMo26C4lZbat8v7dI4nEd3/wQ37y+7/D4+Nb5mMn5Svb9szDm29xuHsg5cR6uXA83dNKYlujjd1w5hdHfvdH/Mlf+/N85+/8t/zNH/33fLTcc5FGRzhOMz0JWjutC23My4QwucTDPrtunUoL3zCLgOAYta68PM28fngTTPouw9CyUTfj8nTlDz7/nFScN68fmJbEcpw5nA48nE5oSvRa6d5p3kluHGVhXk58ZBN/9bNf41fKR1zPG942nEzSgqswlyU61xKwwm6NnfdmiimenbIWLudn7j/+HngmIUwkeoHmNVjnHo4O1WzQ0DzAfAnoIrSsYQ2tpZCWCS+QSriTBucQ1NMtc8djgr2Y3pIIleG7lcHZqFsMdNlYETVSPpLzHMFumF06dfDIMjowSFcbDbFwuLXWwu7nMAcTvkVnEoL+tGiCAcMwymYxD5gjh42VtSh3HQ0OqISXmEhnmpcBE+mtolNK3Gcfc1Cb0TanbuGUW1elbj8HD2x0owew6OO0cZo4uzdwlIhfz8D2TuJwdhon0iCU3dTl2uL9zGw4UATY2ZOSxmSj5B334B4zJDFpBM1uja2tyNqprVFSCZ1XrRG4xJmXQusL1lba1cE6w8YIb0GsM5ywY+nU5jy9vA/rHoLiYdZZtyt1W8ONsg+bFtcxNYcYmkAKqZD3Mc0bRNKw1x0gPILIqDX2Zsb4u5nRPdw2wwanj885HB+a0zQ4Y9vUqFOlLp1cvsODTlSCWNi7DAPAKHHOz0/8+Pcrn376bX76s89xYaj/Jx6fnnj71Tvu7070upITtGbkMvF8PvNqvuPx6YVlPnA045//k3+Rv/9bv8HjduV6DtJsTHVaIK1o66Quw6qlB/+shZWRI+Q8gXVKEsgN1UBq1rqxbQ2fAh8zdgJquMJenl5o7Uo9XzksE5JhOS2c37zh/v4+iNNEsL/XxJIOfKQL/8LrX+L7+in9vIV1iSTytASHShW0kFMmiw5szWk1sNs07n1K4cl22VamVMhaOOQDTZxr38YMwzacg0PzpxL4kGYnLR0r0a32ERnn+0JaIBUJW53BPeqjYYPGNB6VHBIjCc1wGvhY0jL+gG/O1hvX64WlFkouTOVITinCgzqm05Cydep1wzwO1eYxXai39uGP13CSnSVsnXrwKsOuJ4JqzG0NSaHkgnujlDh4pWVSnsjZ0BIYnsXIqgh+4zq8NQSlGZgr3VM0o9Y4oK9X43zu/GOURP+YDGyPRl/bcO7RAlW1Yayme3gbmy7AQBkYECl0UV39lsUxphDfTmpVTIMol8YmKkXpCeZZKEnIs4doevixh5tpp1aQarTURkUbAbOUNABeEFPOvkUZZ0ZvFUvh8YQMDdoIkLVeIsvyYFMrQQjsZoM9HIFCLLIvepxkw8c4ODA9FPUSmqqgY+CQCXxMI/PaXWsZuEEGXEu4EEhM9zYzanNs0xBPS8OvDpNgG3z0SycOkjm34TnsMjJAkDFTsjXj85++5c3H3+dyvaB5AQ0HDVmEum1MKXG5npmnzGW9Il45n1/46OGB6/mCffWOf+7P/gV+93d/k//wP/+PeXW84931HIaUCj1ltIaUS5JTh2dbW4MpvswHjqeghUwLrNuVlAV8wtio7czW7sJF1A3vDRsZbJL4TJfHF9bnK5IgP56p10ZfnVwSXY0ZYc53/KlvfcafPn2LN/ORy/WRrTklzUypjO6exoEXxJMA2iU81lQ1TAusk3Om9Ss5KbNnpHcejq84yhqDYFumudMl8C0bfni7A0MEpuh4kwzLodOd7jM6xwRxhLiHJdNHvdhrjB/LUoZBQzQpNGWWaaZM5UZ36T3Ab5eMNWXSypQaOc1MOeNFbmVqs0Yis/qKeI9hxIShwI0ONWZtSopAnlUHyB+WPVYD6mneYi9ZjBl0C/7anmFpUpY5OJophblBt3pzXQWh2aBStEbfoG6ONaU1HWaMHmX4Nw1gO0M1RiiNYR0W7GC34WbJmFIow4VzpGKmHp2+kYntYD6DR4INjGxo2GR4C3Vv0ITWlGlWyjRGph2OHI7hutqs32gcrY5OTbcbD2z32spjgeyAZ73EPMc+WNTeY5iCD+N/TZFBOWPKz61cHl5LQ9Jh/QPZVDpIH4HQDXoa/xannHjQFlQd6TqGMjiQUDFgdGYxmsQQi4yDGFUN6YpqRzRA8Zi30PB6pTejOGzXF7ZtHR5MLayTB76HZvJ0IOeZdTtzur8DyczzERHnuq3c3T8wTTP+nKjbC/O00Bucn1cWvXKaDzw9vuX+O9/hn/vzf5W/8w/+Gz5fr1QJx84ulVwdzTNPqdKvayQ4g5LwejkhU2I5CHf3JySd2WymE46utoJ749rO4f3eo6lj3qnrhd5X2jAVVHe0Cde1Ytt7ZFXm44LMncO88Ge+/yf5C69/idmN88sZrz2GXwygHFVyXtCUb5VDdL8riX6rItzjAJjnA9ZqGEX2jUt3NGXmDJYb2iZWA+nXWIcWDhThHhHyIROHLJRT4vjRxHSfQtajmU2NMjSR4n7j1yXJzJrJUoLDlxMpZebDjJZwUu2McXsyx/ols10755eVVOaYXM4+pCUOULNGa4FfJ88M5B0XHVI3p6cKUyeZYc3DhYIYwpvcggdowlaFulasZmo1yhgJKJmBhYbDcM7j3/d9phF2YpBYzHYIn7foOtfNqb2iJVH05ykhdw6WyCgjI9WNB7/zVgbP6TZWawcbfUwq8fEzu2xgGLPJ7Zd87X9u5j2DpRHdjzQF436ZZ5ZlAmBd19i041rCjqQMwqPcrrFMhWkJf/Rwe2jQg5MVpVsfonIJiWbqmPThgBkl3a7/CzXz7otFeL/jiCUkxQMJv6wPBopucRC4xWmylyqiNTaOxviy6EtHt4rU4g+xiZLHPD0juEDeK2t32tYoDfKgmmxrD5ylGyJ5ZJudbdsQyZRyZC4nJBeeX15ick/beHz3JblMfPbZt/i9tz/ltCxhYWyNp/MLy2Ehz4XLT3/C97/7A/7lP/bP8B//1t+hLIeblGY6RLb0u+/esRwmzrrx2M4clpnDUfnkk9c81i9482Yhlweqr1zswuN5xQs0Xth6pq0Jr4lQ3BpbHzQTiWzfHU7LiafzC/Vlg+kKl5V8SvwLf/bP8evf/WXmBuu1IkS7PmkODqGH/CbnMjKufrNxYfyRgZhryiQJ6ydSiu7alGkH4S6/om0r1oK20DfYeh22P347pCP5EiwJnp2pKKf7mfmolDmmqQd+lG52Sd0aOSklJzKJQy7M0xS2UWkZk4eC+IkNwqzEIA3VyFzXcyPna5ScWQPsT4IUmOYpsvrgUcTw2jFrMojhY4hO/+DH5S60HplS6BU9aEFbpW2hgZUeXnaSjay7w2pooHOSMdBnDA4ZAH6tNn4+OvPWy+C8bXE9kzMf528ewCKycGsP28jE/PaFiPty+9sO3u9f871KHF260ZpmlJIjXO32HrcszaNbkadCKjkWUw6lvKgyT5lpylzXxPW6DX+jOI1CUR+jr0QC1C05Y8VoRT/Mp3S7Ba9d+IoMblqWAFTHFWYddjm+0zji66KBW3iKjHTfB3RCx2lym7ysgy6xmzimHNbOruAZtHQ0j0aFtCg1STEcWA3T9DX5TnRm7493HNKCd2eeD1xfnkaXs5JUiWFHiXk5ME0zJOfLr77g4dWbMKArM2zCy/kZe35iu5z55LNP+NE/+A0eHo4sc6b2xst65VSPTO+fePilzA9/9Vf5zhe/w7d6ZI7TcmDC+en7t/ST4N3Zjp3fePxd3nz8isf1kdPiHA8H7mfnzeuP2OTCF88r7sIqijVja1f6lulbvmWq1oNSkVLGeozySlrIFmD6wYSjN/7ar/xF/toP/gzJNy5tC3siGYFBM6oTrplpDDiZksRhxwCmdzeL/Rm5DU1fEJNzKdhS+PTNt1jY+Or6AltiawlHWFtl650ugffsZSkaQUwWOJRCmiymaM3E8I0SVtiNkKZNKZwzllw46MyhLMxlIqcJJ9OsR5UiOjK9GkaE5mw9MvK2bfSt0NZGnnMcvs5QqSiH5YB6xv0K2aI5RLuRphlmkDF1Pb6+rhu9G3ULpYpdY45m3+IgXcoCJd4/F40xbTr0m+Pgj+ZVcAWth2h7WzttNS7njZezU9ewD9IkSEnI8keHp39MBnbLhz5QIoYRoVu/AfgqI1CNNql5ANDO1zIu9AN7XwY1YuBft9+zY2QjlVZNlDwx5RmVKORln66iKYzpTLn06y1zi2sddbjAlBMtQU2OlHYzZXPx4W12809gFw57i9NI1EKzqcEm3/3PYgxMDCYxdZrESeKj+WA27KB98NMyMBcoNTK1HD7mUoIgKIlhfx3YA5JGF3MsIk+o5WDdD6lJTspxnjne3UWAT0opE9ftEsHWjDIdmY8nynKPltCKvnp1QjUxTxPn6wt3y4GcMl/97Kecz2fev3/i/v6en33+Yw7HifvjgptxqSv57PTzE9/7zg/5S59+hx+fL0xT4ePTiWTO30f56PSK6/XCi3dOm7Icj7ytB5I63/roB1zzM5/ef8pV37HxjHU4y4V6yay10OsV643NndSik3xYEu4Tl8sVaWGJXXJmFucHrz/hL/2xX+XXf+1PI+uVS185rxtzmcEstH2a8JKiE6ZKSUrRgBfCPQSGqQo61mIpMUTDrMf3lplWEvPdG1bdeJ0n3r+E4YAX4Zw7lWgS0aPr3WvMsXSPuQppmrCkoHlUMyGWFm903w+9xJwKixaWpCQP/qRLR3IYevYWw2e3HtrWWqOz23vHy0xJQlszPR/ZciMvgsW4SUhCLplEQUVHJt8xzaiEf5sMj3hrMkw2fZe8otXY1tCjbuuGrSkCK8ZxyeQpYIWSOnkMGt4HAEWICCugPrKubp2t9tDPiuCpo4dQcogqafo5iKwfJESjbPzav/vOxI/6MA6wgWOFjYbfMrN9kvaoJUdr82tjsCTA6vj/4SHVu3O9VO7uDwNFY5yMjtvQBg7pURBUE5NOJNIHjpXFRo7hIreeQXwe9RhR5jUCxtdTTYIUquxi04E1DD2iu2JDThUPKCZn7xrQOGkG8ieCFEVKH+PcHckWWdcs7NNNYjCr4ymM6wyJnkhL0ApeoxlBdzTDcjjw6avvkseQ3TTGyomCt9BElmmmzAem5YDkkEt5d87PTwMAbjzVxv3pjof7e3725Zl3X37BD37p+zydn2JASZ64bhun1rEJ2uXM8f6BX/7sU+bP33I4nbjLOT73J5/yVDtfPr7H58x3pxPkicf+MTl3locTV3ngh3ff5911QY7O0l54sXd0K/RyxxcrbFvl0jqGc5wmXlNoony+VfKcOOWJh2nhlz95xf/kz/xF/swPfwVfwzCxNiOP0XpC+PClPJHLMiRZzpwDr4wDMtFbRVOUTmYe4H2rMR7Mg+jqaaIcj5S7IwdJ0BtlPdCTk7STJIXLrgfkMEZlj1FyIX1LU0Kz0OlUiBQeQ7uz9U52QXJMlQ+eiWNEqdcdUk/h6KI9NMDB96fbeuOGPV9WqjfO28rLunGyE4d+IM+Cy0TJYZ+0HBdyzhQpuL2g/YJvnZTD0FN8o7crMWot2AABXYdpKSnWIa7x+YqQJ+FwSLfDPyomo9uGeqY3j6lWpmGR1D4YHITfW9BsNIXAf1oK0+GPLhL/sQHsphaHD0A+AzeIMLHXhjdcbPf7ugFdX8fhfC8td4G2jHIvvikwrQgkl5eN88uF4yFjnjFLdEs3b/Yoy4KdLiR00BZC6BVTi8UNtcbO5tgpCzEE1OgaXTIVGc6ngQMYjoyOomiwoTXH50vuo8sq9EEvYXQUb/5XOwaSU2jHRiudbOQlWuqaPdj7CShh/Kg6gvhgOIfDaY5pMbWTUuHjjx44lnu+/fG3yamAJDSHRMN98NHW8KxqNU7m+9evuPoVEeFwOgRmmJWf/vSnPD4+Yt5JObOtF96/vPDt7/2QennhuIQdTyqZ490JNUHLgdPhgY+P53CUaBvdnFdz4dXxxKenE02cdv+auSy8fXlmOiykpfC0PvNp+oiXZYKtcX/6lNVeYNqQ6cTffrqynE68vJyZ8sRdWSIjnhb+y8tGXzLfWx743t0D/9pf+Zf5/sM99AsvvdIUdCr0dYvTOy8sxzvKNEcp6aN66B00YXWLLlzfLZttQAEWMrgBRexW1TVljnd3uCe8Npa10quxSiVpDvF9rzG1a0AJ+JCJAaihecxY2H9XDZPM3hriwsZGlcK0lCgVXfDmmDfKVHA8ZpH2lbWdaX2j2ZW6NXrtrL3z7uVduBZPBz7u3+G1vWI5TUFrESdLYp6cUgo+OcvcsWtji2kxY5ePyshCZuUWPmy1BW8zjTGGoS8WcoaUWxByc0Y0ZhQp6aYXVsLYISosY9vCeWJbjW1r1FbRMcQnnHRhXtI3D2AfRNofUsC9S2O+Zxx8YNjujq076C8xektSANo6ZEA3isUIkjsjXnf/JwbU04WnxzPTnMgpgkrrHT8QgWmUAlKm0QYeqW4zRg9sdLPiYoomWkqYwtqCBmK2l5tKLjF53HofLrH2AduS4DalnEamGd0WJ3CxJCG+le7QQ/aE2hAxR9YlRaNNXaBkHaVoQ4rEsFENlw7o4copM6oTfbhMZJxPPn7gO59+xqxHlgyaLbI9C/zMxu+ul5XmxtPbL/nyJ2MW3+s33L1+TS7BmTvd33N8uOeLn/yMuq20ekXdefvFW/7MP/vP8rs/es+nhyMfPXwSXK220q9PKI2cM6fDMRQVQG3G69M98+EQaoTuJJmo7cpnr16xmXOYZ9xek1KhceJeDvzB+8/x+yO9Gj+7XviL3/khgvBWv+TjuwcO85Fr6zQSj1+9Iy+F7x5f8et//C/w2d2C9I3LtrKuATCL99sGTNMcVIHWORxKBK5uTDkNwD3mZKoEz87ZMcYOKmP4buBbR83YtFC2zCyZbZpiJkPZyGUlpSFF0tAsNjew3RIpkUwRH0RuxrzMnZNYHe+wiYC30FMSPmdFE8KM28ZlfaZ75dpW1rZy7RXzIFfbFiPRmhlrN84XJZeVLtHFV30ga6LkcCnZ0nbTJJaSQnvM4HiZU2sM95BBwl63GiVqj66mpoBzSgn7+JwG3SLJmD0xqqPhBrM36XrtmAl1i4wx5hcERar1sL2eD4lpidmquZRvHsC+LuC+jVf72r87jAm/vse4P1Rmcvs6u/LphjOZhOsqwiCTBiAe2NEO+juXl86X/kJrzrV2TneZ1o22TEwl5iHmnLA+NFmD+S4SY+B6CynOXqLiGVWlJBv+YIwTJIaD5F2PaBYj3b2Fq8QuoyKyIlEhFQHt9BZ1qXVBxwIIFUC0z9mJjCPtjnY+eOqDc0MEO41A1D3TaxjLuWWsOQnls08/5rvf+pj70z2n6Z5PDh9xmDKpRUYpSZgOM9vmbLLRm6M6UabEVDLrunL56U9Ya+XVqzf01vnks89IOfHytNJqZb2+8PHHr/mD3/lN/vSf+pO8/+nvsmTFmnN9emZbFuzliW6NcjghrfH4uHE+nzkcjyxzSHTyXMCVuiqdTnt65DDl0dGDaZo5+h2vMXLObLXzW7//+8jHr3n3/MwPv31kNjje3bMC79YN/84PSVn4te/+U/zxb31GrY+0LQK4DdpKKQWvwjwfB7YaG8kdcpowayQJT6xWV3oPi6DeO5p33Skh5h7gRQxpAS+QD4XUr7cumxShTInjcqD7Cr6x9cg+TDSCkCW8RrYuFpPVqTYE34HDqgslA3QurNG5bomsDnoJcq93Oo3qK7XVEQCG31gfmuMWwdA6bLXyJM+UPFNGJ3OaOtt1Q4AyF7LEBK1cgqbhLtStsl4jq47BtDtUFFrHmPyUyKJ78zyC9D6NawzfiWpCbhXJB8J7gPtmOpoLhi7hRtzxYVYamG4M6/mGAWwPRF/HqvZg5T0GfRiO+z78fRBaIVJfZ5A9CIDOCcrA/vWvty8Z2Z3c+plR2iGcz41mZ2qvmM/BreqNZU7MeSaPErb3GHFvLgP0j38LaG5kKRJ8FtWQMUVnJk6gqcSkGFWluXGtV9ZtpbUQjpmHXk2GMgAPnyWRhqXA2TJ6696YGYghu5Yz2ZiJmYKqIXzdRHSYOYbrZdLwJrPWwYT704FPP37FR28+5u5w4DTdcSoHaqusq3NYjnTrvM6Fp/fvMGsxXdslCMBFmZaZNE2kFELsx8cXfvbF3+VX//ivUrcNbxN3D3fMyVlwvvjZj7mfM19+8VPuHt7Q28b53TvW6xMPd0dKmUAT3Z3z9cp9ayCJ9+++ZJoWti2C//n8jAhcciJPhePxwNZW6rZyyJnXH3/EVo1kQUe4LxPTlGjnK/Oy4Knw+iTc60xdO9959TFphtaUFroY1KOrWOvGlE5cLyukzuGYgl+1VSR5mFu2PqZpxcHThjqk1katLdQCZUyLRpimA8vxyHtfcWKUnmaYjhmXCbeJrU2Yn8gJLpJjgrtGCdiHrbdLTLcWFXyzCJISEETRoG2YdKo1fO24ZnIK+Ztr/DGxCGQWrSfrzj50t9eQ5NEl9LyirOeVr+QrjoeFw3IgndcYDtw6bauopGETdR0d38iWeg/oQT1cgkNMHnboMhwt0nBhTQMbjqAlt6AVcWO4dXjQhnOK91MRcjKkxEBnXDhMMbhXB34Wfng/B5F1D15f1zjCnnlFxLGdbXzTEenAsfeuXQDm0ceMPEyH3U7fkTXjhrX1wRfb+ThONCavl4pIJycjo3ivtKrYbMwl5gjGAqyIFJTISiL4DCE4ESBKgi4yeF0w5VDyl5QovnPJdGBKAcq6h8Vv0CfkJg2KieQgYkyTBhfMoklR1/iM+yQfkR1fyDDm/YGOyc9pyD6i1SyEFYt4qA6mJXN/t/DqfmFZFqZ5Au28XK9sF/jOtx54VV7x1ZdfcTgceXl64enpPdt6xa2RxDm9egVJWA5HvvXt7/Hq4RXnl2fefvEFn37n21xenlgOmdQbD3PiZX0J2cy28tUXX7IcC9t6ptdOSYl8N3E+XzifL6TpwNaM9y8vtGY8PX7BWjfSOIp7q+SUmG0iFeGrt4+oCm/evIkGUNv45OM3XNeVu+OB63bhqVZO80xZTtjjmYMpp7sjDw/LmJjjQIqgYNC9kcsU3HJJ5GmKdTWoATE1SCjTFHhMD6NJ1ULdglfYWkNT4KIw9Hspc3z1wPv6BbVvGA3NMM8KVvC+UPuBbhtmmVYmptJhVqo7VjUGyDB4ZhYWQwGcdmRIhELRAa3bB6J0A889qDXFxyBfJxUd2F14xHUbfMw+HB6aAB3pzurG5z/5nKSZ169Dl7osMfc0KD/QrhvX6yW0qu5j7Y81qowpV6FV1dvA3PjZUKgkdrOHPRnRMTvW+nBB9q9h5UFjDqlhCuqNpmGjJRLrZkBA3ziA7RnXHsRsJ+rFV8doNR2ODuPCzYfX+f5dEfFkZG47NqaMbh4f0sqvl6l7Y4ChwMdhu3auL5UXrbgpzRK9X2jTOL1aGyfTMITr4Zbpe6o3sC4UknfIGQGmYds8lZnCFC10H0aAHsF1rRW8jfuwY3djQQ6JgapSZjAfDH8LMulOFHSCeVxXDe5R6ogY1ifYwLMjeQMM9xaDeYdvkmsjL4nlrgSxNGw82NaNYzlRSmGZZrIknp5feH6+sNXO+eU9VldIwvufXZGS2A6nGH9VMvO8cDcduTstHB4KZWvU6zMv7Yr2hrVOPt3TLhvntdLrGXV4/xjP/Gc/+4Ln5xfefPIZaxd4uuDSmI4L9f1Gd+Hl+czxsLDVDtrwpwsvLy98/OkniAhP79/z/vGZV3cPaFIO88zT+684pAkVpdWVtl5QnB9877ssJfP+/G6AwQEPSAqcsJpxmBeSlGjwqMY8Y0mkNJPdYduoYaWAmbFta8z7rHWsQW724zHNCdpc+NnlHef6hKjHuC8LKhAZplmYOnRXLluPha4JkX4Ds/NOKTILY8sOktLgm4UjC2O4CwpNWmBNEuUVEoejiAVGO5pG5jbkY8Gkb90wi6aWdadfKu/a+9DrVuPh/oG6LNFRJ3Bb6c66Xtm2K32Qh0V3WkkYMeQ8xqCJDaxbb5lVsBD6mPfjMZ/TdzNOv9lBhRY5skhj51/utAn5Qxmcu0dC8I0DmH8A8m+Uiiigdqj9lm05g7W8B7JbHNoh1UH63HeyMhwt9/p4x5fGg9qpFxYd0NAYGuf3jTlXkijuG7VmrEU7vLUWI8i0xVSjPoZlpPQBnBMG/SL4YEGRCLHqMhWmFGZx4GjXOI0lPl9vcS9a75htUetnRpD0oWHs5BL+8Lk7raYYiNEq2sLrHLcbLUN06CSTD7H6XlcbIi0Glk4JcqXrGqPYlzFm65qZZeI7n3xGTkIuidaEqcw8vHrD0+MjSKYsYbs8TyWyj5x598UXnF4/kBKcK7z9vPPLf+rXeP78Z2Qxqjjn50dqXfn293+Zy9Pn9B52NG/ujrycX3AzXi4Xam+8f/qKMs3c3X3MddvIqcRiTWFdg4Qo+bJtnO7vyTmRpbCtnZ98/lMEZc6Fu/s7nl9euJwvHI8nXs4XTOB0PPDZd7/H6f6e56fnqAAsqDImUKYDrRnLFG4LIpFF4DE8t6RCmqao/FuIu2PyTcd6Z10DKpimAibDf07D3HE54MeFL3/rS75YP6fkQm3bmF3Y2VqLjESEUgqnw4F2gc0vtNqxjWggENlGNwu6jAppPHEhYT7cGhwkB47rFmVcGgNtIIwEgiIUonfrglgm7H/i5/cBIIH/ZbwrT++fx54yWjtGwyFO1SAfb1fW7ULzOrLJHut14Nshyo5hMztcJO6Dmxn8NCcAb28hkUv713oI+kNNMaqfobcQ3eV/IBLkYnbfP3ZGwzcIYHsJt+sbg4fFjRM2VDIRsPAP/3//nlFORnUYrTwxDZsP2S0Qv4axuXyY6L3/+8jebMw7X8+N53xFLdE30NKwAmXOsUn6jm1FB1HVEAuRed/lQ6OsM2Gc4HmcLmOeZA4piaBIIa7VnYt1WrUxtCOu3iwGq4oEVhHAroZRYwt9m1gw6n2NB7Wfmi6Bn0mOmZfkAE3zGLNuEtOCwmN9Y23PmARvbVsbp/aKxWdyhsv1CXuBy/nCeu5cLhfqeokp4ppIeaLMR44Pd2ztymGZub9b+Oj1Pa3GXIHf/+//Pr/0p/44n//oPWVdaaWACFtdmZcDL08XNCfWZkieeHx5Ya0VEThfrryaZ7bWePPqDdvzhU5IzI6nI8ec+erdV9zd3/P88sxyOPHV+3fM08zbL9/y3e99l9P9PefzM+/fv2drjSW6+IhozDL0yno9x2boPvDNAO63bUN1QiyenUom5UIeLiWGkAHRTF5CeC4CtYJlA86jSx6Tq0+HA4MHQ8mF7sJPfvJTvty+5DAvMdPSG7XXwKy8B/1iaA8P85F+VNr1Cbs0vPUh9k63LnWSGHumSHi7eaw1S2Oi+jjcY7hMrDuXgQpZmER6k9DLumKuw8JoBA4h1jC7qahzXSuPT89hPS5K6BmimqjtSmvhzWVjErhqdBZTDpy7DLucvW/HyLJEYk5qJ6YSOY7XShn8xN5tCB38Fmh3l5YdoYpJ4sI+gn1Pmb5xALOhpRGi9PLhLxSdQ6K8GxeyBy5GwNOvlYXxAcMbSnzYyTSBNOYf9jETUp10q1DlFiid8UAtNv7leYWWmeawLFknmOdIwXMWPCl9DMxEorunosMJdKdV+CDNRi7Z9/I4EfrLnEb2FoFvqyE07XXFW4zNMougk90phRBnj65qsOsB7dAUqcHib/RxfzK7Kaskwn6lEEHeO1ocU6VJ4yQw5Ym1XXiqz5SuyGVi6ZnvvPo263rh/bt3nJ+uuDmX88raDDxmL27bFsJ6d+a7hV/6pR/ycr3wcnlkKcYyF9w6l+dnfvvv/Xd89zvf4XfffoUkaGvjD37zR3z7e79Kme/IcuWyVR7uH7heLjw9PpJzoiz3lHIIWxxVmoRh37o5KWVKziF5GU0f18LL9fnmJHp3f8+6bVwuG5fLheu6MV0vlHngWO5cn96R50brwuXlTO2QUqH2PojSxnBWxM0CkBdlkoT3Tms6JlkreT6Ex/7zE7VtLMtCrSu1NvIYBrNnHse7I0/XK//wt3/Ek75wtxyZpoKphXQojc9lPTaqx99LyRxPJ7he2Fp0OlHF6GS10DymPOgGjM0c1cjW2k0QT5cI4Ht73oezMEN3qDM9BSboA2w3wD2aEAxO2zTPZMn0tXPpL6N778O9N8jb1rc4IPbgRQyhluFIkTwoUdbl1qHcG1C19yh1e1COXGNUn2q6yZ1wGTrhAezrB0L77nKso4TsDh/UQN8ggIU97AghO86lkYVpZKJfo1Z8+LmbCPyWte1pp4PslFiNbogMR8gxHOFDqRqB2AbXLGr8CEDbucHasVnQWWhz+KCXIpQi5JyGp3ccV6FS0LAr+UdKX0HoW6WmhE1zBONBpk0SzOgpZ+ap0GrMrmtOTP7Zegxt6B2W0Lyh+7TycKhQ8liEgVPcTkOJLLMP/6c8OENhqR2nUdZgd4sqOYFqpW1nzmvisCmLLhwOU1i6tBjM8fz4wuX5OdwCrltgKWuMCEunmWWCt1/+mDff+havv/0Dfve3/wFqNabblMT599/CtnJ8fcdXn/+U5ML55ZlWP2XbLpTjTEpwuVw4X154uTyxTEfyHJKPXArrGt283ozjYaFuVzyfWJaZeVnoJlxeXtCUeP/0wny6i8G/Co8vT1wul5AvXa84cP9wZJpntusZ2zYeH5/DIyvN1LZCKkgqiGacRO8rWJAh1RKbWLTjpcfEKMKZRDGmudB9IvzgYsp6zjl8/V24Ww6U+zv+4PMf89s/+n2uuXJcJqZlohyUphUtxjQVXMLsstZwfLA2PLTGqjM3vNbAsEoiCUx05pQQHdI7E64eE69CLO6DhmTBJUvR+Y8p3Zkk6eZBZyPiKik63R6SP5XIIudyYC4lOIoWXVA3p3YjpRQDNnpoK80Jh1eJgCUuMX+Snas58K0RdHYNZeBehNEkkTWKfUhwIgkC0K/5DSZ2WeI+JxSJ4PSPNhD/BwWwKEUHvjNSsUgbB46FfK0B+YfRtltA8xE0+ugoypAdETYe6uOBpL2bGQ4BezDbQXIbwSu6SWFcKF3Qnuhd6FaxWXHL4Ss2VPCh3o7TOX7/nh1+aDCYN+q6UvPEUiakxOmy90xzypQcOFvOiSpBpuhbp18dmRpuyjQlfMhAgpMWGKIxvNNsHKDOEFqPQbuaAsAfvkwiBQ3CMwnFk9BFBsHwma1NvOK7vJpesV5euJtPLMuCd3h6/8Tz0/u4D30HUePP9XKh187d/T2PX33Bdp35+JNPePv5j3k5P7PlhPeNH/3oH/DDX/4lNGVsq8yHiS/e/pRPPvp2AK9DQrVdw0Lm+eXC8eE1121lPhx4fP9EorOezyzLTEmZl/N5+EKNAbtZw2IaePXmE9bWmadpZKVCt0bvGbdwHnFRWofr+YnWK2WauGxnNMUgFsTZthrK1t4CtEdoPaZHHY9H8CM6hsaIOH1dEQ+rGtWMGWQtLNMxOuTWWZaJlP4/rP3Jr2Xpmt6H/d6vW2vt5pwTJyIyMzJvk1WXVZdUyWRJNEoEDNOCBxpIAgGPPNV/IBjwxEPD8D9gDjwwAWvmoQEJgiHYJAcGS6DkKpNVLLK622Tem110p917r7W+5vXg/fbJWyRkyXmdwAXyRkbEidhn73e9zfP8nsDN3S3vv77jsR0JyRNHz7CL+EmQVBmnyLix3IFW+4NJAkNILFIMiKmdDy/KAHivRKdE3wXPOFr15GqG7Vp7RB8VX2y94vp+zttuA2PaGTJIa/8cgeFx1GQ0Ty6Thpn9tU9EXR3QGtRsHtO1H8LscKBoEOSMEa22x6x9vJQ+bWjrkxmYBs1JJ9KcKTXax3P5lUZH7QEfHZHQIw8xq1FvZuiyjO9ewPqv/dYaY7KIvrGyLuxXLoffjovnxX/79kSsfVHf5Gnh3ycu6/R6nfGd9noONzA2eBdgnPP1sALgqqD5fARoUIWaKyFAHPoCuV9xBHtRDaB+NkX1PYpY8kpptbfPJo2w7tvGz+iCif5w356PSyPniisNadCSUQZUCiLBCmvPN9QG0iFtTdXG5/PyMjniqMRkJm8f1MR9vjN91FHVkRusJTPOjU2c+Pj6A+b3N8ztkWmTaLXactx7Tku11B9HhzXaMaOUyjwveFe5efuaUwpMaeSUVw6HB1KwYNPPPv85n376I+ZS2F88Y14V9ZGHu/dE10kBEnFu6B2v0SLWxa6CpUMg52VhHAZOpwWfAuu8siwzzXnTWkkkDhOqhcfDidNSKLXQtBCCY1lmQ4v3ixfBE73n+PCAOo+fAk4aua4MwwQusRxst7LOtmQfxkQ+rpTDAijDNOK9kLzxsFpVHg8HWoNx2qHAMs9cXz9nGgYkJX7yzdd88/7RUEk+45JjPKyMe8VvKqVExEMcI+LsKORktO5jG2HxvH+4Z60NidiVOQR8qE8HgOAHmghVvGnEHLZX7YQMlZ7+1SUZQbofmER1gvfQxLSHrTVLHcIjYiLT8x5N+n7a+HEmvK61seZKq6UXHJMSWZ2RpwhEAXsQ137t7JdaAYMeYDYpWm8WXN9tnRse+/QAzv5czvabThoxRXCO8wbMxOe/TgE71zERuyBiL7R2jRX9mnD2Ln4rej3fKE3f8SS+EJ5+nlab988ER6+2LmpqRnDb5Vlh9CJPbDGpDdda353ZC9iKJVBbtqFSk1Krwwe1gAbnLPGo6bcdY9+PtqYgnlYsIfvMta/aLIhBDEYYvOsttS1IfbDlJiu0Gsmz/Ron561k7Xhn+7/i+9yIxztFgxKiEoeAj4KPpmYWJzhpeBdJwfpoCWodRrEg3uxtXHZaSWNiPR1wDaJ4xiERNht8O7EezSMXfCBOER8jFSXXQs0r0SnHwz2tjLx4/py3b7+mVlN4BzxvXr/hxcsXHE4zH778iLKurLXixLOuGRTWJROj+QsPj3d25g8erQuVQvDCui7cPd5zcbFHXKLkBhG24x4ESwZfZx4fbpnne8o8c7HbGVJ5DGz2F+R1BbU085qLafmcPTC9KruLPc4NnFYlDFvO7LBlWRjiZIvlulDyieP9gneBxQkpeh4PR2oVpmnPNI0s6wmqcrHf0MTjhoE/+8mfUeeMj8nWC2q7nro0iMbKOi0HNOwJzjMET0QozhKuN+OG07qyHg/U3DgclGnrcChOg0WnSUNCwaH4hhFfutpZVfugZWytoI3gBlsxBE9V6+JVG0ENrUO1GLrazjInpeXcMdj2hG79yl/WilYFDYhTko99/d+1jdgHp2KSFJtOxIol2gscNlVZ0XjSP2qTPlW5J+kFCE49y3Hh9vjIZpPYbsenz+u3ZtL/PxSwp4si387zT/8u9nDs5vmn4iC/8oWfRtBf+XX24zyNI2cNvzQ1RtaTIl+fTLZ9M9kX8dq9mLbsq3Mz3ZXlsRnhMYA2Z9qV2F/EX/la3xIyzAZUskk1Wq604Lsf0ixPXiLBBZKPaLInzDpWytrskqU2FktzvTXv7WXw9k0N/RveVdmShDh64uCIySO+Ey2cIZLT4ElJkGAfdh/swbGuhRNHbvM9j5sHK+Qd911bQZz/Flq3nqg+UPLCGBKtrlBXSoa6LjhZ0dZYlyNv3mR2uwsOhweCVHbDiDRlPc2kODLPB1LY4P2ARIfmlYriYqI25fF4h6rJEja7S1otOBEeHu45HmdCsKKnuphaftyCOHJd8M3GtYfbd+TTSkwDiMf5QBxGqio+Bk6HE8syk2Jg2u2YcyGkgTRu0Qa1FLwGfNrgJKA4pqlBzdSy2vdZzRmR4oamjcNhZp0Lw2ZP8JG8rsyn2bypwbEq3D/e8od//ke4yZGC7XcMh11sTVAshX1ZV6o7WkeaIKrt5Zx3hBAIznZiJZs5+vRYuAjRTP2ugzCD4mkEtS5HPRDsvWFFweGbZTlGCQQCTRxefd+tmqWK0g9K/XJZmzHKDI1jY2Ap3esr1SgQYoy64Aw8aA9T/yTAbk8fbu1OFnk67D3JJH4l6k46cea8x1LEvL59FZXnhcPjibwWkt/QJkN805p1f45fT8hK/5A/dT8YJdOr4HE0J08cI6CLz1pvE/mV4infFrK/UlB759aFiHoujX3Xr33io4FU/4S8za3i6Gdn1fPgTcNw1FosaSgmjyh4tSin5v3TbI9ieY5nXDTO9gnNFM51taW0852ZrkKQgSkpSQLBedYlczpZMGfLxhFjMXyODxBSb5a9p9HQeL6qKm4UfFJctIRj5+zv4YMyDJEhOdIgNm540zQJQs6ZVYKNfMcjF9sttVRKWfEeYrIPnhO7grnNhLgBpTI4Ry0LIxEVWJcV7+zCW5fK68d7rp49Z/ADUhvXV1eclpXcMmuE+ZC5ev49lvxAXg6Mmw04zzIvRB94d3MLTDQNXOy3rKdCbjNNKyLBOPnVOPrRB47HR2prTDFxOj7y7s0bxnFkDFviuEOCt2TunGnryjovTONIKZXSHA0hDsPT92l7cYlKYJkL89GCQspSaGW1ByOZmk84B8sxs+aMVthu9jgR5tORvJ4oeeXy5ZXJCLznX/7pn/B2viVtEoM61HdEePCo2FW1FaPf1nWmOeFYlE3c4mvABc8QhegTnmDmaxFO95WTj8gecKVHCFZC6LnyYhmqCl1aA06VKFZk0vnK2GX7Coj47qvFpBSthwI3pazZLodNKKWnXTVT9osXXHJPus/WL02CIN6uiU+Ng5hB23yQttMp1Zw2TcVCm88zmPSL7tlu2PpRwnvSEHtHNpJSZBiGLqUoT5d6lV+HB2YX3KeL4nlX37rYVJVvI9e6HOEsajs3Wufl3vn3e/r3Xz1bIqZKFrEwAex70jdraLPlPk8xUXAOEzlXe209tMD7p8izinUnVQo4/ySnMBvP2UnQx9d+BWutmB0k2x6rVUumgdyDOnnyaE1TZhoz9bSSqxUmAxIKLkZC1P4GN2KFmFIPj2GEnVfbBfZloPOeOJgROk4Bl2p/8zRwC46JtsIgE0NMHA4zVMXryv39vb0mCikOlLHg1xnVzJAiwzjgpFJyRuvJujHJNrK3hrjAZtxyONwTdhdcXF7ycDhxcXmF94FxGsg+U8qBGEean4jqGZ9f8Prte4aLiXGZn3IGJI0klMf7GYA1ryzzI/vLKza7PVobD4+PFD3hdy95vH+g1cJmsyfERNXCxe6Kx9MRESFXGx/n0xFwzGUlbTY2WoodbR4PCi7hfcKFRqxA8iyVzj/LtkfNlu+5lJVx2lPVktLn0wnyQtgObMYJFIbrZ3z5x3+IBkg0fLd2gfn1qtoH+ExmQSoNb/x8d2Iw/T1CYXSe5CInzYQi5KVxc8xISIyiiCsMLoJXu/ipsrZKkICvFsIsKngXcRJRbK9lO++uzq90qKZd/twTd6/7hAuAOUtqsQW8iJq/07lfIQ5Ln47s7yXi7UHUFOeV4CvBG9ffjn32tzwfYVptnKMFVU21IPR9Wh9fxcOw31jmiQdDG7unvAIjtv8aPLAnhdlTmbEfe5qo1ThZ9vL14tW/kWcbxl+xB/0bv708fYnzzz0v8+F8sdOn0VSeFoH2E5qe/xx9ND0TSxXUKa0YLdU5D6HhvT6p/+04Yi+ioahtt9XUMvnEiTGagGGINPX9TWvdSxgi45QZhoUlFkqpnYUE0ndarieqnN9M334j5cmzbsvUfiU6/50E8GrIYS+IS/Y6lIp2+oWTzDrPOC3k05GaDcGyLBk0E6SynQLVB0QqaMF53zVOdMpGZUgDTWZL0HaOMW3w3vN4nHn27JrH4wMXF5ekMOElsJbMWs7Bvqa/evbiOSFGxt0R3yrLciT4l2SB07IyDBGPXR93lxal9e7uhvn+HdvNwDLZgynEAZVISCO7q0vmVYDIui6As7zPmqkN0rCjZuH4eNsxxonN/iVD2JM7L76sK6fHE6fDsU8PFWkFLcWW1iXjY2HcWO4CWQhpYhgHhnEibLYwjtzOR64uL3icb+1Yc97h+L5CoT2Z8rUZ8qk64X65ZycO1yJKIw7KdgqcVgfVmWTiVDnECsFSy3VR4uBwQQhBEAlP13JbDfdDmYPSr/Ku2nqB5iitUZolXFvX1JOqnuQcrRcXkwoZPbnvfrsp3mFhJOLl6SJ57ktE7FLoQ8D7Zmw+cUg1Z4B0d0uR84tjli3tqxijWny7yIfwNGae5VftzOXTM9Lov/uf/0FeyLMq3i4b/Rt1/qA9/Vz+jYL1qxSLc5E6/+Pct5fJ8+K/nplhKk85dmcctaqYkVStY7Lq1u8iXc8l2lXHBTLmR/Pi6PkEaK4QAaddp2VPB+md2Rl326qQW6WUinPGYFf1aAvIU/ADhuCJgWE0PnnRYhdOW3uZ3aevO0s2Y6x2D4xK6BqyjivpPjG8EsrKRiIpBZy3Maa1CZXKPN/zNlf2ecs+bSktIuW8p4MQlKQQXcJvAqV4jKqZWddT7zLN1xaisBl3NN3YGOXMBZB8YD0defPmG66vr41bPlU2mz358R7BouPHaPTb/f6CtSqXVy94uPmKZXmgLDN5zQzjyLocCeOA+IQLiVYa715/wTQmhuma7cWe+XhPmrZs9peM45Z5LizLgiMTgrCcjqiatcoFz1orUmYcwuB3hLjpI/YRbZ7lOHN7c0ct3W1RDRroWjHzvEuWejNYOvZSVuJmYHuxg9YIIeGnidt5poyBZ88uiDOsy5E1r2ZT8x0RICBqr3O34lJa4eROJJeIbsInSCLsmmPOjuV0HlGEUiEXZW2eWOxa7r1/ovRaDmPXJ7pOWGkV11bTuzVrHCzrUSm1kGvpQnM1Auo5+kztM4ET1rKCVgNtYvhs7cG6FvNn3Xntch/7GPfVioMYBSR3X2+fwApQHK5ZF4WCc8HkWM1UDK1pP4QFvOvZEJSn7k0cHZE+oPr/PRjyv2eEPBcc6R5T1/9Q3fPU59qzMO28L5Nf0W48jXjQW9H+h/zV7q7THfph8Lwn7PaIX4lYP181NIDU7neki0/pY62iVIIaASBLg9xAPGVoTwVLxN6Arr8MQQK+r9MMoStPySzaHDGkp1G1t4ZIbUTvGaeB6kDWLvqr9vvUpngxGYjH9mC0TnGtFSnmf0M6O19MKOhHx1pg0HMQiBXCgv09jm3l3XLD43JgHC9pbSXgCDEyjBPDaM+2nI84HVEtrHllXiLLbHaRaRoRMeZZDA5hQZuRHeK4YzM9I69bpmlkmnb4YctaZ8boDX1DYFlXduNECgHnhMFfcv/+Kyzn03GczTu5Lie0VF6+eIW2zDevvyDS+OjVD7h8/pIhei43O3bbod9pGvOSCU5Yl4X1tFqwxpAoS6Osx6fDkAuJ0JTjw4GNBqZNsCuneJ5dv+Tu4YHkA6eSKXmlFKX5fvHDOv55Xbi8vjK8t3i2u4n9B894QPiDn/4Lvjl9DUNg0kSKjtOaqV1CI62grXS9ouuSIfDOIgJPbSH5REj9iFRg3Trum3AqFhxS1NMwZ4aE3imVZpFi7VuFemkNKQWyjWChj2qW8+BAPLkV1rpat+8E19UCWtVyUdX2Vtoxwtovfc4FVE1/Jpw1XtYB195InJHlXuxaHgffxeIV70yuoc0znxra0pN43ftgv1/XoHlvyVDSW8umQIu25HfhW2V+pdtZvmMBa7+6c2/wrVr2r/7Ta1L/b+dLBU9XEPtzfku1ONeup87sryz79a9+jf5bWiE8ryr7EhWLcW8d4KZnaKGYaN/+3YgFLjuKKM1bxXcdEy3e2xMJOprkLPhv9iYtSlkXhtTx2mrjXynmzTRTdiD4SvH1SRMjBXS1cF7JHi2tF9f+1G5i1o+QaVgkmSltPGkQ1rExp8qoAZvMhZrtw1GorFii0rIu7McRoxl4hpigd1ziRkPNqNLW9tTCb6YttRa2uwtUG6fDiZQ2bDYbtFZiiKzrbL5QP/Dq4+9xqsp6euT0cEPOmWEYyUVoIrjoSATWpfU3q3XMKW44no4c55lNikzTxO39DYfDI8+vP+DZ9Qv2F3vWeWYaR9bmjcasMEbh9HBLrRUfRxBhPi20MpPLjGcghA3BD6xrIw0TKSVqroBnnCbmkyFiPvjwQ3i453R/bxz4ITLPM0Eiy7wS00jNK04C23HLGD0yJP7i9Rf84Zd/xuv5HW70VCK+mURiXSpVV9NJOxOoEhy148C92lhXSyW7lZQ80QVcdWyrp6iyPopdrJ0BAkrrKfT9QdZaV02Jo9ViJu5icgejH5enEOVabDSragGyzjvUuSdBqy3PrVjZe9wuhyLSlY125M8to64RnTdCcf+8eVsUE0Ji8I5hcqRRCB5ismIlBFoz5hct9oPCeQVk6B8fAimafaz1H7d8DPc0CTmxxmVdC0P+q2usf/2f/14axblteuqczhsoNWtEU2stVfpC+gwywn1btc/SgfNJVUwZbvox/ys/14qU1bvzCp9e1OgCPGxEc4bKbTQsVc3CAbS3oYp0rZbtKGrp9ogAzZ8lFWpGn54j2WpD2jmsw4Ihci7kbtWI0ZJcRBt5zazZ0qNFfNe8dHM6gDo0g/bcO3Pq299Fug+riVo4SG/VtRfvYfTMqSLBwlxDx61oNRNscpFduGRIE8EFWmvstzsCQsmm47KCCNF7lnV5WrRut9v+4BhY5sLFfs9m3NOaFezL3QUhBj5+9Qlv377hcLzj/e1bPv7BX+NWhePDvanMtTJOeyQGXAiMceqi0MbV5UeAYxg9dw+VmDbsry6JKRKCZ5i2vHz1CbuLS0L0tOKp0eOz2q6xNd6//4btZsOz60vu708spwO+Y1xCGEluQ2mVw/GBcdqgRI7HI+O0p7SVfCzMc+HDVy949vKK4+M9IUZqqTze3UEr+L0zD2BeubtZefbhC9LFnv12JITIm+Mt7/IdJVRS8LQ4UIrZlNQpLgZcOx+bCmeVtXRDt6eRECKV4CFYs83YHKfcCLP2lYa5A3JVfBFLqEKMHxatADk8rRaj/6pQiz2YS1bKUlizWrfWl8WuC4XxlmBvW2J7/doZ8ikWOYizQvK0Cw5CGiIpGT1X1DovFwIxOMYUGYZGTIUUIQ6Kl2gBHc20Y9rCk8bzHHsYXCSE8HTprNKo2owIGywq8byKqrWRJlvdfOcCZifWc0GhV/BvC4xz0q0O9mLYlKkdPNFZ7X2mtWOmsyeVAHQzNF313mzX9e3t8TxtP90a+yhFhw0qLvQuyinVwVKNCBp8IA2CD9VMqGKn5nU5Jy/3y0iF0kdUdQHwfU4XxAeLVaey5sySZ1qwwllqZllX8rqg1armeSV5VknjWg/787SuEzO1hrMrbjP0jo2Q7al7zArHZG9qlYpWC2BwgGuR1rMAhjQxTRdMMTLUjoSpzYSR/fVupbHm9VeOEQPDMHbGVsPt+pu3rFxfPQPnSePENE0077j+8BXv3rzm9Zs34EbGaexq/mLxZy6SYrSuVJRlXdhdvmC33VC1WEFJG2ouqDNygzjP8xcfsru4JI2jdR/lPY/HW9AR9cqyHrm6vmbaXHB3e4NqYRgHVCNSB0qtPM5H4hAYxw0Ob5KQBPPs0K4DjClw8WzP7c0t68F4X61lWjkRvLfXZ56R5vBTZDkcmMYt08cfMd++5pvTN1TNeOziBw7R0FcL9ib13kSh4gKq2XZSYm4BqeVJRR+9J9aC+oKPgvMV8RkNAQ2wiqI4e9i3iiLWOdOertd6zndQ+3CU1eCErZyDSbozpvXDlc2BVlCo+BjRakTZUoyegdrnXKKNhNNmMqFsghQhBbNXIdgDPJz1ihYJGKJ9pkVWSwRToarQ1BM09KZDelKY2PLsyalznrbMMP4t3Ud7BupZq/ldC9jZ9d6X7TwVEte7J8B1nAZWgKSrWu3IJ0+4ZAX7NXL2Rf3rR4Jvv65hRb7tZp4udhgpAOfMztGX5CKdx7R6jFPYiMkT03npZlDBVqqdwMX48dosBKE1hzZ7E5oiRJ9cAjEluyythePxZAeGmll6YfDeRoBWrbVvq12DBKG0Yhyo/oa066IVVcOUOBvDqqMVE+5JheXBeGZUhVyooyM4SN4zDROTRj7avuDZdInUgraZJS+kIKQhGdSxaLdy2AsbQmDohFInkRBMDrDZ79htdhxPRzbjxrDDIdJU2e83bLd7vvrqS7788hvWNnN9uaOosjSlZWVzkWiqlKosa2bY7olj5HjzHhEzCKc0Ev3E6TRzdXVNqQ2fIrVV0+g19xT5tdTCbntBrcopN6bdnsPtLYd5YUiJZbVLZEqJEAJ5Xewa5pMZ7NuKeEeMgRBgXg7kZeH25i2neSavJ5w2xumZrRfWFfUDTiHPJ9J2xF+8gOVkpmxRSu6ASv0W0KdqFzjFrGj2NrdDlMdCjb3YB9hKX8UIp/0i3X+d6+8L7bkIVRuldYJDL1RSzw/vgFOhZKFlqEX6/uts0QFpzSxazn6u6Dl7wv67j6lvzxa8dHyNb4QxMm0S4xiI0eRMIQheKt5bApi5UQwzhShOLaQDNRKBKfStaJacn45/50JV6moREamH5/RgaanQWjApUv+5tZZeoX6dAtasw7K+2MY+57rV5zzO9aIK52rPEydMzkUL66IsXpyuNwHU09SMz7W0XhzlacSSXgTBzKcxeqZpwoXS6bBGnGgEJFfKLKxacaGSBlO5GxSud2fNW5tdTMOj2tBajdE9ut6xRcD4RSGMhJgY08Bymrm5eWA+rRbIUDIqDZKdkJfSF8Rz9z3qtx2l9sJuOJFGCMI4CmHwzKuSM1CVnO2pUx6VRZW6rNQlkLaeGJWaHINzfHT5Ia82HxCBeXmkrkf7ZmrEB0dygdPjsX8PO0rlDOrzg5lvMURLXlfmQbl8/gFD8GirLOvMzbs3/Pwnd1ztd1xdf8C0ueaf/6v/N5999hO+/8n3ETfQs7SY10bo19JhiDgJXF084+uvfonzMI07nj/7kJvbN6RhwNduSVPACbV54rCHVpniSIwTVYTNMPJw845aViKZw90D6pQUIyFEWxwnM/LPy4yskTBMbHYXlsHpBfJqvslWqXmhLjPjbk9pQlsOjIOnOSEMiavra4Zxh6SRGhJahcfHB47rzLycrHjm2qkNhehso4koqxZohhD3S0FGQyxHEaOKYDsrlQbO9+JnPshzl2bvFSVX+fbAVB2+9Q9Zw/y+1d5jtXbJQbfniAhDHEzALF18yllXZMlBKTgcjShCw4TdYRhoXgkDRN+7xWg1z3mDQMp5l1YtG0KcUJ0gvhgcAbEGQCE6R+6Ui7N6obWGhMaqjVRj3+ua4NX80Q1Xu6bMmWPA8FS/Vgdml5qni6HIUxf1JEF40mKZvKC2cydmM3fDOh7nwEfHONnOqFbpuhEHWqnOvhZdSua9/XhTWxTHFEkJ0gBp8rQ2sKzZEoupeGmEwZNXASeEpMTB5BjaHFI8uEprHvVGh3Xat/2S8aHTMF2w/ZlzDNExpITKyDINFGwkm+dMadmU/qEg1dC3QWEpHa8j9uE8j842GTe8UzZbYXNpoMJhlyjZcbw1xMs6Z6iOgvnLVm20VSmT/Zo9jtAaa86s3lTu2koPLB04HZanqLC8ZnLPkgxuxKczyseWu7TG/Djj/SOPdeX14cDN+/e8ff0N72/fs+YVV5SLiwt+8Js/4sd/7W/wX/83d/z0F1/yGz/8MTLtKSFQ55lNMMFmTAGpMGcj1qYY+oK6sJkmxpgo0WQs5/1Ik4E1H6l1YRh3TOOGy/0lORsu/Og967zgQ2WctqQ4krNdt/Kc7RKG5RH6kBCcdU+1oXVlPT6Cg2EIhJCYtlcc1hMhejb7PXEaGTYbnr+4Yl1uSbWwlsb7dzfc39xbbNs8sy4LtdiuSwQKggZHk7Uv1A3Wt9ZCip7RmzQHraZ4cp4qttOUqJAizWEex277U2cPurY2mgs0Z+QTrXQKq0e02poD39lZPYOx45idBPus6lkobW+/mMRG/tYYUv+5QwQnLG21hXprtAxIIAyJ6NJTYdVWKX0Zj1iaVCNSnV2lzxNIzo41Cw+nE02NXGEdGZb56FzfhUlfndjvFZM9gGOw3SStGFX3uxawf13T5Z4ujeeuCziPeb2g6dNJsnsOe8HzMTBtA2kA49VbUGbtZFbX9VEWkm0qeWtBrVMKyTHuHGmsGKFYWO9yr9itfw3FB5u1ja9dbT8mDRfc06nbRH0NxSw33jcg9zFASGlDCokxRqZpJCbbGcTg0Fa4CZnjo0IsuKGhuRGajdxhhLpa6f72TdQtGg7SVti8CAybQoyBFCYe7wqsQgnWNczzglOY/GhPU2DjJ54NV3wwvuDZ+BylcVoPyLoQXTL/IFbYynzq9hEbV0K/+Hgx9r+WleCU0uxsfXg4cH9/5Pb+wPu3t3z+izu+en3HabWA2Iv9yhfvDrx5+8Dv/Phv8d/+sz/km3cPvNpeIaeVF9OW43KyLEgN+AhHCiE55kPj+nrPOETifsO03zPnQooJiZHDYeZwWjjlTF5m9lfXTBc7lEZZT8yHB3Ke2e93lpitAi6SvF18xNvYvPaImFYyZVmejkKtNsQVnn9wxcP9Hc5dULJlNF49f0ZpjauLPeNu5FQK4+MBXR9J445lzZyODxyOK8tppa2ZUgohCCE6JFi83nkdcT7CVDEgYfMDosXeZy7g+oPdYdRSH8+aJ7tGat8PnwnI2pqJco0rA+KpzX68VFA8Tkyo7HoX5zpC6rxbsqCNZoXMCTFZJ+ddH7OTZ22ZvJ7Z+tUYYL2jCz5Qqx14Sml93WEibWmNcVSohVWaeWdz43EN3D8W7g5m/hfs4BdcIrpgXWbJ5I7haVrwIqQU2G5GUgqWCKWKC79WMnfXQoB94G2R1St678b6hdD6LZ5+7lPZR/EOUgqMm8Qw2OWxNUH8yrqsNFVi8hbf1CujQRDVloJdHzPsHMPQCNHQHYejUrJdM13Afjz+ivbDG1okOEd2revaDNEh56inYO575ysNh5NGSsIQvUVRTRNDsnFlv9uZwn5s5NevqbrgRiAJ0Zs6WRWohvetdN+l2AnbD7C78myfw2YTSD5BDqxzpYzWOpcqjC4yjYnNZkNMo7kKWuH27p4vm8k62nLgo6sXbEgohslZ8srpcIScaa2w2WxwzjqgWlZcdV1GEhAJNO9ZqnD7kHk4ZX75zXu++uo1bhjYffJD9PDAm/dveffuxC9fP/LmZuHm4cgPf+OH3M0raZhoWikE1rmwTQOuh044EVIItGAgyJgiYUj4NLAfNuYN9ZHaVirCvGbGYSRNE+Id8+OB9TQTXODFBy8pCI7AII7D8QDODOnTMNmRKAh5PuFLptYT3nvL+QyJ3cWW02nh+fMr3r01ge12N+FiYL8d2V1u7d3rPOMwghbevfuc2/sblsfKelzIa7YdW814Seahi7ZK6Zsf2606QZ0j01i14GujOY8LCV/B+WqSlw5OVAcEtciy7hSRfmSq2rrEpy/BtdjOq6vqzWfLUyH3ztvviY2lwTm0H8u0qXW7WhjieZ1j0p3kITsT1FqHZLux1ppZ6jQi1ZwKJRdytpCavNhYPg4QfabVxlobtSRKKSxLJpfzNGU6xibd3qQJzYXD4cCyzijKOHkO48w4JlwPynW/jg7s20J2Zv5ILypdKtCL1Fmjdb4htv4C9hU73itpyMQhWVS4E2pzZBW72mCLcOcC3wZk2OWlVMUXxXslDDYmupBNyzWY6l5QnFT7b176m6ihTowE4BJtExhnS9lxzXMmZRIaLmlXEjtqnSllIKWJlCLjODAOIzEmdmJPt+JWTvrA48mAeh4IyVGO9oYOo8NHR22m6ldn2YMSYXMJ2yvPZrAQXHVKSBBGhzpIKKEFLq+eMaREcgZTXF1B3ELYJDabHc5Hjksl+YZ3lSVnluOJsma0LWgp6LExTVuOD3fUemRykSAWJOJTYhivuXr2KTXAn/7sj3l9d+QHf+NvgYfvfe9TqkZ+/7/+J3z9xee4WvmXv/ya28cT28tnTJd7VIQPXnzM8fCeqnZkaURjoTvPISubYWR/eWFjrE/Umhm2W4qHmi2Cfi73+Djw4oMX4CfuHmcudhc0FS72zyn5hDY7xZc1k7OJY0OIfUntGQeHUNG6Gp4by4kMyRBDF/uR9++PlDYzjiP4yjBFrp/tSV4R79gMnpYcTTw/+cmf8+b9Ox4f7lkOmZobIXi24479boNzkN1CcT3Gz80sUnEkVAq5KbVFRDvqBlOux6KsxQTgwTlWV7uomp6t2O02FaR7CZ8gLKvti0szmZK48zW04DDVvnZqV/BWVmtrgOvJ44VlPRmY09vOSs+7sL4OasVoME49vgR8CzgX8D1Xtart+Za1QW2IFrxv/VJpa5rgM1EUqQstC9Sx68AKwdsO02tkdI7xciBX5f7xwLIcyaWyLNmsVL7nQ3zXAuZ8oWRM5/SrLB93zm7slFU9d2JWBOg3ScEqehwcaWzEsRE3HbxWhVIcOTukGUu7nVtXDETYxBaJtXhcqIgUvLdoJ+9hs4lQnAUmdK1KTIL0kTONHpFmkEQUnnuWg9JmgEpIgRSdYWuc0mphzRlTVA+Uben6lMgwJZwI+3bB85J5WG7x8UStppHwo1CDMKTIuBWGrSdX4bT0TWAwYV9ImWFSxJvBWEIjbaPJOXw1zpdEpl1gO022lA2BU5uZ15nmMoRGiBZjr9BDGAwg2LRa0RdPzoWm98zLgZpn1nxEBMYpsQ2RyxcfsfvoN3j/i/ecXOCD7/+A+7sj39zc8R//vf8luTb+y//7P+bLmwPX28D3f/QjPvvzP+MvfvYF/+7v/i4pJlJKoFsOpztyXtjuPMvhxLoshBgYQiSlyDRtKa0a437aQAs2OqXM9YcvGP1Lal55eDgByuH+kXEwisYUJ3BKDJESTgQfWI6PnWGfzc/XhDQE6loRVdI4EoeJzXZLShNOPGHcEseJh9tHYhy4vt7jvFJbJnZ0z3j1jOXhjj/7iz/n4fYeshKa6fCmMLAdduyGnaGyqdysb1jdES1ih5F+UvS92Q8Oghjxw7RZQlDteislODVpgpN+KLR9qnkG7f/XbslRoTPvHULnzas8OaXP4u4YI16sa3dyzheHmht5UXQ6azJNRK29gWir7bJb317ntloxdJDCRAoRkQgcyCWbDq0UcskEX4BG1ZWqnoaCLixz6xXBMwyJTZyIPnZEtdEnUhj53qVwe7rn5vGWYznZ300iZ9/zdypgu/3A4bFgKCXriNx5Rn/a65+/gPVfgunJ7T82nO+E0aiE1AjJEk5qrp08KvhkyTHVd9uD/cb2VAmCasS52iPQhRAj4hzTJJS1sJ6yoZdR/NCeWutzBY9eWIOJRkWEVWwhGgeLc4/eLnINoWbl/f0Bypb9ZqFeroS0NT2ZE8ZpYrPZsdtvWTVRcu1PIsGliFxUhjESx0CTyLjYQtnFBCp4PxOH2cJEndKoaAzUkKlxNhJFdISk7PYbo4Z6ofUgihAcrR88rPuAlgu5rKzLjFdsb9Eatawsj0dLhmED3sJE0+aK61c/5OKj32DxW37+83+OBM9mN/LZ55/z3/7hn/C/+l//byg43rz5krIWHu5mXn74io9++Nt8+eae3znORkaIkdgipVaWlhlCoAaHnyb2+w3RBabtnlYXpv2GtLvGD8+ICjlnLq6esTwKn/35X/L111/yeP/OOslSGYeB/dWWj7//Az746BVxHPCScEFIKXF4vDWwZjXeVQqjuRu6p9anYCO4Wl8SY+CDD6/Y7QfbcrizeNOkOXEYCCHwy89+xu3Na3704fe4PGZ+9sVniFvZpC1jiAzeMcVA8gMzO1qrRF9wtv02aq9a5JlPiej7BdGfD2B0qUcXkHZ3iAUaK9Jssd/OEqP+784ZJtuu/1bRnGLXP7qNDfC+9q8plGIFUYGlFJZsV86ov7JKoaLV1gDLnIkust9vGFNCQiRGbyZv8UgUmkY0OeYsRjFumVZzn54WaJZ0JFLR2ljzTAojw+QZXGSM9mAObujgREcIkWfbZ3zv+mO+eXjN+8c3NLea0Pa7FrBpM9iocljJpXY9Sb9qiC3mzjaEczGzyvnt/w+xc7GCPXGcL5a7qM1YWMEWlL7LGmopvwIx6yGyg8f7CSdYt9TH4jQG4qys/anrQ8Wn1lOE7VRvHUAzvrcM5LiCL7g2kOJA8MHaafE0hLWuHG5mXLljv99zcTmwvdiAs46hqcWojeNImkdEVoImpHjbpwVlnAZCdMSw4egXShsQN1DbDM6RoiDFbDdHXXFJ8QMEcWZiTUIMFqIQnAcP0QlePBEHrjG3IwEl1QldVrQsZnFSJWLHhlwsHisXh5PEbjtydbHl1Se/yfXH38cPFxzuDry9ueH6+XPaeuD9uzeUpnz22WecQ4c3my3/h7//f+TrX3zOP/qv/iHvvv4p7+/v+wetYcBGxdeBslamKTI/2rjONNFcYBgElzzDxXOcm6CYqlxa5p//wT/jF599xbxE7k+RdSnU3NiEwnRzx+uv/5jv/fArfuu3f5vnLz6kiWMth54cJdRs33Mnns32krXOdiBxFtiCs47Xh2Q+vLilNSi54Jyn1IaLgfFij65H2nrg1Ucf8fqnR07zg+3ZGsSQiD4a4TY6fHAMxVNqRGQik1FxBGeX4aZncsuZutA1WXQnSr8WO3X9Adz3sk76iqM9YZX9mRsfbHflXUWqQL/mijryefcVKjFCzRVXG1WFpWSKVqRigSvJRlXvG2g2wmyp1Ay5FSuCQfrhoYujnF38hxgMbBkax6xozaiGXlAtAwKtiBbQHkHoHJGMlxUIxGHLftriGSmlWSMjgUFGXHJMyfEwv6Xp8t0LWBoCMQnD4Hh4PLKuHVbY/Ut0idi5WP3VfyohOOu6otq/92JmHHBhHB1lFWo24qhRHwKl2JPMiWNZjVEfoscHg745gwfhRBlSYI2GR3FYAUjR/tzurGYPDodj8I3YIIbA4C292ZaWANoV5ko9KXfLid3+lt1FYtwduNgZbaA2pWnuEW6BIBOhRnCWRRiTZxwCw8bO0+JgzdDUUarZloJE1Fcwq5gFl4bKEKwQxKBmOwn2dG79zUTwXPkNgwpLzbjm8VXQkmFdSD6QhoGyrgb960eR1hq7qy0hOPYXz4njlt3+CvwANwdqzXjnubk7cjoVSgPno+2UWuO0ZH746af87r/9N/jDP/hjvvj8J7g42qiiduxpJbPbXoAP9vrEiDTPuN2yvXhGnQ9Et0H8QAvG7/fO85d/+hd88eUjf/rLAyfxbC72jNdX1Hrizf0Mh8zlkglyT6s/4bf+rcBmmijLSlNHHKYn71xTRYNnTBOtFnJeaMAmbo0x1QOORU0SEKMjl8IwjeyvnzPuL7i7v+Uvv/gJf/HFT/nlN1+CJK531xzknlYWYhwJ0ZuEJnrG6slLpLaCU4/iCZ1V56pJLhyeIJ7orWhMTTm5jMj6JAgVbXxLg6ez5k2sGoJ/wjjH87Gqj4s1dyosg+lJnRK9UV3FNYt+UyGXajsqlZ7VYGTap0aBjOsdKRLIK+TVU4PlrdpI2P+7a3iX2KA4dl3PVhAxSZEXIXnHZnAcUoGqjIPi/GoQUWBZG9tpZDNdEOa+B4825U25ILsrvBQO6+13L2AxBYJ36GQj0cPdiXku3flO10z8mwgda2Md42Zgu/OEVAi+SzGc6T5iigyjsi5QuwcrBPuwpZQQ8Yg4QszEaC2XD2YG9V2dm7WaGt/bM65US6eetolxUoRKbYWmBl7zSUEcw+AtMq3D07w3nUrOK5obtQrUxvFh4fbmkc3+wfxozpuNqM2gjSHYKOudIDWBGwneE4MyDpFGIRcx7UxVtDQcncWktveIwRasKTokCESHlx5D7xvqCiImfhX1tO5z29REqt4uWyJUH2hNWeYFY+oL282G2pqFu7bKNI6EKOy2k33wXWRKns125P7hwG57wTgkK0rfPpuoy4H/7B/8A/72/+hvUvLK/aNBDTfThIgjpcBu2hBTsEDaECh5ZjONhDSgzuGCx6cJiSP4RJDM67fv+enPv+GPfvINwwcv+eTDF0zhOf/iT/+Yt28/4weffI9Xn/6Qu9dv+MXjgYu64/U3D7z8wDFGD7V/eEOw70EXVntRCkYTcd3jak94KxFVGjFKH1MjaTOyvXhGU8ef/Mkf8wf/8g9YyoGLYUN2EzhPcMLD8p7SFoyAHXChMkRPriNrzSSXzP3QVrxYtFnA9+xGIYbEOCbi4E0DPBTuliN1XczDqhUX3JO+UlUs+LbZMSJ4IflGih7vQVu2jIPV2HVOTCDrqU/cu+AdwxDZrJ7awInFukFFxXIbzDBs65oQA7QJJ4njseBctWlmEBt/OzWYHggyxQmXwKUDqrOZyWvDe0iDsNkYvXg7BKaNicu9a8DM7eOXIMIuPSM003viHYMEIFHTlqbH717AptETwkAMiU3X7tzfHTgdM6VU2z88tWG/WsCUlAKbjWe78wxj7Grj3IW1DSWTBs84CosWSxKKVp1rtbNy9IngPd5nqhY2046UrNprs6uT76nBZrXyhEFIkyMO9kdSdfiurI9TJKZu3anaM/McQjOxZSmUVVlbYzsAWTg9ZB7uHpEmDENgLQtLWyltscVwiIZOwRH8aAcIdYgOODdby6wrrXm8j8Qw4WSmWSorSYXVrYQBUhhN91J9f5pnECvc+G6UV+M7JQKjs2Lum+t4bDWb03qyHE11jNsNzgtDcnhne5mHuxuefXSg1Yn9ZsuPfvAD/tHv/yF//ce/xQ8//YR3N7d88+bBPkwo3/v4OW+//ozff3jgJ3/5E1qulkdYISZPnhdCnCgNNkPAUfBeqAhDGhnGkaKVeHGBiqOtliXwi8+/4Y/+7Bfods/+2Qv+o//wf8HPfnbLf/H/+Md89dVX/Owvf8nf+b2/xY9//FssDwuvHyq/+aPvs91u0XzDXA5Iq4SO/a61oljWo/o+eIXBiBa1mojTCa2YPnAcEmMaSbs9aXfB4+nEF28+p7WZ7RBplxPHpVEFQtpS/YlVH8l1YW2QWsBLZvSe2XmSRLxTmnhaO5HcxMZtGKPrPkJhGCLjZuq+w8jznHk4Hbl/eOC0HFnrbNOaBMYQCck+zCkFE2fHjGfFuYxTWMQyx5dqnbAtQlrn2A145/Eu4huUrJbkxMmu+tE0WK00IJLSiOgIbUsMW1RWalWOi3lYU/K4WME1QsAyXbslLgQPbmDB8OpelVgd01AIrjGMShgbMdqfzXI4H/nq5ue8uKi82L3ESSOXFSUTomeYAqvE717AxikRYyKGgZ3fsdlsGWLi/d0Nx0fleChPvsgzSUJVSUNkmCCNhZiUYYwED6V3TK0ZcygOI2kwY7G4YhFo3puyHUtUiQnz5hFJXQMmmN6qZLu0PKFonRBjYhoDw1AoxXYCvpMtvZvshXcYa1zN819zNtxu61o3HCYsc9SirKfKEla0FEpbWcqR5tdOxIhEh/G9HAxxIjgxqFsY+gbEbCe1GSo4+Q1ZV2vlu5A3+MiQJmJMtOrRHNDWLK0oNdRnVAqeZB677Eyb1Czxes0ZOrSvVGUcN3gXO58tWQrPmjmeTpzmz7k/3fHx938Hly74m3/tE37x+Rd8/uVnfPLqQ2pVfvrTz5nzievrF/z2b/2I6+tr/viPfsLXX33Fb3zyimeXe/OWFqOR+v4GD17QZcarcrHb4YcRcb6Hz052PHGBN2/v+fO//JxvHgqf/uj7/LM/+uf8P/+bf8Hj8cgXv/yCss7U5cA//6M/YbPb8L3v/xZ3t0dKfEZtiaiP+I6JLtVM9c4ZRifEyFoyIQZysTFNisIqNvb60KmfIAHG5x/hnr2At99wcXHF9u3I8vjAs4st02JaqEUcGjc8risuFsStiMskAZFIqp6kjiGI2YVaYvDJeHExMiVjlcXYH3rTwJgntrmyKzsuL55x//DIw+MNx8NMEGEIgTEOxLghpUQclBAztRyBGS0eYbXQEpdYsyHILSuimifYRWPFjZGWoGikuUJwlRQMgJi12sHLmzCWaqb/cZzwKqw12wXcRwRlGCAGQ1kZkkoJUWyfLSBeUW9oHyXSiu2PU8I6RBErPb5RRLk7viV5x27ao7LS2mrJSANMkr57AdtMIykOFkbgA2NItovxGZEDVZV1Noqj9b02U/ugDJMnDpWUhJQaMYAW6cr5s4Ylk4ZAXo0fJb4TWVFaKYiYtcCYS8I4CSnYHs5JoBXH0S04vxDUGY7Ee0Kya6Wqeb1ckK5DstHRJU9eS99pZQozpVZaFWqxE3bJKyF6LrZ7BhIUIbdGdUbEDM7jfTB9S+9Ca1vx/rIzjaqNctJIoXU7hSNgGjQn0XApIohaURrjjiH5jvMd0aK2LO7JNK1CKyu5LNb69weGrAt1zWitOOcYBrvUeSdMaSR0O4ooHB/uePZsT3k88s0XP2Nz9YzNuOU/+J/9O/zjf/oHvH7znlcfvWB7uSN6x3baMi/Kv/wXP+cXP/+MT7/3nL/7d36PIQrr/MjjvaPmB2KwFUBp9mdIyY4kRkVw4BN+3NJw5IeZLz7/in/yh3+EjNdUzTze3/JHf/ZTk5qocH11yd//+/+A/+z//H/izVfveHb5immT+OLdWz68+pTHm1se333B6fHWOoHgiWlgWi7Yc8123HDsUosYhycNI2hfSQSaVqbtFrffo9EukJMEXmwviEFYq5J8orWRQ63kdcvKCu6xe4AthT04M2snB5thwPuAk4Q000B6v5pFZgxshkSII4PbUkoil8aUPZupMYxbps2W9XiiriczZgfr3mz0LB0csMW5QCsntFVWMUN1KbYTczvFOdtJTeMWWmI9eULYMqSJynOqvMa5I8qDoZeaMowToqaAT17ZbzakMLBq5nCCw3qwFKNOYE2+EYK99i44QuhkGddhCCQGHxCZeEJH43EyIZr658PjamA+3RJ9I8aE+a4zLtgk950L2JQCQxrsDSCO6hVl4NnVJWspZDVMyzprV+3bebdpNu5TEoYBxtHkElU8XY3RrQ2KjxmflLy4DlkzGmprQJkJIRFTwgdIURlSJHoPRNCBZQrM8wmtGR8iKXpi6GfpYFHsEkCrKd2DRLwLqIclz7bTcBkJDXERVevYnKvstxPX19eIQHIeQmVWCxd16ohECifrcrxAyzRdUR1NLufsdK/ibV2qABVtnYPvAmoyQgYfGeLANgaKU7wfaQHS4HDebE+WFg0tZx7Xhi8wtUQombqcMPrDCsUzoITBEL4VG5vKvBIc3L0rvHz1Ice7b3j95pek/TNevfqUf//3/iZ/+ZOf8vnXb/HNkZfM4ZvXOCd878LzP/17f5ePP/4ISoAy8/7da3YbIxZYGnk1WQdKHEeaM8KrE8+43aHdD/f2q7f8/Ms3fP7mPX/t3/4+dbVL3OjNKtQwxPKPf/xb/Hu/93f4v/0X/zmlFVo1AOEwBcJ+x8M7xzIfURFKrQzjhlzNubHZ7GmtsK4nHI4qnlxmxnHDkBJhTIjY4YhSIC8cbt/i8sLGOw4opZ64GEYOVWnLShoikyQKEaSiDAQvNFcZBxPAjiEQk9mGgos0zThfzIcr4Acj3KqfUBK5wrp6jqcFvO2gchrQskWbXTODC6TBpDUqxXIWcWRthOgAz1os3k9LJcbKJoFIZhgDXieSn9iMz0kxkvOJ6gISHyg68HC4Z80V5wu73QWteJI39pcfDFHeiMxNOdaZkIM5CaZmmsRkRzrfM0xVCt5FghNqMNad67h2kQhqaxKRSPQBqQ2RSmkHonhi9KxlRrXyayVzh+CIycyV9JP6oGY/2Wwip8VbxJpbmE/VFqq+o3FcNWJjqvho+621NXKx9GjvIkjGhYILFVZPbcadtyskVAqqGR8KQ0xdO5MY4ogQQZWr/YZWC7d35moP0XZGMXjk7PYPQiMRfSKFwczeEXI9QR9nfVBSqoQotOS4fnbF9fU1293UbRzVmEnF0bxRJ036Yf40EQMUWhq2Q2rGq7Gc7EK79FBWerq3XR9iUIJv5k0LgegjThXnEjAwpIiLJ0qrkB25KIs0KjCFiNdEqBjNYs20WvsuI6JVaK5fj1Bya8xr5nB85Lie+OSTH/LR1RUuDnz5+U8YNhd8+vE1v/HJC9uLqO2xzDy8UJaFNZ+oPlFqJgRlPj6wGQdOy0zab1jmmcEpcTdATCx5QWKknmY24ch8MNV1XoQYNjw+3vPqB59wff2c8Nnnhv9GOJ6O/G//d/97Tnc3PH95jfM2ym/iQNoL2+0rjvff4LRwe38LZUFVeDgcyNk8tMOQCGF6Eru6GOBJXwVDMGOzlIrmhaEs7C42POQt+3qi6szt8pZZIo1G8I6LMXIsAxpme7ANCeeFLBCiY4gRF02PRfUMQUkRk1VkUD/gN5EUB/AbShHW02prATGqR/YjpXikVmpdu0XIJpuGoi2jpVrIsjSLxlOLE/wW22ywEGVhs/mIuPsAbZHgG84NrG0gJCV1IomwUNs943iJSyNOKyIz6gYclZAqYxGDdDKzkhn60UBcQWXGuUaKpj4vrisFfLYrcVug9Y5cLTSkFcu/pJm1ybuIshKi8cdyaU+Sqe9UwLS7xM80iapqo0l0jGNkHByrFpxLeG8ETG1q1x61S4fzYqOlCD5ESusn5r78ay0zDA6tkFdo7XwgMGRIzqbjSSPGrQ8jMY1AZMLhNZiITuHu4cgwqBEiY0NCx3K4ER8GUtzgvWPtgsNhGBBfOZ4OhOTQSRm3RqD45JOX7PYbYjR9i2ijkclEhmGL890g22yPpii1ZDIHG5FrxbfcYW8mtnRinj2ziHSLhnfkYgU3+rMq2iwlXkzw5yOsrXV+2WLLeT8ia6PoisNyNRuQpg3jzqxGdv0CUQMQ+hStoDczEB8O92w3WxyNl9uRtSzko53yYnDs9zvi5oKUNtzevOX+/VtGPIfDI4eHW/wMF5vAzfGR/Xayp78Xyrra1/ee5XBgQbh4dk1eDrz9+p5lhZcvLvjo+QVff/2a3/7RJ3z66Qd8/ssP+fkvPwd11Jb5R//wv+THP/4xv/npb5LGCVlWPnp2yeXza/xRefXx9xijIGHg/uatiVGDYZDX9WiTgI8dreOJcej2tEzIAYYBt9mYC+TuHe30YCb+ccuwPBBqIraZmBKDwtqURRtlDbSU2G4SY9gzjDvSdOTh9ECrCwSxII1qhNrqqz3MzJqCxIEwjKRxohYbRXGCBI93jVkKeXHQzDbl1C6azq94r7TcSaYSCTIQvTJEx0Gl+76rGaRDQPwC/g4XLpBm+HRxCafmOa1ytOW8SyxLYF3v2G4TFLvyy5oJ0RGCsN1EYinMazGOlxhaq2GfTaUS3Mhm41i9vcdKyX3pHDpiy3yZRiOuqC4IjuDsmNaqsuQV54wkU0v57gWstQWldVd/pXbLj/dmBxrG7oWU1gV11j15bywfJ2fEh+mRxtFbbNjZZ4SNkyFCGl3Xp9R+rrULYqlWxGrrIlpv7n8nES+RKAPalJQmNtMjyiNTWglDoZQF1UCQPWjs3Zdij0LTcfnkKC2imFzhQhrbeMHV1TOGwRT93gW0ZVo2yUIcLCFIPJTlCFhxWcvRFNPemTcwrMRhRbH0F/OCBNPU9NeodFid9+YIABP3xuDxBIKPRD9RdIbW2LgJH7YMLTJsBkYG/JKpYcUlY/K7EBjGCVVDTDs1QCQ4kxxgmrTTPHP/eMP24jnBBy6jERIunn/ItNkxjBNxcwHquL99h7aMtJnT4S35dODjj36Dw/2NCZZ3G1ouiGRKNiyQyAnqyul+MYNuGFgej9C2fPLqQ/7ef/Dv83/5v/7nfPGLt7z63kf87d/7HfzQePPmPfvdlh/9tR/yySff4/rFK8pp4YcvnvH9778gPbuk+SMalO1+w+G0oeSJpsppyQiew+HEduvZbCZC8IjzxncLwb4N3ltEXhzQPHN8+w26Zi7GS9Z1hfXEFBJXZeFUMseaOaE8isAY0CGxHZ6R4kQaBnbhgmnecv94w1KO5m8MhWVRVDIuQGkNCNSWqM5CceMYUZfsPYLgWiCGRg7OPIHiqPmEEVwORL8ibqGx9qKmpGR2uHHw1GrCbOcerTCSKe0B194whoRq6g81j/hgneoYCTUQw0jOUNYD0e9A7BiwrpUYAmMQkh9w2pi74LxW4+THuOlHgIz4B1pcACFGbylKNVBLQ7TSMFR67Ax/Y5mdk4sqrYmJ3UXoaYHfrYB5b0+yGD01V+Yym+CKQgiZYQBtAbT006+3CHNn59UQ1MRxzgSmIpCCmZzdtzDxXiScac6CsehrNQBbbQ3tNMumjpwzYzI7kQ+JJgMXLjIte3bjNZVb8O8Rf+pJ3B4nG7xc2O/jMviRZT2AZEQq42jcplqE3SaxS6aPCkPp/stIlsrx9Ag0gkwGevOdSVUPNBaqnqhZiSlaDT5lmphXTFujlQA0YhzNIOLAtWZ+Oac2LmNR7t6dlfmBFLzhtUNgSJ5NujLMTmv41jCqteL8txl73kec82iMHB9vmecZaY3tlAgukpdKGnbcHWYyt7y8fklpFa+ew81rolbu3r2mVYtgy4c7gqscTkfWPDNMgcfHG0qubLcbWjkyly6pcZ7j6cg8H3DeGHGH2zvubw+c7gtxA5Ibv/XJxH/6n/xH/MPf/xccbu549eHHvPyfv+T27i3TOHJ9/THTmPjqyy/5wI/89R+95PlvfYAMibo8EqRxnI/QCtvtBaeckbLQWmNZFlRObPZXBJeA1mU/9hAKTpA04uOGdrglridKa4Sc2WH5kAcf8DlQ2y3NO4oXBtQsSOOOzXhJGkZS2CDi2G0vGdOeb958Ti7HJwnDuj4CymbnqdUbk45IrrYmcYjRWOqKVksqqsGb/KYcwReEjGpFi+DElPjqFmJn0qdBGDaedbHJRzoFBSq1ncjlHtpA8JNh0CWj7Uhzd8Tk8c2B35iubTWzufNC1Wzf01ZxIeLFsUkBLwHaauEqDDgGS1+KCu4C6gO4I8ZVt6Jl4STVPvvesS6dUKueGE0O4rGDnhdHGgZaHL57AQvRU+tszJ5abf8hFecizhdSwuwDBNO/lIJ3Ey44fFxwwYMv+GRWDqUQUkTWgHcOlUhrR7QVfOjfuNq9aZi/MLhACGrVXR2lGDyu1gXnAkOa8NKQtjKlHeI35Daw1NdoPJDiiOcS0Q1VTHYgoZDbSqkZrUIIiaYL3itTGhliJcYHnL9kzTMlV5b1wOl0z263J7iE+GBM8JqQ3MdJcdSerOuC7SLKagb3dS0mAnQCFBsfnUOCklBS6Nl42giIoWBcD0DwFWpmrjc85kdmfcZULxlrwhchium/vLff08fwFOpxOs3MS8HHLVJnHh+O5Dwb02qw1/eojtf1HVf7Pa9fv8VRefPmtR0ipJGGkdbM07muGXzg8XTi/e2XXOy3TJuRmisSAjhPigPruuJcI/SYukUCD7c3PNw98jI1AjsciU8/uOI//Lt/nS+/eeBn3xx4e3rk+faKeZlZbh/JVH7j6oK/8ekn/Fv/7m+xjY27z/+U8s1nPLx7w+lw6PutaDvOKtRcCWEEKdzdv6e2HbvttmONmz00fMTvLmkhUOcDWjNlmSnrgUQlDJNdjVslTXsWWXE6MzDgAmRXkJAZhxds0saGCh8Y0wVtLXzz5jNas++3V6HWQl4WlmVmbJVMQ7UQDOSKasHpDHpCW7JEeHGMccvSoLYTpa20lok+9wW3x3kYR0/dmZD5eDzSc1GorVCqgMxkvTPufkuAME0B5Gie4RhNRB0arSRCSJTVBOrTcEWrUNuCkhFxxDABBZVi6dzeGdZavOkqZbCRMHqWfG/dqNN+sU296fEEzb8yIoYOFQ1GTEZx2oj+19CBOR9RzazrTNWV0o5UCtLsWhejYAgJefLqOZKxiQZhTEqQFUcxBbo4nItk8UhzuDRQfUPLEbQRYusLWNsJoZbHGIfWwW0mW2i60jSw5MY4XeJbYBwHoh8JcaRK4rB4lvyGmHYEucSyHys5zyzzDSFA6ct5BGJ0FKnWVcWGxBPaYud0Nx4PB0qtqEx9r2czfgwJHzxVte/LsPGygztKa9RaKE2RVhEctRbGOPQcyErwVqBtB2aauFIPoKbGlVZprDRZOfFIK2LMJW1sXCdh0jg9Pph5PUSjd1oFoqmSc0bLShqSOR7yyvE0Mw4TVWdQSCGyu3pOzisSI5txpJaFmldaKZS1sPRgV22BEEcckaAjIW5Iw8i8rpTmzEyshcf7e/K68Hh/IAwJ8Qt5uWXaR1xK7C+3pGc7hvQLPv3wgpubC24fqyWF+8BuO/Dq1Ute/eAjgi/c/+KnHG++4vDmaw4P71HxuDgxpAGvdtmdjydKLhxPR5RA8B6DZjrSNDJOG8L1c3Ta0OaD+U3DQEqVdbEUb5wnxMhGNiyL0nzAKRzVkcLIqivr6US8HEhxtHEwjTiJuDYwhIk37z/ndPoGxfUosztqOZjrYy0QjYICmG+yQi2FmoFmTPvoNoRhINdH5kVZy0KMBS+VqtHorqjtrbSYDahGtAV7iIgYMl1WvJsBu7y25ghp6G4aR/TSWV0GA4huNIadi7Yrq4FaF/DB0oW8stby5IBAtBNjzxy/ZkE2zgqr8xFxE9oG88Jqwosju4NxyqogYhwwSgDNdoUV999Rnf4HFDDBlPWtKaXlThlVIJvOwwWIveVL0abLHgMWUyRIZgjJwjWrdShOE+JGW1YriBtQl23RGcDHjK3BBgvVCJ5pM/bUEodqYS1HRBytKA/ta/abH6DiCT4yjtBEkXCJO5qC+CkMTjwkwWXraKxgGvzPJB09yiyo2Sz0kVwzpaz2BJIAzIRQqGr44Bh2hk3WTKTLQ7yJa00O4snVs6wZ3wqSQt8Lmr7Ht0CVQEoO72bWutib3SlVgmG3m6LM9rojuI7QcxIopaGlX0m9oqWwlkqt9r/j6YFSLT3Zx4QTx9XFJaVkluOR43xCtNCmCx4e7nn2/Jpnz16Q88rlxRY/bFhnpXb7VszRHBUONtOW3faC7eVLZJjItRDc0P1ynnVeOR5nlmUmpkyUZoeT1gitsN1Gcj1y+cmPGD94yTf/6k8IW8fzZsv43dUz4mDpUMfbL1ke3/HuFz8jHx4Jw4gbfBfrZh6OC9K/7mYzsi6ZqvNTqOo4RMQ3gvfs9te46w9g2uMeH8zahYVvDN5TWsDR2HiHD8kkEXoiAbE5qnesMvF+eeT+8JrN9BtEn4xwGhOXVUjymzzbfsCX3/wJd4dMcw04seY3LPMFfgw0N6F9hFR1VAZqXWyMJHLm41OF5HeEMXJaErW8Azn1z6IRYnzMpAF2OtEYmBehVhO7Fl1tFSJW0NQpq6PDQxslQ/AmdsYFWo44P9CaUPJCiBXvBoK7wEkGVkLcmNo/LKQwmEBdV3JbEW2UOjMvB3vPOo93CaehszVsxx18BAKtmdrgtNyyGZ7jw9ClFnYE+e4F7Cl4AbPeYDRPM5jYsl09DIPDS09Bkb7kd9rZVhbIKThatVQc8b5jRbJd4wioq3ifn3LpbAfkTcEcAi6ebUuOVmaqCMFNrPWeOb9nDC8RL+a4d7ZTcz6YF7ItForBSuOIi0rUAFRai5TWN4VqZ2eVkaKFGEeG0dGWEwlPyYWqB5BClNQ5X45h2FH0EZgRsSeUnevtKlurs6LsAZE+ahbUJ4ILIIFWT/hQiN4ZpbQdwG1p1bDFyopTYZIt23jFJlwQNeJ8w0mlNqjryrocEMUAh6dMLjO1VYIfEA0cljvK8QHvQy9yK2jj3fs7hnEka+bVR6/YP7vi4eEdV1dXXO2fc3v/CH5lqY3HBaoU4rjl8sVHDLsrpnEwZX4uhOBwXoGCP43IeqIQSJKo7R6AdX7gmbyk1pn1m1/ShsDLF3vqOnFz8x7azOHtI7UV5vlEWVekZYZpgxPlcLwjHzP73YVx21VwyUJr7+8feoKUAfeO9w9QCxfPnzGOO/zVMxgGyDPkGW3m36vrap7EbGiYFKORbZ1HGU2DhzA3RYOS/MQy3/Nw+pLd9JLIyOQiwy4R3AQ4Xjz/lGFqPC6/oHCLrDP1cEvbbFFvekCVSMuNVjwiEzXf09YD0vZQI6oLTpXgE7v03B6I+Q1Lm43FryYhSn2dUgo43bKuo2WE9inCJWdFS6EWtTHR154cFfFuRNWhUhAqjgjqyKWyGU0TSvHkankDHgu0oR5NA5hMq6hg6xkVtFk3F9ymH8OAZvmR4PFuBDJIodXGcX7PZniO08HSqr5Nvf7/vYC1Hl5gJtGINyACYN0HzoSSOEuei10WcI5+E2/FpGHtd1OHVnPdtw5oc96CNDwV7+1y6b0FasbQrRfO44OzRX4zF/5abpBYUB14PH6NKfcqwY5ETwG1pWRDKpNRt9KYaQgxbVG1mXvNi9mgWgVdjKOe+tf3I8MYWZaVx8Nbit5R6iOpB3SKE4a4Yy0TKgWyEdEMa0nXuNjF1F45+ztq7XFdzoNGxCXUWWfrCSiFVlejzGXD49RsIbnqK7nMoIWonabR95TLcqLmlXkprGsjryulNeIoPBxnpk1iLVixEUjRxKGqShi2jJtrTkvj8jKxf/YCiZHiIsPuGQtHODYyMzGNLPPKF5//nJv4FR+/+pD95QtcSkhIaGfB6dRgXRiC8ctqXVmzsndb2rowXF2j0dFu77h7/UujlKTEw+nBAI210XJlO42sS+Hu/pbT8QGnmXEwa1Kh7x0lURZsnBTHvCx4Z5dw54QUEtPuAomedrxBckZafuK/W6apRfCBohmz9HhPaY29RIJAqwvFOyRuOerM6eYNJT/gvWMfNqQ0QmqUcctxGZnaFWFsHBZPbgfaek8+XoJEVEJn0Ju8QDGa8Gk+Qq2EcGHBF94Ali5VyiHg2eJQG/8RnEtmvfMW3ksTymrC8FaD/Z3UPb0XnYRucnfUunCaD5BGe5g20PUINeLcYIgdKlO065I0C6KmKeI8rS7ksgDJwKBiE5uRnAPCFi+XOJfB2biorXRhq4EfLPGsUOrMaSmkcEkIkZZ/DRmFajWZw9kOIIPxg1wzEinndN3c00Si8Y28dUpy7jbKauZiALEoKBVvBU4KwUW7XoqlC+GHTrTwxsYOEe+t63NSUZ+R1sjlgLBSGzwevsHvCvjEsiq1dmd8q+T1xNpqT+VZTavmrDhVFxjcllqbqb2peDcSnGFwnQgxXkK4Z20D63KyApZGkEjt30wr8o7mK2h5yppEHZa+NqD+/A0zs24tXU7hzIhLw8IdOsiutoKWTK09mrSaUHHRxhQHggy0ZWUt2f57KR3BopRiMTchJoIPuBBJKeGHwLwsxMk9gQ/Dxha7aXfF/vkHvPz4+4yTyRLmdeXu9oH9xQUvXn1AC9/w9d3M5z//gjq/5YPLER08f/nu58Q0sp12xDQxXj5j3FyREdz0gtEvpNAYhgtCGHHBk/Mjg7xEhoSPiVZg1YXj6UQcR3voucayPvL29TccD/dst1su9leEbuMK3i7fFFvCh2TCSyeN7WaEZkeR4CNpGCEmqgr+eIL1ZNmbWPhH8Lb3ylrRtpJCQIsdy5tCUIc4O++PpbAAKpFHvSMfbvEhMriRqV4aoVUXYhjsdWei8QzJUMqJfHpAGVGXqKwmWWj6FDjbZGGtJyiPjGGL1ICLoJjZORf7jFl2q+80V4M6agiUDDkoebVkKFUlhpHoR+hWPBFbdZSiND1BuyeNG1RHhNqta/SwjoWH1tjtd2bXU/PgNs3krLZnXa3/arJAJ7Q6CbbH8o40TJTV0DHKSi6ZkleaFixPIOMl05g51XsGt0Hir2ElalpwEow93oz66MPaeUBmyq6+C9P8ikZTl4uzTDvTiWRMwRZsXDyHaao3NXrTvqgze4IPpm53mvA4hhBxwa4YiKUcSVBaDbTiKXpCS6DpA4cgXMQLinjWsppimUzRlVpWgnSFtChSI9GPRFXEezQIy3qkNYgEY5nXhg8WpluyBYZEB6vOuHKwJ0QLdp1xDq8R9XZhUe/7SCpU+pgsldg55uLoCBWIxpfGuurWr1La5Smmk1nbYk79Vkl4ajOb1BB3tHxAozBs9uQlI7GxGzfkJaPO+E8iQm0r+VSIyWQFc8kgHu8GXl6/4Ic/eMX3PvkhcZiQcWA7bbmOI4JSaiUjbJbMDz6Bh/uVn//pG97+4is+fDby8asL5odb5pjYXlxS68rh4Q3HFnn24hWSBnyqjHFkux16jkAw0u5+j6QtzyRw/+ZLTo8PpGmDlkxeFqIomzSQ3DNcL1iqJh9u2khpxKfRjiVaCT4xpo0Jf5cZpTFME9NgQmYNAakbtByph3uzr7WZNa/2Hkyeki3jsKHsvCdQeCgLO3F4PNs48aCNlcasiVzvebz7HJe3jPEO7zaGUiqZWp3hpjXgdaRVpRxOCCdIgmo2zy6OvBaW4wx1pemRUpXSEqOOqAvW+ftCcFCyyRBUHdInGRcDeBg2PdIwegtXFmEaJ0KIZLtM4SRa1oA4as2snBAXjTNWCq1EpBzwLiHimE8rLph9qbWFUh9oatGGSu2ZlYBzSItPwEljK2e0BZxP5t+VimpPAlO14pUMkW0/lqk84N2vYeb2na8kiH1hrABYcrXD+ZUaKrnYKNnUZlmHw/kE6qlYarbBXO0iETw9KaeZ79AbrE2cYZbFBaTFDlczrr241sdRazmdOIraEwKpqJ6oFdasRN2Y3IJMZ0maIlgawSe0dUGn93YhkZHWGsFb1FPJZkh2Tsx7hgXxpuQQVfJ8hxRHY2N/x1rBW5x7ECtaqKCdwCFgl1nvLf1IM7Ut5FKM6eTUkDnBtnyGhTHVstPaQ0yraXKkUXUmt5nivLHDYuqvS2Z3seF4gvuHIz7YUnvaTKQ4MG42+DDScFQqtSpx2rDdXhG95/rVh1xcm1/u8XTi9uHGxo4QmbY7Xr78kPGHz9hf3HEscPP2ll++O/LZZ6/Z7YTNIAxpw7ubhT/7yU948dFLLj74AeISEhrPX1xT8tFO59Gx27zAbz5Ahgvi1SXu8hVMe5a/+CO+/uJzTvPJeGXTvhuIG2tPUPfOs7t4QRgGfEjEGEEhW4gDqo35cLCRsFVbh6ii64n24OxBUxrrciTnAiUTvacSqDUTY6RVxSf7vpUlM4ijgAVPsOC8EKhEAlU8q95xO/+EtLwENTRNK5VSCyF5aBHHiGgxFPlsr4W4DgrFGWM+z9Rlxol5DFv/fvsWCDLgoxAVGp5lsY9wKWZ1qlINAR0aaRDExU5l8fjUE8VK7Q2Bt3xJb6uQ2iolH0EcrRZKsSSlXJUQbEXD6R2DBqqeTKoT4pNJW7WQ2wEnCdd8P+rZgUS1seYjaKZVZ7tmbE2lPRTTiUOlB+RSUcVcNt+1gAXpQR7iDIbmknmZ3IKqo6m303Q0WUXTjNAI0QIACAOlGi+drnspzVOrJWuLU1PdOpNXgDncS7M4KO/8E1FC1fRfvYN/uqDYKG5esoZjWQ2TY9Hkfc6GftqlEye8BceKscdc8KZP8Y1WF9Y8M58WQnTUdiLg8WFgu3kBPOeuvSEvD+S62LJW7Tpqo6GNzYpDA7Rm+z4nQjhfUqmUcrCna/EsrCQdaL4RPH08D52Bbp2utmrYGBZOrZHEE6oiGO7aDZ7AjnG6YHvVePmxQHPkWqjNtHspJpMdhIFpv2d3cQXO4bqE4urZHpcS6gM+Cof5wGba8vD4iBsan3/1DeP2grTZ8pu//dtstjv+8eGIO3jm1Y4u+3SBdxt++NHHbPYXXD2/YnctfP/VFZvdhtPjkXXOdvO7/BiuPoWw6dofSBcv2Vy+MvmKv4dauXs8dTw2eE4cHg8EHygtE8YNTYWQAptxsg9fsctXjAGNIzF4xos9ddxRhtG0eOtiHy4XcWKAzdYqrajthUqxD1j9VgAbsVDZIJm5FEJTRslM2EpCRFnKPc018hpoJeDq+LQrDXFLLoVSA1VX9HRr+wQRW3Cr6QjLapQJcc78lGM0bI5bKO1ky/Wg+BIJPrKUAhqMFpwN89TajIoBQF0ft700WjVqSZNGk4bzhgSPMZHXxrzeIxKgVVSUSgSt5HUmpUitSs6LFcSwwUcr8BZkXdH5kaJHK3bYbrF1yGEIiVYd62q+6Vorldn+bG4E6IieHqRLRi2B5zsWsNj5QHh7g3lDyorztrCupn0KGJq3SkXEEXwixQRtMouAKzQOFnLgPa3m3lYmghvwzpzwqt7Io3hz7nuDALp+ray6QjVQG1iMlBSr7iqZnK3fcs5TizH2c2v2xmwG/KuaUbXMOnQCSSzroYe/KkhFRCilseojUjITG7ZxIoULs1i0iXf5S9ZshwHnAl76tcgZy1ywdG8RGyK9177PkyenQW0GcKvVvH84R/VYx6rmJ23OJBziTMHcaiPESNFC8Y2Lqxds3M72Jkvm9P6G23dfc3i8oWZBIuwur/Bh4HS6pzZhzQWcqftrVXbbC16+fEmtH/Li2SUpDAxpxwcfXhDDwKsffMq43ZF84s3NO5b1xMX1c5xP/M7/+G/zr/7JfwXVcff+yLBd+PA3f8yLj/8Gp/WGcVN59fIF4/6S9OyavM+ktbGeKgzXNAQ/7PGusd7PzPnEYy28uXnP+vCWfLghOBNPupCIm5Hr55MJQVEoqzHQ5oasMxJszKrZMhpTjJbAPW4YQkKzJViVvOBbxblESp71dE/JC61laLVrm6xwlVo6UaEHzVSYcERplsztArdqVIgUlTUfqG2gZbPqqAZa9UZjIVJyo1EtTSmfwzhMYtB0oRQ7dNVsmOcaEsPgjJkXV9CFKANSBSWwNIdUj3emecv50agozZoCE5FaV486alk67tzbZbwZBVZlNQZ9KzgSwTsaFanR+HRLJWNeSoOPWodnB7YM6hmGa3yJqNyjzdQCtMZSFsRfgviO/45Ih4n2BEVDWYntx80PGynl16FR+GT4C5Nf2lnaBbxPKMUkAer6rgfA3Pjej4gkjLYQcGoUisoCWJHzIRL8hPfBhHkhk10k+EwtztKTxexFqHmuqvNY+GzsupRGFcPhtNaMDU+j1ntMX5JtVFIbVwFKXgDXrfoLPhgXrBRD/ij9CiUN7bluIgMpPDPGE5XB7wl+z+l4g4sm2UCs7W1qlAL6bsKsFFZE7WnoEInk0qgtk2uGGmjeUNaqNm46ErVlqmbElSer1Wb7ARebT9inl7BuWQ6FXN6xrDNSQMqKSxN+rOwvEsM4cnN7x7I+8u71G1wIfPLD3+Tq+mOunr8kDonb23ekcWL37JqH0yNvvv6c/e4Z43Zn6vbNiI+J3faSNFjW4cPjEXGRv/Xv/U+gZf7i//X7phX0z9CwRWJjOwovnl+RYmA+Zo7lDjdOCJ50eUHcXuCnPfjYL3+e9faBcnjk7vY9QcBvLjk+3hHaEVdmlrZQasGLx4eI86Hzx6J1UeL7g89zOh4oMnP94kdcvPwE9gMSBmQ2zVxFqc4kHZoLeZ2BRvTmdXUs5qzQRgyecn6PO2h5JXtQEQJCahNJGsUVnM8gR0RGkD6yV0PJaEuoOnI5IdV2XyV3n3GTPlEI0YeObc40BzFGYkzdzVIRPTJMG3wKSBhYD4myOrwfwNlIWduMdFZ9qVZgWonkmqlFiGE0G18QFIsy9Dr0KyFdNmV6Q1Vzd+RsanxxSvQBrR6CFaJaC6orrc32+XG23Jdm6VtrfsS7SsMDHicjKUyYnMlCqGu1nbhWoVF7Huh3LGDOR2JI9kH2FpyhGDIHiThtiI6UMlPI4D0+jDgZrUtyvqNoLBYsl2bEAOfNiO0NIuejKbS9H2httiUj1RZ9ONbSaHWlaMNFR5IR7yYQpQbT0pRqCTzrquZJVIeIzdfGIBJDjTQxXUyu5HzCBcxBEIq9GGoBBqU01loJsTHPR8p2IXgTGlrL7zonTUyvIq2jhLN5LInd89bHVzG9lnO2s8nZ0qNFMw07QYPRBIzU3Z66xWkTub56yRBHypyZjzPz3Xv2wbGXC8Y0ElyklEZZRlJo7J694v72S376+WeUZWYaE7/7e3+HaXPJzd0t33zzc372s3/F9uKa3/6d3yUGzxdffEPTlY9efYJ3A6U25rUwtwNpLNw/HJjGyRLUm1pk1trYvviU3au33Hz9S8aPPub59z5k8ywxxeeIZnKtHB7ueZhfW2rPMLHbvWSzbVy8jCQ8LgXSdsfF9ce49YD4wtdffYXS2G2f0UpG80KTahy1qixL7TkKmWVZ8XFgf3FJa5VhGGmjvS5h2uDSgLaIno60u3vmZUFb7k6MRpz2SIiU08EeKlI6vVWpzRDa3jfWskLJlh7VCqPtJ5hlQyLSyj2N0n2NAWQmBmsCzLcc8bJF6z2trTivtDbT6korjtYGWi9qwZkA+3SwYrhznhSiuUUkozSiRILbsWhgdUIpggtb0Jmmjz15SWwc66TWoplcI5VsWQUBW+o7Dy4gaqJFJ2r6TjzECM1TuvD0dMooDlzqIvPQL6kPKIs5CtTgkc55pFXmeSb4hujeds8YDBE8rULWFftwApiSAPk1dmA2LlrIq2JPllIyzgned/GgKMF5cm00Zrxamy3icS6gXmjdBH4OxFYZESbEGREgJUfVirjFVLqyUqn2xKiGjq7FUyokTPpQpfPKvMO6zO43FCXrSq4LlkvpurDUWZRWhZptySs9bTnGgNeK73ogbc7+Ps1RMqCFh4d3hMtoF9daCRLYhB1opLWVWjNVba8AxkHSauJExXL+RKWvBuz0bbSJAfUZF7Lp6cTw163NiKu8fP5DtsMFx/kNh8M7XAskf0EcBVxj1dmggRjaJG0HnARu3r5hyZ5Pf/PHxhpfVn7x+Wfc399Q1oUf/OjHfO/TH3Bze+RP//QP0SZMmxf87r/zN1HnCSniVCi5sd1Z9uGpnHj35q09VdUhpbLMC0UVN23Zf/AJSw388usbnq+VZ5cD20E4zfekFNjuEiltUDcwbEfjvbcK+UTLDo0Qnl2w43u0YcBtn5EPD5ScTVeUbce6zDOPD3e0wwPH4yObyXIbWlmYD/fENOJ9ZrOZ2F5eMW0GtM44zei6knPpJvhuk6v2YQtpIO6v7L1RC6IFrSuiQsWkLr4pm+AJPW1obRUVYXKOBeHEAMuKZMdS7qBWZNgh0Q5ATj2ju4C4kktAdMViuA1FXWsht2oaLRftmLAKaNez+cgwRFIwXlbVgmolhgmNgvcDuUJwBdWHfrG1dCvBwnNEGjTpeybf9VoFaWphIWp6+egj0W2ty9JAWSvRDahW1nzP4XDLUo6sxXA74irIYuNrf9d7b9IN7xKlHQBT9ZsawRP9BtFA1kxxEPzaiTS1TzS/hhK/EECV5D2u2TJOpKG19bbREnl8gqFCngtVlOqMHiFdOmB3uYg2j9JMnhE84pwZSUXNEB4UdRYCkJwphVULIkqpnrKa17KIkS7UCybiw7ogki2CAwiRXO8Ng2M/gdqUks1v2ZrN3g0PUqlUtKwIJuSrxXWTutEhHg63qFSmZIpsEU8KCSdbcg7UciQXparD+dqBcBaY2ho9ZcZ37r696UU8wUFME9MmIn6mcU9eM7vtNZvtMx7u3/Jw9wXjEHBuZByvaNXTdGVud2htTLK3N9tgGOqHh1ueXb9k/8mWL978grdffYEuB8pS+OT7P+CDj37AL3/xGX/4T3+fabqAYc+HH36Pv/W7v8sXX/yCH/7Gb/Du9h0pJmoVcs4sq+XzHQ7WhR0PR3b7PSwL+Xji3dev2e+2VBc4riv17TuWdYOwcrmbmOd3bLeWszntArs4st3tu1h1JQwb2rrQ8opsd8TWeDHuuH33jnU+sNy8Ji8LrRUcsN9vmUWZndj7KHhqbtyf7hi3GafKNIxMux3OO8q64Cj4OBGvLvHLkfzYyMtMWWaaVmo54nCM4444bm2JXxq1nKBV1lzQYGEprZg2qtSG84bBjsUzyJY1K2WZmZcTJR/RNrDnFeqM2x/DBuc+oIQdtZ1o7YaqD3i/ImRKzlZUndoBgA159Zwe7eHnNBEuiwk/e7fnEILfUFtA8YiPlOCg3VDqjcXIBZM4xRiQJt8CBRyA7YjPQnS0oLLifOrqfUFksPQtAe8SS7lhWW9Y8sIwKj5kKzxOiH5iiBsb88WQ9M5dWWPk7MimVNurdzmMrw31DqVQtCDe8ha+cwGzMahSnV3FUG8qc85dllkCYvBMwyXzsrIsM8qJEExdK06MJVYaWs2SVF2mtkL0g/3+RRFvS1Ak4eNAYAXov4cB+4oquhgw0cd+VLAjDiFE09u0ilMYwxYnK5mjfUPo2iqxJXtr58zETG4FKdY5tbaQ10btVM8UR2qLjBo5yAPaFsZwRXAjLkZqEdABqZW8LJSmuLQSxJLE8WZcb01Ys5Vy7xw0W4CmNDDERvRKSgMSPiZd7rm/P/HFFz/HOZhSsqQiVprOEOypFfslktIoWghizPSPPnrFzc0Nn3/5BcnB1cUH1Fb58MMPuL95y5/8s39Kq5n9/oL8/yHt35otybLrTOyb6+rue+9ziYi81IUiCIBokWxro/WDZLTWg/619EY9qM1oErslkWw2QJAAAVRlZUZkRJyzb+7rrofpEeATaVZVZvFSmRkR55zty9ccc4xvDM9/94//KX/6Z/9H/tW/+n/yf/4//Qv+81//DW/fvuV8eaV3YQqLQia7opE+ff6EDHj/8QPltpJvG+fPZ2oumOMjcY4glpQ70zxRhsHHA1s1mK2xlhu9BVIG4yxP7gStgwU7RaQK03xkhJnHGLl8/sQYjTv7LWxfthzeHbi8vFDKRqkZGVXBm+kGy8IcDoTwgDk9grfUbYMxoGQ6kPLG5fKKGZWBvtyMUUE6rZ90ne/VDGuMIm+sHKEX8jbwDGJ19LrpwTciqRqu7cpokbRO1LZix43Zlx2q6LDW03OjN4vIgjeNUvf4jlUvWh+KwumjIiPT+mDQsdZzQ0GZMRhqG+ScVE/rqkl70eXB7J6hCcVAN69qgxqGGDwOrwRlo9txfRYUh2QdtFpo407tncm/xeIQN2NEUVCtVliBZmlyprULoBYnQcAMnPOaENCNCM6pt7TWti+3DEij1M663cjblSEKTcU03cTyB4yQrVSQRq8VYzWd3ho0NKU+xYnSM8N4QjwwTZ1y/8RWbrBV5jjj7Kxr1J4ZNFrr2jrCVamLvWkXoQnKDOq6hfQWusmIaEmC+sU6rXZaF1JeEZeUBDk00yUuMIrGMrwdePH05vWbj4IzoO1GOh1/tQWZfYWtloOSKve1qTVj0hWx2xcKSTI2qjamdWeOKk1NeyLUVjG17+Zb5RphjG6+mmHQCX6otr8zj7wpOJeJcWaeH/n46QP37RURxark3PF2pUuglosy0u0RY4zqZP1MlGeWw4m63fjpp99QEhynE210nh/eMEbjx59+VDf78YRxnlyEP/uTf8o33/2Kf/n/+L/zP/1P/1f+3b/93/nFL3/BT7/7kTEab9+8odaN1vT29Xd/+xvsl6KOXJXIYD3f/8N/QGuN86ePjJ755t07gtVAdy2dx6cDy+GJw3JkXo7qKzI64qecsFGQolGeWhK9NnJa6emKc47jd78inJ54/eFvyJ/fs24XnDV75T17c3xljpGnN9/z3a/+iHB61p/P5YU+VsywpKbjdNsS+X7H0MnbijcqR4gI0sH5qE3ruwWi7y+9UtVQbFtnwYLxDKef24beJkat9KpFMWN4aob1fsW6qKHmaBCcfub7RsudUbQaD+labjPQaNHojLExhlByhH5EhmW9FVrVrWnJnZaveIkao7IBmn5PvJnobaa2K5isNAlR/2AtQ7eqptPp+xZUpyykfSWnlnrDxUemMDPbE2PAer8xqoce2Kqw80YR+m5SbZTqmedp94zqbZKh6OzWlJNfmtpjSi2kquBPN9xutP0Ssf89D7A6NAJUSiZG7d8rtZDbRm8G6x5x+w/FWsNx0tzWNWVKuxC6foNar+qSriroChq9WddGdAvWTFS0X9Ht11njvmB0B9YOgjF078jd0Htly3dsLBgTATXCRWdxw1KH6Ji29+ux3xvH2J34GjelD8GOgLEWMV9yY3032TY6Dmc7zQ+K7VAGwQDyijEabIeKC8LIhsM001nY2qaWkZ1qIXYgzSrhomtTt/RBbeq1M165ZssSuNz+BkzGm2dSvzOGEkC2XLGjYMbKZIRsF2x0uOGZwxPvnr7n5x9/glqJceGwTNxfN57fvuG23bi8vuKniZPVcXqrlT/50z/D+wP/8l/+3/gX/5d/wf/r//2v+PUvfs2nTz9hjKXWyu1+V8bUgB9++B3bbePd8zc8PD5Rc6HOkx7uYcH5wOX8SiuJ3/72Nzw9PfDweCDOkcvaGabQ0ZgQbTBZj49OH3TUHN3FkNsgbSvr9TOjZVpOXO93ttL4xR//E9L5V3z+4S/4/NNvaNsdb5ouj8xMU7cWtXeWhwdcfKTfX2mXM6Yleq1QKr001vuF7X7FjE6w2u6tQL394bcO04dSfHddSOrAVs0B1ppJ+UqTShVDHoNUCrSMpWK7foZ6h9t9Y5rgJhctrBkC4mgtkIv66PBQh+xEY7sX3CREtDDZmshoAWdOeNORnuj1QilnanmhiwM8wwmtWZV6dsClFk8LRna0dvd6UBmN8wk7BBP2TbqimAYFYzu1XTFywtlA61W7J+ogoqjt3LT/gZxpvSjuJ3/CusY8vWUMq8/VvkWuWTtfNVettWyhW5VHms54+tn/A3hgRS5qjqNT+sbAMUQo7UztlZAHT/HXWBuxdjCFQG2d2/pK7TeKyyrid80kKvdHyyIKK0jB0HFVa6nEOIZXbLTekthhaIqwDl51N+2b09+zjVV1uG510+cCJa203hT1Kmb/Ae7BczF71rDRmvnqFg7RfB2LjQjOOVo1e0GsVqnVVamwtSd9M3YhOsFI1Nxcd8zhwKhpLyTRt5lel3U9zr6c0C1L4d460/yMi5GX8097C8ui35thadLI/Y70gh0eVyMiCedfIK+8PfyaZQ58eP8bPcTjiVYG59cLz49v+fDhvRbbTjODhj88crve+eNff0eplf/9z/8X/vt//s/4m7/5Wx5PT1yvd6Yp8vr6yul04tOnT9SqG7nDYeLpeMAZgzWNt9+9ISXNIMYw44PnuDhKLlxvV67XG1va+O5Xv8IUw9T15ZRSghBorVFLZV4W1nXldHqiCUzzgqWw3YTPLx/J641DCPj1zJ//67/m4e0vmB8e+Gb6I9bXF8r1BTuE0/M7To9veHx8x+HxCUqhjSu1CTI9st4/0XKGUiklY41jmSZyWvkitXStw8KK7NYXJQmrRtSxzmLDxOgekw0metaW2Lr+DlvfuOeZ2VRm3xWJPgx1NFK6I6IvSoZ6nkotrNvKWu7IuOOi4nW66Zr/bdrf4OyBGBzGQsorLmjy4Gv3p830fmPLoj5LM5GblsHUoZvIvsv2essDH8Ke91XkNmPoGG47rScYavpWy4Plfr/gDn53GDTEdK1QM4LtjtESuXVq3TBjEKPhdk+UcsM7xzB9t6UsjPKsnk5nqG2j9Uwfu0evCXT7lcr8ex9gvSpUDHR0EzdpkNMbau6kfKGz4e2J4CNpNEJVsH+tQq1ZNYku+9rUqCekqeejjoKMpCwiUJKr1Srz1ps69psePmZvWdF5r+o3l0EdDWsc1jpyLZRcyEVr0r7msERjTM4cEONoslG7wezOeTuEUSrm66G5q3wDjVqgWlWtmzb/DOjtohjnxbIEHTPLjssV2UsMWsUGo9tHK7RhaLVjTNegd9tJZabx86f/iPMrRgJjPOhG0urtq45OrwXE6JuZjS2fOSx/xDwdOX/+xOxPlNbJ6428DR5Pj3z48BM7A4bFH5jmI9u28d0vf4Ug/PjDb/lH/+iPSakyhtZfLYeJ9+/f8+btWy6XCyEEpmlW4bV4vBiWRctOWt2wdA6PC8fjA2ndkBgw5sjz2wc+fvzE9aybwniYNRXqDMbo76GCsfr8nHPc72fmhydq7dyvF+bgmeOEzSu3zz8jMnhzmrm//kDzlvlw4OHtN7h331K2xNPjEw8PTyxP32BOz9AN9DsjCmME+nphPb9nu72SW8Zg8EZ/zq0UTLB6azcG6/TXkN2UbK2WZLQ9A4zyrL44ynPLRCwPJpLCA8MabrmxihbOGLHUupEzONfpfdCbpkl6UybdaAnTC8FbZOxxNNNVpLeZwR1j9XDLZRB8o/a91g+j1W3lSm6DIZkyEqVnWr9rbSA6vbS+P3MmKojAdAZOTbNdtSxnI715sI3WC2ZUWl/5fP6Zw/KWwUaTO6Xf1TvZYYijVEMZDtmXLfRELSvBB6z3MAKC0OrKwNBbRIxmMUtZ9wWaRpyogd7/AKS0NWPv1zC77qGbgzkGtqaO7lQTD4cJ74/0seKKipWmCa0pMBA1R6jbvULtMJrRMtm+qbG1F9qeiao96UhXx98jmo1BvEZ0Shuqz7WuKOt9/h5jkFNRgJ/VA9PIjiEZC4aOlZ0E22echU6mtqQfUieMpKSNWkGGgtV688gITMGByaS6Ks+bjBkXrA1MwWNpmC81XQP1gTX3NWYizjD6UM9aE1w/8vbpOy4vPzEtHrvzw0T2v6ft6tbeGedqTfH0kZmnE9Pyht+9/ytO9i0dT76t2B54OnzL588vHOcZjB70PkTGgMfHR8QYrpczj49veDi+4z/85b/n3bt3wODT5w8cjhMvLx/1Q+4E+mDbNnJaCctMKXdKNZzmA0+Pj2zbhrHw9HwirxvWCS4szJPlfJoZTghRmOeJeZ5Z5pllWRR3VNTMeTgcKdsK6YaLkfnxDdePf8d33/+CSzxwmCJ1vVJb4vHhm515L0TrmJcT0z/4RrMdy4x5fEs3WpAxmsPaTpPG/M2vCdvgc97YtgtWwATPFKImIdgPL6sROQ1572XNOxmlj675+i67o1+7UeddOK/OcrkraMBNR1y+0FvVwyFnas1MojCDUrKmS8wgTo6KHuh9VO0+7YphRiqlv+o2tYOThZQKzgZKabRa9HtuFX+Vt8907pQGpSUGG6ZXjKgzXxBkWIxE3ZC3DhI0a9xXjd8Z9mcnUIphDIUh9v5KaTe80024+rSqUmd6I3f92mwDKjQRuhSFIyiUWW+f3WgmeOzPcK9q2h2DgaNWFCvV/wAnvgrvXRn4RvEbY3T63kSSq6O2QBd2w6jTNa9RMXBUFfPMjvQd7HGaHTUzxiCXitgNoVKlwfA4l3G+UXeyqKAmQhU3RYtI6briptPFaSp/9420phENnLqNrZlo7e+1MifaeC2SaQOG+K/JeC3k1NjIGA7LssedAsFBwzL5WW9628COwe264p8M1ntc8/jiSUX9ZL14Xb2rWxAxnVErrS28ffyW88sHlge7R5QclRsi+6FlG6Y3vSEOLUtlFGb/jofjW354/685uV9Se6HUGzTH88O3XK43DvOMiCGXivFa4TbYEwdOD6Q3b77lb//2b/n++1+xrRvGKYv/fr9ptMQ2ct6wxv0X2dXEMLvIaibGqDsu6Y5MB5ZjoO8ayTwdeXg64uOkiOr5yGE5EKcJsYZ5XoCBc5Y+KvF0oPcN1xKHhzeUfiN9fM/h8Ug9erZPvyUlfageDm8ZvTIfTyARGYbp8S3hzVvG1jA7OXhQGWXsN+2ZMS3MD29JJdPqndo6o25qLTBm10O1+s97r/Gh3UozgJSzitJGCQpmb90RI9TWybVx39t6QjwgxuN7prSBmEbKGyY7BQiMQsoXRbPbLy8pxSvJF/qxgJVOY2UQ92mmIl1pv8ZmWl+hrTjXwAZwg1JWUnKUmrSTwhgG7M9Ioo2EdjOonNFLZxihtqGeT2NQKIGjFaNop127sy5RveywA6OpG1Q3ZDiM1X6L3hpi90xzNaTS6D0TfMfJUK3M6YTXR2EwNPY32CNHm1Kgf98DrNSbYmBQkmqwEzJkPz8MZij6VsR/RcAYu7Osmp6mFqMnsOyzutj9tO+aweqeVAp26FrZMAi9QVcjKbsQWdmLAfbZu7bGqBpKqPKl2XfQzWCI7Dk20daUMauz2E2UnkAK1nqtuxJDcIHcYNt1Eef2TrumTP7oZg7xwDx5cl3JTfHIa9oU9mY2Uh74ODF5g0TPmlU/aGIVGYPfNzEdRuPN0yOvHz/tFg+/x7bC1/W5FcFbdoe9wUcVqWupvH36ls+f37PEGTeENu7kMbB+YauJ2hUrnUvFO88yR9ZtY5lmaumczx95eHhg2248Ph2pNbMcJi4XbWjund0io3TPYTWsbFwgBqPltTGS0srhcMAaoeREnyIhTqzXwuF0YFmewVricsCFqBtXpyF1Fxw+LLs5c+w6odPPyXqF7YWH519wHYZ6/kg8PhHizP3zB7S0dsbZgJmjQiObYCqwZszhxOiddEuQb/i8kj9/ZLWOaD2jdS1a3gptmL2/sGKGQZxVLI3VlzaAVrI5ctExvg+9MWgTucV7Q6uFtdzJwzD7iWl0rQAUQUYAV/CbpWTY8o0dBqSm0Fyw8sX3BdZ8Mc6CIIyhAMJSM5VOL7s2S2WaHD50hihgwVgBBy1VWkm0DlQlujRWvNH2o96vDDPhUfZer41SG2UUxBf9XnTl7o067TG3gjP7AqA3htFFWDeWZpQkwdDvnziFKIzWAasY6zQouTN8hqgG77H/Po2swflqYHhkaAC9jT8gSqTrUHZmt1Crxe+3ldYGBqs89qacrjEG1li81eZsIxpS7S1r2QHKBRuoLibitUxjJ030Xkgb4HQsRAQh7FlDzWtps4xmH3POeBdJdWDp1K43MDHaYNx7QIj7ARERZ+kVRen4xMAzpGKcNlDbAa3vJajO0Iq621vt9CrKvw8n1u3G1m5EJ7QKrUHZGs6oAOpDIMaZdc20qtpbMwWxFSOWx8dnPr18ROzM5B4xYjB4nIv6dZUr1jomo2N4MB5vn/D2xMPbA+vlM7Y1MJDbmd6PxPDIm8dnzh+v9LRR9iKFaVlIOXN6fOTl8+uu5TVCCFyvn3m7a119X7IoEtxoIL59sZxoHZ71kVpXYois64p3npwTpSTmY2Sg5k7Noxa8E0yIHA5HjNMboPOO4D1tqIhsjNBGw1pHHwXjjojxtHqh587Dm1+QjWOUBMu3+OUN6fqKA6a4INOsRBPFr9G2hDNgjifm53fUF6B0Qjhw+eHPec2Z5XDiert+JYf0rl+nsuig1j3na4wSLcZgXVedBozgraHjsGah9syWV3xwPNkHWu5UEoduGb2Q1fyIpTHPMwzL9ZpJSUtinBsYgbY1JAhVVIdzXhdVzgp0TymKje69abFt1fKXUhMPD4FgPa0XxF71ABEorVHa3lEqDmOi4qfcoI9M61fUd7QX8Lb9uUlObUCs0A3ShZy1BUmpyW4X+PeIeAfpHkHTNKDfR+sco7U9VzkodbAmpSTnDmK7YuR3Tbg1u3P+UAeC+bI9/T0PMD/P0DQCQYfSOsNUlARZ1NBWKnlLFKOInD4q3keCP5Gr7MgMba8OXlk4zky6ngbMTrEcskc62iA3A9YDHWO1CZyhPpw2zC6cDrass79M8eu2z0il9foVDteChk6lB2WWdcN6v+JDwocCplFrAmk6Zo1NHzTn8HZvyHaRVh15GxyPC81BMZnSB3b/71Oy+ntqPoHoHD0MTJmRsQA3GBCnoMZDozA6cYopGhjNCZojBb0NePvI2LFBRiaCPRHdgU0uxPiIq2gXH284+W/Il0y53fQDhRC9J20r83KgpKSxKecID4+qkR1PrOtKjHogOef2W4E+tL0r3vtLu5H1Tmkbo2G8Q6ywbnfsFy1IhC2lPTnRGGbgY9QQu9lBl/to8uXD2fcPeu+CWEXKdHGK9VlF2VXHd9j7nVQ33OGE8wfaeiX3hKkZPz8obcRZ7PIE18/0T7+DsDDqBvVKXl+YTs/Y82dunz9+zf7phtp+Rd7oyKgr/Fr3TkQAxteQOAy6CL0rl84vQfOhJdFqZohw7orPHkZ7GmtNWBNY5gXplVZeSOnGlhPOVgYZW8FNhjAU/YSoDvyVYNHcvv1uKiaJttJva98No51S0v41BMww5E3tDdIjLjgNXZeOsYrF0V7K3ac49N/1ZmJ8AS+AxphEGE37YYN3TCFQ2kavCkMworG80cNOtHDQHYZG3/l2dG2vV3tH2Z37ScGRLVCy0Ip+7UYaLlic/AE+MGcPWBuQbqBGyuhAUSytCKlkrvczIRxwPmhwW9oubE6q9/QV1e0qbS/9MN5omcWQ/VSHNnRhsBfs6KqVjDIx7U4r+AL5d0qTbEJOjcMCIejbCdBAd9VNZkorXg5MmjhlilDLyuj6ewu6OjcOZOzu32GwpuP9l7ezXmtrEXLa8F5JCNKymmPHYF0TYjtLXJimR2Y/GGNFTNQsqajbu5EYZmMKnl4snUQ3gdpEPWGi7S8ywHLAmokYA6105nAgnwtHe6JLxRiPrWqU9M7y8vlV84tj4EPQ7SeCM5bPLy8cHh55Pb/y9u0TtVamaeZ2u2rzeft7FFL/Ilwbqyt3MTupAyRM0CuTd/pmH+gtBe2gRCDEwDCe0jpudNrQh18fBEPJGec9LSestwyzE0ub0MeqCJpqEY+u9cUwpgOhNnK64OYHbJ8Y98r68qI3+dMbqBnKijy/o/yUkE8/MRC29cJ6/4QpidNpIq0vSNfAstqxVPdqfTBK1dvKrpVaq7V0tdRdQ3K7BUJHPEAf0KFE1cU6kjgewkIqhbVlUt0QC84tIAEzFaQZ6IafXu8kqbowa51ZNA5XMopqNyBmT6vshyiiI3frEOyiN5zulMZrhmrV3WsGuDdK0SagtiN1jLHYMfY6vqqUFaNY6uAUldOHI+f73hOhuUTrPDFGtcx4bddiaDFMHTesFVr3lLapVoZu4PsQNeX2nS+I6tXW6aUDLL2go3EVRrd0CrUpRfb3P8CGtoZEd0KcpVao9U5vFS+ObgalJs7Xj8QQ9kyVbtoMFprF25kQ9vXtKJoHkxU/jhpDaPL3p37vKlBa2U/hgP2yhOhW66pFtQNjhRDDLtaqVcJ7ECPEYthWrRtrVbOP0RhccEzG0PpM3grWqM+n106t61ezbe+eVit5ZzX1OgjvAibtf07/EpFYNRw7JlLZuNwSzgqxB5b4LTaeSWtFqmg5Bndqy1jvNf8lHkPUHgAU/exswDtLKwPDjBA5zM90D+l2pdVNfTsINa200jgt32L3LsJWV/05iGjPgLVs20bb+ylbq2xbZp4P2nI+zby8qNdr7AeN20epLx8PJQZ0zbUBNoTdCtII1hGdx8WIWIMPgTAfmA6PDAm0PrhezpxOTzhnyFmDvj31PbmQCUSMsWzbnWADzU04b+lFm8rpBfGelo32EV4+YuMjY1iiWO4fPxDWlfnNt3D7RC037OMz3H+CkjB+Jrz5NZ/+4l9z/bjy+O57tvWuupe1+rkbQ1u240SMM2IMtRdqyjhrEedxxuK8+zp2mqajXK2dnOu+ENI29ZYurOtnbvlMM0Y7I4NXzPMQgpmY3IFgI2tqDCPYMchJ23r0dhqo0hhScM5qVtJYxtAIkbOTAg+KZbuK9hjsYWqGUmNCgLIVSrWa+x3qsXLOYI3SYHXCAYPdG8S+QDn3TtF6xVo1bD8+fosxYfdzOqIPpGGgqdRjnRbb5KRlLM7aXYZXTVlE/2zntcTDGoPpHitHhkS+dGiKDC0B+kNaiWgWbw7Y4bA2KrURxzou9FYwYnFWuN5fuG6NOHkmd0SIeLcAnj4qi9dOvtpXtvoJkUSwUVH5edIbl8gOMQQrnpIyViaNAY0OAmIGplXVLozqKZamnhMzVH8CQtBxr7RO2rTJONu80yMhTobRAiJZD8Qh+iYWzZsB+wamY/og1Y37elEcdQvaJWk6U5i1N2+awERycbR6APGEsBDdzCqfuF8Le3EQY2iJiQRPwSon3TkYgVbrDkF0eBdx5oizkZwh+Anx9x1R3JDWkFrwYyI0PUAViogahXPFWGFePOuWFIJYKiFo8sDtVoF1Xb/qDE6NdnvgXGNWipQxux6zM7hkF4vpTFH1F+fVK+e8x4UJcYHcByYXYpy4368cDgcd1UR1r1T0cKjqq0Fap9SNPgy9CbZmvW0YSy0bvUO+JnrZ2LbM0/N31L4p0UBQ8R+DuZ7pbUOWB+5/99eKx3ER//wNt/d/RUwXDo8nXs9nalfIplMMA60Wyl7JN1C9VDN9Bucd3uvLbXRleNXWwVi8t/ReueeNS17ZaqaiC6I6tBrO+pOO50oio4+idhnZNSP0cKlF0yPDNA2vG7sbUDVqZdDPh2HeK/sMaS2MZgg+7np0Z7SGFYMz+jIbXV+ShT3b7ASzLxKsVRpGH0k3naNQ27bf0oqSUl3UQ9FFmtEMZqtGl21On8lJHD1E7q2S6/iKs2ZHYzmrxUDO7lh6mZinZ7o9so2BEfVKtpEwTnE8v/cBtubMbAVEncNWPLgDdTRavyKt42yk9sZ1e6UOQzy4/Q1uWOKjsulHw0oj+JnaFUmNvWKsx47drVxln8ebfvPtlyuvvsWHgWj02h2cY+DU/yKCNwa7u90He/5Qf+TktFHjjRYDfRgMalB00SoBEnZ09UQ1OzF2GJzzDK8mRrGBdbvw+HBSnazv1e7WENyCEU98cOTtiDEOZxemcMRYDfreze9IpdFkRayifp2b6VhG1RHZGEcflVJXjESsOWL9jDULY6+FY3Rt924VV0R9ah2CEXIrzDGQjaPlvBsWB/12VX9SV7PjvCzUUhi9E0Lgfv/7sPtX79NuLP0S3t62xDSpZ2j28es/l53/P7SRmG4sW67UvuG8RkCsqF1lmiZSSkzTpBabYci50F3F+olSEm7A/XYjnrRcVrYLzoJ1E9u6Ypyjd/UbsWXuawYJ1JdXHr55y3CDMT3Q60R7+Rl/W5kOD2w//Cfm44HjcWGsb7hdL8Rpxu9pALPHbYShuOnRtHRVLGLVUGr2QoaS1NwtovVtLni2UmjrSjcaQzLec7APnOhcbyspr2xbRzjj3ZFSV3LPdJNwfrAYoy/zLy7qZhAX9xhPUXF8F8utVay7GZExAjRHzYZWHddWmGYUiPml6WrP8crX6jKn0Tazb/ekUdsFsR4x6C09Wzp1j0V1jClY77FOUVdq2t9R7ejBb6zBOMFhCN6QiyYOah0YAsZpxZoxBnpltMoQQ5gfsO4BEc9CoGa/L7Je6aZo2/3ve4Dd8wuTm5jcM46O+K7OdxeILKR+Q3CcpmeMNdzWT+SyqqmtOazMGDPT66alDLRdy6q0seKsx4amptY6qcu5DZANMf6LGMZoULt+qIxYJh8Zo5Jkw+yr267uQjUZavOIbndsprSVWgN9WMDvOa/KENFCCIya73D0cKeYwWhKkRBjcOLpLTPSGbwept4aaitM8xFnDsQ4MxZdRzvrsHbeKRpHusBarliXsL2S60YMB4IRKroIqmQtNSkGSHh7ozDh4oK1YXcpV8Q2crnTRiA0Q8QwatqbZrSENG93JWcaz7Ci6f49QoURrpcbz8/PnM9nvQ013UCeTidS2gC7a5OWbUuEMDFNM2OwH1yips69OVlQcGUtdS+lUH3NWcu6rV9vdFqiooUpzkaN0aCH3RiGtXXW24XRCv70hDM6cnqpOCds2x0EHeNunZe/+wsef/HHmtW7fsY/PdHNBCkTw0xqmXFXuOHl02eWQ+QwLdAUUKD5Rr1xalMTO3FUyDl/XT70PQLlnLbOW79rZyJI7/ghmEnIZWVYS3SBqen2XumnuuD6eHnP5Dd6036DwgZ2xdmGFf/Vf2bwuv2WRdkvcscJeOO12b47Wg960HS3V7FZWtkoreGDxYX97ycdHxwWT0dtNd4boutAYmuJ0W9KspVALo2UdiZ9axg0EmQdxMmDVHJ5Ia2GUoaWorSMjdp01X3DISyxYwW2NCilQw8wvkwPSpHx/og3DzAC1nhMmPEmUmtBzA3xFmf/gAOs1Mx1vWCXE0Khy6BjKEWvtc4uX99Wxr6l16LGOavxiGt5ZZkf6H1QStO849CbDwaaZAwO4y12wBiOPiy1V4bo9XXdHxBjrVb4iULXtP1b9bactTDE2I4MIZdGHY1uncLoRqKysjWDN3sbtuY7lEdmPdJ05DImsN0TY1i8dGrWN5hlxlCR/iXuM/Z8pXB8eGKZTnjrKKWR6o3eBG8mjBxx/oC1d7aciMbijVDSivcRZwKpJA14y5csmkFkoqczDMcST8omt0AfGN+R0THDMEqn9EpwamMYfVAY0DOtKvO9jUal6ahnditKrZRS9I2Iljp47/cbmAbfnXPUeuXt27f7A6yGzhijBp/F7FYApzqSikDESbeaj4+PSn6wjdvtxjLNbKMzzzO39Yx3jlwr65robbDMkx4e24pfTnRjMc7Qyh0DeNOpNnB+/56TNfjtlctPf8XycKKnTE8JMwxtdtSfzgQfuNTGljPLHNnWzBw9JTilogxDK20fl9VwqXw6Fbqdc4jxOxlVExLsyZSBFvUqf85SgEjD2US6X3m5feaaXrTBqzlqSVxz4m70BtO7JjkwDaEiDkQikz9ihqH2SkWoXf9/Kw0zIk5mjSBVtQSVmqhN43mt68ZPC2EUVRScmss1N+ex1u0ZRt1U2wapNtq4A0LNkLPdO007YW8dYlR6u9LtQs6DtHlqRW/RQ/2Xk7OKgDYV62B2Busc96uhN3X2i1EmPs1gZGYMQ3ARIzO9aleASMHahg0O7/6AA6y1QuKVLUfsZOhjRkSJn6XcdczameHBeJ4Pb7jmz2pi7YM1r7SRFVFrOtMYGBsRDvRx2/HMVctX90xWyZ3ah9o0bKW0gmEmMCOMvYux7EBES815F5eURQSDkssemDWEGIBCaSum7LXpfjdQjkGXjvUWYyPW7e0qdT/fqOR9U2mGUSqBaGyiDt2cpXwleEtwE9PO9iJ1ar0S/MRkj3geOIQBY6K0D/RcKaPR6x0rgVoHadsZZbJhQ0D8AUviev1Mr0X9N6Zi8EQXMS3g2sxhORLDA2nNOGuosq/+Ryf6qNEUBB+jMsx3vUtJyPL1V4yRMZRIkFLieDxwv9+w1nA4LKSU8V59W845WtMFiBv7mJkTMkTNqn4nie5jqh6E6rhmv62FvURl8pNii23DtMLp4Ymy3rAIa1qJUugl00pm5EJzkcPpwPXDT0Qx3D78NcgvqNnwZBq1d+TxDWOKbJ8/cHrzlnL5TC8rncbltjLHyG2r+7io3y67z9Fq4tVkiWmOYLVFS+UJfehNN0hwXwPQdgwGK7YabFfdNPfCvSdyb+QxaDiCseS2ksqK0HFmKJjTGEVUO4e3alo13dNrwUjU8XEkffEPlQ6kW8woaDltpw81goox6qp3YI0jBDWMdjOwzu0ECI0rGeuwzdKTNgUhldaEViKj7wZjo/tLLQPJJG7UBCkpTlw31kLVfBBTVB9lNwouiJNFuifdlW5ivcWMCYvFjojW9gmmw3CdVjdKuyNW9R1j/wAnfu+NJndS+Yx3AVci3jiGCeRRSPlM9NpgTS9EN9NH5bZ+JpdO65VLvoN0QtRbVDSCcZFRO9KjlmLaTohO+/lQqH/uAhgdB1vFMtBScEfHKZ2ByDBqFOxd/WqtJxUvO9rx2AZTmGhVqFYbgqRvGKuo3db07WuMNsZ4d6J4KGXTrkcv0IVeK7l3XG/a7YiCF0e/s+Ubjw/f63hjHbEv3LYXxF4JdibyqGA4sdS+srY7YQCjIqyUYrhtTcPfruFMxtYLwVRoldutEpznsDicCeqRcRFPgCb0pkJsLUq2dF6IBEaLaFOOp4th3TZlz3uvt6b9JiYiqk0p54UYw+6DqszzQu+DaYpM0/x1xPJeV+pfDqltXb8ehDFGpmlijEFw+mc5Vwk+aJTMGpUPrKXkvHPZ9OXR6cRpJudMsAZpKiPk9cr68hmpN/juT1lOR0xacS6wff7E/P0vYLtq+Pp3L8gvf0mRTv/0E94J55cbyxy4pZWff75oY1EI9K7hbB0j9evpvTGyltiM0REXlA9mjRYWh4g4tzvJFavTc8ECD8cTxkXsvBBej/xd/0jafiaxEowwmt7cjSlIr3gszmubvbMCJH15GqOkiVYZsiJWDaG9aZeo7DSW2gqllt2D2VUG2f11X/DnxluEAmZgJGIIdAZ0qzGg4bUxW/ruLdtJF9FqmbMTrCg373r7RF438haoBbRJLMCAWgfBPBHdAeyrLgGAadYN5/3edLlkUKJzd0gz9Fx0aTAqjaKWoz4YddtdBr/nASbdMkxjK1e8O+Fkwzj1Jk2+cr5q6YQ3KvBaOxFcYLgDtVx3d31G0MNKJKKNJm6/luqtotUMLmNMBRcIAEVF7S6K17FKldfDpoe9L84oZYCkwu7QoonWKrUp9td4NbEGF6Frc3PNFfGVbjI1K0jO7Q1LYwS8h0qntkSzVYVT0Xbue2l4UWql3a/mL5f3vH3+B3QiQqah41kpNx4P6Ie26ZuSFigMTOkMkb3Mwe3pgn2JSFND7bC6YNgLIkxzeGuI1mOqHly1NuxoSG/0plxz6UPHbge4jnHq4YrTzDQtyO6yt1ZbbEJQamYpWQ8XpyRe59x+67IsizZI/ZcH2JfDzXv995y1hBiY5/nrzavXBgLH4wFv1B4Qg8cYj3NWoYTG7P67Rq+FnDM4i4yoP6teCMZzaxtte2H9zZ/z8OZ7nueZ5fiGfH+hIVxKZTaVVO7E9z+wLAe2a0Z6wwcdTd9+8x3n1wt5u2rqwNs9fWF2H+Nu3sWg9hzZD+C4Fy1rU07PdT+8impQQ5Mi1lhi8LwxD7RmKE0o3ZDKe17ThdwaYqPilqXgrPaX9l7IoyIYnNjd+6g9keyXfyONPnRr3Jujf3229eeLUb4X7Dgq28mt6K1NlLzR3cB5RUe15hFpODdjS6eVtk88Bu8th+CJcezQA6MXgtooLVEqmmQZTbU1cyD6id4NY3/evF1J+ZXWVYwP09hvcwG0m11LpEX7K7rRTgvjhtbi9U4r6fc/wPStaGndkkrDm4KVwmQ9zhpCmLjlV5otBGtZcyHGAz4cmHuj5YEdhtELfridtuqgK+enZG0PQgytJYxtYDacn7AuUKsnbZoVM7FimDSjhUXxH1oam2vR7Qiq+6tfDUY1jGIZYrFuxhvLmq+kelMNySlhE9NZ/KPmuhj0btFKrDudTTUSrNI9u1536Q1pGwbPPb/w+eUjx+Utg0yuK6VdWdePjH6ndNjKFYwaE8GroVGEL2BF/2Vze7CIS7rs6BmRgLe6gXXDYEV5Wn5o3buMgbOCE4Xd1Vb2WnqjY9DeQ4nowaQH4mCYzrLM+633i3jN18NsXe9M08LhcOB0OhGj/yrie++I+6jonJYaT/OEM5YY447gmQhBrSEpbVhniT7Qase7gPeB2ip0peOKEVouBCekVOg5001npJV6f2G0zsPpwLnd6OnK5Xpju7zn7Zu3ONNo148MhJoS3RpM7+SPnwjHiLQFM4Tt9olUN56/+Y7rZ/2aEfAmkHOm0fdgud4YnN3LgOnUckeM2XXXXfdDsD7gJ6vI7Zqo6U7ulbUV6ih4BksIvD09UsagpxX2F0BwEWsEMZ3cVtZ0AwpDjXwMYxRNNQqlNJwZYAelZXKuWmkoKp3UUcFpwHwAuTZcB6jU3FVLcgJDN+hxl1HcMAR7AD9zzxul3jF2MNvA5CLBdXzQRUepXQ22vVO6cu1EhNaLHnZhoreNvBnmadbb8ThT6h1nIEarW+Ts6c1wv2ec10nPeGWLqcLRMQJWROWn3/sAE52DR1eo3lYvODvhh87R1jlssWz1yrAG1xdGdrsZzeHlgHMTvSeMNKTom8IEw7CCmLDrYGZfDhScH1iv87vg8YfAuW9axioawYnWYiQQvKF5XcfXlhTV3KFXGM0xxNJbhLHQx4QdgSkY1lJo5UbYxceUM56Ec5VaLVsplNappVOb9uo549WLNgzgFQ2Mvp3GEM6vH/B+UkxuTaT8mVv+Leftb4lxIfcboK0xQTS4LQjNWsQ0/D7mei+EKTC6cs4tDUtVxn4Hk7WnEvQHbp3F0LmvVw0FWxWijdeDe1hPbkIIUe0bu1G175s27wMxaj1cSroxTCkR48ThcOB4POK9Z5om7vdtHw8jy7LoNtGpLy74wDxN+6Hmvv43vbZ91NStXymJEFQod9ZS9g1lCIHWstILnOd2uWLJ5M8/4k3m9eWF43Tg4fQdW9iQ4xOuoLTgEPj58088P3/D6+WMjzNGPN47ri+fCMZxu16YgrCtVyiDaZ5x3tNH+xqb6jnvVgW9jfXeWLcNsUIME072KJzfW9Nd+LrdjQilZ2bvGRaGEdZ0p0ljyyuv9wutZ4WS7CNz8Aec0eq2gw0cl8L1+on1WmhDMHHgw4IMQx6drX1JtYguqaqmJ8YQStGuCPFKPUWEWXOjFgAA8edJREFUNa9oz7VhSEHagOFoTYU/YxyCxfcjrYLpXywjY0cKOQQFHvSRqWUwmkeG1+3tF9KMdFJZmdyCHQGHg2YZBEY9Yq3Woxn0BTiaUPKg5IGvSqVwpeKj3niNaILEMmt07Q86wExl2I6MQWlBBf29pl520+VgcE8bs3PUYTFEhUig1AkD2jKdG21UchLspAcgDLay6UwuA6ke8R7vLMHMCAuH6Vu2VLncP5HLDe+ecSYwKIR4JJROuqykUaCpg9kaD7XTS8UuJxiWRlWWvz3yckuMsWEtVIRLO1MjODlScyPnSiqdYRvON93g7IK3NmeDJSBdby3WCNRCl/o187WmF2q/kZrfdZ4Ja/aqKWMZo2INlK5GQprDiCVYB04jWaMqZWBIBtswdtdpZCCjUerK1jwYx+lZvWcyhv79U6bVzBQf8HHGR48iYSrGGeYp8vDwgLWG8/m8B2gNp9MjcXekq76Ghm1rJ8SJ0+lh317CNE16c4taohuijo7GGM2+Os9g57l1NX1uWdvMb9czzmkmp21CjEERz60wB0teb7Ry5nq74Y3h/vqRlu/Y+UAvZ60sAwqGZTpw74Pj6Q1pvXG5vHA8HDiennj9+DO9N15f76oHus59S3jniEF9ba3tBJTWaL2CEbpo8iLaqIsmF9lxERgb9smhMHrG7rYKZxzLboTNtXBY70y3G94acrmSRmWeTnpw+IVlecCLYYonnJ1pb/QF8vOn91zSe6wpSowYg1YbpWYM2k6vz6WmNnqpuvGMHufb7gNse9azkYv2rBJ32WYrBH9gjBkZQ+WKPRUiUpSkLFazyUn9XHlTCowRbQFnf2ZhkNKNuzgeD99Cc0pFJlOrx4UTnTudjrEVHxyjB1ISSgOSoxXdoE7RY4JXCURmYvgDokTBBk1QjYIw6D2T6x1Xph2lUcGqD6vVwpo3BLVLjCY7RSFq1djoMDJtvdNwlDYIiyMGT/TCli44r+NLLWhgdF6wnPD2xJvHA6l8z8dPv6HtOoCgN44YAjeZKLljmRlduVbOGe73lTJdEHNQquuuUxgCNRUIHWsMua/U9pkYCqlVUm4q5na9yg+zYb3Bis76tmkcSN37Becbra1Yo6NdCAdlMI3BIKvLuXfEBoY4wGqMwpY97FoI0wFq2Zf0QxvFe0KGp1JZ631nQXXcEIKdmQ5HDu6Bkit+mnAhUkqmj7uK73HC+QUXZvpoXNc7IUamSbUq7wLX2yttt0M8P7/ZdTBNStRadqe46KE2eWIM9F2AERGW5cDtdsF5+xUIOMaglEIMeruzzmEZup2sffdRwXq/YI2Qe+N2bniB7fpCNw5aIt1fiU41Oxet3oKBZYqs50QqlYZg6uD4tDBKZxa4nF/4+cPPPL15JvigoyldiXBiWEIEUeOm8/arFcTa/Wc2ho63pmvRSKsY11QPtZYv0QrZmV2MvV7NOdLtRq6aMljczBJmluAJwWvkqFaaFNZ1ZVkeWeZ3PJ6eeTi+Q3Bs+c6751/w4fU3fHz9LblcQR6orVPyJ7qturkzSoTwwdOHY1CVijLpTcdawdkJKwF6pdVNP4e2kXumVKW0sLsGYDAFR4gGK8r9K6g4X6rQetCOSKsv2mHGjubRbeo9vWDEMapDpHyFNe5F3ICmZYSK8w5tLK/UrhpubYnWEwd5ZIpPmsttf4CNwjPTpdD6pvGEnsjjAlloMqGkyMYwYH2gpqqZwnrHjok4nzhMj0CjpsStvHDJeugFu5C3weRnjtOJyR/J5boLzsK2DoKD5XBgCg84iczhhDeBz9cfKO1OyQbY8Far1zsWOzzWGIITbO8E07i+/sTR/AKwjCA4bzksj6ybpbeLQu2kUXqitEJvQkVpA2N4fem6ogefBMDh48xoBZENkUypZ6Y4gzVYhMfTr0ntyofXv2D0TVuQUdF3GCVTBH9AbEbQ0KyYPRXQHMYNpDdybdSxIcNgnGPyKoZPXfBmge6preOjxzpLLfqACPq960bHHaRTSuVwOIBRn5CIYd0UE9Na5/vv3yn619kdgdw4nZ53I6rmT/XGpcgdrctSWqlC/uxX0bv3vRzlfifGSGtN6+EAFzWONM8LMgqtZmrJ9LyS0o2e76xl4KTR80ozVuNJ1mqEJd0Yr2qRuV0vzIcTSOd2fuH0+JZ0vRCnSeGMtwunZSbnRJwirQ1iUB+fiCWlTNqaRgS75jt1NHZ686yDYcYu2Dcw2ojdjQr+yh0ciojuu43jv4jvHZeFePF6+6QyqPSeaM1yucI8P/L2IXA6nnh+fsfp8EAuiR9//A2pnknpE6/jAtUxT0/7AfeK8RopGs5Ry0CIlKrmUZpu+qY4423EEfBTZDRNeuRy12dyqPzvhiAymCbFPovVv+MoKGOvD9pwWALeOdVsvWPUbX9eBk1Uj3u9f2COiy4Au0aDFKfTVI7aLR9idBRXpHbTcdQM2rhz3xLGOLz7HsMfgJSeTWDYmW4y6346GhF6Feooyq8aVX+QdLBamVZHx8kRI6LeqHigTpl7qUi9U7tqDW4sUCzhsDCFmeSmnQ3WySXx+fMLzp6Y3SPWC30o/dFZPVh7b2qZcBXvMq0Ylsljh9UvLBeqGWzrlZfPv+Px6RFkxvkDT4cHHqYT59tga3fEB3WzjIIRh3faf6f/G/QKOcMUdHRqDV1ImAljHLWspO1MeFigW+bpRHj3P1Bz5tP93+8VWZZtzYRgEFuIpjLHBfJG7hslnTEy01A66DCdgnriLA07MmxnJgnQI6VdmPwjzQZGzWwp0RvEqAhrazRH2Bm0Vjkdj/hpIZWKtaodjjFIKfHu7XeEMO8xGX2AHx+fEBG2bVViaozM06LjFtBG+3pYzfOCtTvuZdeUAMSYr4bZvBtGW21E77C9sF3PpPVM2y640SjbymiFMB+pKe84bkstDYwQxDHNegin9cZhmhEEf1zYrlcuL6/YXumtYgVicPo5AUqrTPOit8OcmQ9HbPCkNcPo5JTUgW+tiuxx3jeQEz5MYCy1FF0hGY+1gdYLpWwaSbMO50SLZcSyDoMNnrgccWGCZklbJpF5OATEVF5ef8vbh2dy/hZrHE+PB2b/jtPk6fbGOb9HEnvbdd39dQdaTbpZdBYv0OwgZ60s67ZpfhGH6xGLw1eLl8hqDFvdlME2HIL62bz3hODwAZBMTl1tOSXv2ciAc0eCcfQ2sOLxfkZG4N5faOOq3xMiDEttTgEBrRGl4QKALmtKrpSiiCiMpWSNZqmJeENsZku/wxjDHL79/Q8wa3amdTdqcOuVJhudTCNQ+k4SNcrQtr5jR6fUgkgiRsc0LTw+vqHWwiVdKf1M7o3SN2a/AIGSYFo88/SgvPmy4rzhfjvz4/vf0rvjMD/g3QSit4Gc95JXMphCCIJhYXKTipC9Utse1qZyvb5ggmU5HDAEgllwweGM4dPtJ9XurOOLD8cFr6y3CnUYbQnvnVETTGaHselmVdteVq7bD4RD4DT9khgD0+T4/s0/Ibcrn29/jUinoriTKai+MoawhEekGQ0di8Pg1BLhFD1UsnDeblRfaHZGzAPWOE4xUkvDW+0IKKURwsSnz584nY7c1wvGDbATj2/e7WZMHZNCCPvtY2OZF+Z5wWB1zV86x8OkfZx7ZKoU9Yh9waGIGGrOyDztMSFHjNMe8NV8YVq3r4J+q5WtVrxz1FJh9gwqp3nm+vF31OuFQgYaOW188+aZZgdJum63elWrSc7kPpjnk9pqht6Actp48/DEWgrr7ZW8bSxTIG8F5xwheA2gG22issYqHnrHpTNUL/qS/wR0bNwx00Y8xmkrtQsOcTomixlgvOY7MdAGpXbFPdnK9X7n1gtZoDYd3XrNOBOYJqHmlb/73f9GnALH44Gnh4XD88LT4xt+Vf8hl+2V188/k/qFNjbGUJqsHYqvsqgNyJiBswfG0ARx7pV0zyzLo96GmqUbhzcTB3OCvFHHLmW4QAyTjshuL4+pgVE7Ld/35ZDHWUewJx2h94Mv+DdM/sDa3oPJyv9i3Z3/HaziEfoQLd4VTXloF2TSDksZOtzLnqf0AzcqOf8Npbz8/geYXneHmv2afvxHy9QmdLfHUMyEQx21A0C60iTbmRgtIcz7wxx4OD1z3t5DvSMjkfsnHN+SN4OYSpwt1gZs1fCqdZHz7YX1d/+Bh+OJeT7hzaSxDjqlnmn9DmI4no5IP0ANqrfVjZ5uWg5Cp47Bdj9T0xN+UZe4c47ZPBDyJ2gzdWi1+hA1T2L6VzJCa06vwq0phieo+39YwYg2jZd24/X2kYfjLwkugIFDfObd4Z/QWuP1/gN9NEXKzEMLDHrH2BmRjnAD1Cs3ZKdJTIHDdKIlsNURmIl2JshEK51gLM5YcssclpmcG7/45f+By/VGGzDFiIuLGkadg1KpfZByIqWMMXYX7SMgvL6+8Pj4wLZjo1trunjYbRGX65nDctz5YbJ/Hy2lFI7HA2N4Sk6qO4XA9XYl1MBxWRRbnDLGGS7XV/p2w47Kw/Mzt1HIa4MuOD+x3VaMdRwf3jIwtLpR74VcC0Hg9vIB64VqLVM40Hrl4+3O4fGBeYoE6VyvZ04Pj+rVagZrJ2hay2dCpGN375tXMdw3jjuwEIRlXnRDa5TUoGSMnRWPAhCtMxhdRcNAiSq9U3rn2CKn7jmlxuI/8TAf2PoL95SpfaO2gLXC+f6J//S3/2YneQxKS5yWByZ74On4PcfjEy+X31LHhjcGNzwmTrQ6oIlio1qjdYV3GhsQ26gb3O5n5rCozmoNsw9EeYfInSuZMazeqIwhRD3QSrvvXa5FI1dtMFtHT5XcV6Z4xNkA4ghGODw+kMeJe3qvLUkNulTEKjEmRocPQi2FWnfIQBcFPuqhoS8CqTgvIFU3vL2T/5ADbO1N+T0sRKMiYZPMHgijV6G0Trdaae+9x9mubuFhWPMVpNEZTH7h4fjMfHsgby8YMdSycU0fmd2JfIYxHomzx5AZUhjD07tjXX+k9o881XcEt3yRuJGuguHh8MAyf0OvjrIOemlkKskI1Qgheg5W9YHX888sh2eCM3vxrRDDkZGrxjM05KEr/dGxFhXhm6PVQR+VnApOEhLNHvNQ0kPtiXt+4XJ/5RDe4cXhY8D7E8+HP6XWzvn+I6VmSkkscmSIitC9D1pdCGHe1+wdsVld87JgvMeVA70IgcgsB+bTwqhD19t4unE8f/OO2/3M9Xbh4fSMtZ7WOz5a1vsNFyc6cD5fMFjevHnL6fSEtcLlcv6KVwZIKe2GVL//spxfPxBDoNSKc5br9cqyLFqCe7siQE6ZkgvH4xHGoI9GShvOWPUZ9YF3jiRwftXwNsYofQHDNB3BOT3s7NDEgziwEVIieE8RS0krxlR66Tw9PXO7r5R7om4bp3nWrGUrpJLoA46nB0U1ZW0gEhlYgdQqPRdyytq5aDXvua1XeivE6Yg2aXdt6wZlxZsBe26VPTEwRAmsOrYKc4wcT284PrzDnX+LT462dgYbg5mB3upeLr/jL//63xCnIwjUdMNL5OROPBzeQJjot8E8B6ybldBiOz2jn0sRxDigYYYnGoOJ+jlLa+YUnpmHyh7eOd6YidgS917BzmACzhq8HJBuWGul5I22CbRAFRgj6UjZKssSCHFBrFp0okxM4Ylb+ci2/UzNN/3+mC+Y96BjfNFzozdoXYPy1nes63SKamESFZaJ8N+AUfzXD7Dr2pmnBW8OGFcpbUX6jdLuKJtoUJog+3q094p1e8KfzKfbD7xrv8alQB9KOF2Wmftw9LzRWuK+Zkbccc53oTPhnKNshaFu1H2kytSyYhk7+35gJPDN85/w+PSW4BZyqpzHlXu9kFtBnENmh0jmKMAwdDrn9QNPJ8dla1jbNLHvLKMIXpw2sDSlWqjZr9Nt0hFhGGpJZDfwk3pyWteqKgiktLLmM6+3D7x7+gXOqZPdO8/bx19T28atvGdd74QeadVjfdEtn5vVP9O98p6MjtLGaHQo18wYQsdg3eD19RNjgEdX/MZOXC6vfPz44av5tI1G74O0Gr1lWcN6123mN99+w5vnd8Q4c758IuWN4/GoD++2fc0zLstCa+MramcfChjDkFL66sy/XK70pujwL4QHY3fdaOhLR7s1V4zA7Cyj6e3V2UAxuql0zoEY6ujcbjeCrUr9dRPZrGypYMZgmhdyToCwpVUx1a3ReueWVg6HmVkCLkbStlLud9zTG6Z5YTu/0IemEbz3bFlvG4rV6coFK42SC6VWwjyDNO0GMA7tYQzIzhHrZmjsac2UsmoYXDzSK1IzYjp+8dhssdnRNtjShneqpzknfPz0O/7yr/8ty/LEYk8MV3k4LvzJ9/+U+3bjP/2QKVI4HXSzuq2F++2V3q7UpBx/6xV/NcbAx8CQTl4La80cD9rxoJEvj3MzIXfG3tnouzrlpcXdWaB041FhlLKbboXUK9sYHKVymJ8ICNYeOc5PzO0tL0RG+w2p3nBWOzRyUmBmq00xVkYXB2J004vs/bMdPKrNaWTuDyi2vawD6zxLWJChtWnWWEoee6HmbqAzmomqteO8YkRq6SQ+8dPLb7DmxLr3Bdo987VSKG0jF6HWwjyfKFfNgJ0OJ6wM0vaBUTLOzIplHl7xt1IZrXN6eOYf/PKPOSxPLPOB223jvv41a/vEPW3UVpDg8WbSunczGBLY6o3b9ll1DdOwQ/E6lqGh0qFb8r5v1MqoGJ+wPjCacpH0wL4h4qjD4K3FuwNb0WVHahdu+YQP076lg+AOvHv4U+x1Yt3OXOorhzni9i2fNw94pxVqo89480DwlWE2Wt7ABVrW0G/aNowYnA/qlxNHK5WX159BOjknfrz+QGuFp6dvOT0cqKN/9bh99933vHl+SwgBZDevhsg06dh0u92+Gky9D1jb2LaNy+XC23fPe/hbf16lFOZ5/hpHEviajzweD9SkW96B8rR6a+R0w3hHK3dcCPTa92Ym3camlABDcJ7t9kLNicPpxPHxifX1sx4MwXE8njRNYJQ95pwjJUil8GAfNMjcixpuAXNfcbYoUr7rWPMFJzRG03q1ofhoEcE7r9u8OCMYRmvgnHqr7hsyhhJ1RbHUxgm+C5frhVtVs7DphUM88LC8o7QbaWuct6t6tSp06YzhsVb44ce/4HQ8cfQLj6e3WANPT+/4k1/9cxDHz5//BicnHg7PPD961uuZT59+4nL7mfv6ia0kYow472giekMMlnteubc70T5q8Y5zuKhoqeECLirn7kvjGNUxisGaQKoJGMzHI91ok3a63bitmecnsP5InB6JzmHdATuA2qj1N4qURkg50WpR3RH1eX3pSDBGsEYJM18mu2E9uD2z+fseYHnrlDAYblIBeLAHp7XEoHdRFnc1OCe0mqir0lNLrxjb+PDxbzjEZ4KdACg90WqitUZpljIyo2dMFh6XhV4MZVNUz3F6R8ob1mWCWWAESvOEeGRehMeHt3jnOR2OPD2+I/rE+brx088/0tCoA1XjS8MZhnF7Fku45zPBnmjjRhCDjIaz0KTSpSGidoxpnplnWJMjFRUlrRWtfe+dIQOspQ+HM+BDJqcb7sGy5StTVLzM5WYxAlM88Vb+hJv5xD3/hi0XDtZiRWjljBmDEI44TkRzwPtBlTO5nOkknF2oNJybmGS/qcnE6I31dlU+VfQsyxG4YsQzLzO325nrdse6SJyPzNPha4A65/Q1uzjPM9u2qSPaOEX+uMD9dlPW2oBtS5xOJ2qtesDvW0bnnGKsRe+jxhjyulFToozCsAZEezu9dZTWCGGmbLc986abZlqht6S+OQwhLqS0cvl4Yzo9EOeJuq0g2shujKYfrFcN83hYyLlwv2eOx4UYPGICtezmXq9jnnT9GeZt3RMhg3XLWMvXwLsY7VpouWhGr4oyNZxXAb03LV5RUrTi0cOBeTa8vn7mvl7YWmXUSggL0/LIw/HKei/U1BjOgO3UUvfIF/zlf/pfWeKRf/xH/5xlWvDG8/bhkTL+lHk+kVZt1j4cDxymd8z+iR9/1mKNlK/knmk9I37SW1ZLVD94ya8EG5ntTCvQTNsD3kow6aJezzGUMGGMw5nBiI4lPPN0+p6GkPLGZXvhfP9ILpXgHnia3jEydGk4Y5n8iSAnet32Alt0q9lRlLTVVMBA849i21fKbxWd1hCPN38AUjqlwvWcmayoR2Q3ojlJpKbVaKPu/YFNexxLLdR9bR2nwcvlI+8//0eeTm8AQx5X1vLKVlZybtqAbSDXVfncYigJqul4dyLGB2q/4NzMFE9YE3c/lpYvyAh4dyRtjevlhnTLYT5SthPiO7lWGo06ivq5rKHUSs6JtXZ6T9TdN4Y0sNoV6PwJ6YNeB3GaORiH2LuifDD0PvTvjmHxM3NYENsJJVDyhdZWZCys7aJ5rlHpdKUaEPB2wa+R6+03lLJip4DxhlwrUgsxOhgL/kspiPck+8qwmVRumNHpzWPyhvTEKJUt3ZiXyMPxmbRlGJYQAq8vn/n8emE5PRKCZYrz7kD/gioR5USJcL/fdy6++5prFEF1DBSrXGv5GupW8V9vTM5qDdntesU/P+v2cccdnc+v6rKmQ6vc8505as6wlEyvhZoT8xyVKV+0e7PVivWeeYr0UthuL0zLA9PhiU7dUVeOIEItmTwK9MHxcACEbdtwXgguMEfdmG5pY90So2rmTpzH9w4efGg7V6vsRtW9r20ojaH1gnSdMpyPEBT/3GWAsbuNpBGmmWfR//7+8gnJFY8SfJflgTePnfW6UevAieJoBHYv35l/8+/+Z0Yz/MNf/3cclyNOHE/HE847Uk7crq+MqobvcTjwkL+h9FU3iU3INWsp8XA7517NuZ/vrwpdGAVf1fXOyPQtgwgpbZSqXjcUgMzh8C3fPP8pThZaL8TY2NrAbIne4ePnH3g6zuoAsAbrOkM8cXqmjZVeDLle9dkRu3/WBrLz7vvewtQpBK/FxX04JtGaxt/7AFvXC7YHbv5GnDQz1yVzMkfA8PnzhnSHtYqfqU1UlByZIZoity5w2V4wVsmOaWg1ee9F39ISCF7X2K/Xj8xvn1SYbYZRHd4u9J7wbmKOD8zTE3101vSZddv4/Poz1s1Id1xvGylnestMkzaiuKKtzx2HWIt1Qqdzu6/c7hrQLaPwhdrRW6eS8WIIHGgVamk7D30vu9jLfVsFaR5bDSEogtiEwEg/s95fOc6OhvLIrdwZe/7RmkCvlXl6ZHChd4cMRwwnqnGkNJhnrZhrWcBFLI8E72h9Q9DNUx16k1mCY/jI6eER7wwp69/X2kheN87XFxhdaaLOastRV/RKa119Xr3xej7z+PC4e5+U6QWQc6aUjHPqwjfG7HnJ+PXmBWrknOeFdN9Iadszj2qxGb1T8oo3mrUbvXG/X1liwFpLq7KbbTNglL9uZkarbOmOjIK0SnSeXgrmIAieZpQO62U33xbl1a+b1sVZqyOutY1SCzFqT6WPgdfPr+RSuF1v0CpxCirQiyHEWUcbZ5mOR2ycqXVvaHLqARu7hWPs5OBOhdHoDWrvbOvKZb199UONUXEuclhOeBHuIbHeM1K7WotEX+DGTGzplX/3F/8zW1v5R7/8ZzzEB1wMTHFvDjKO8/mFe7khzuDigp9majvjjWZde9kJx2YofMBaShv8dPvIaT6xSFQarHV7qWwl1UIumcbATZ6eDIfDGx6fntVmkxptNOTV4P20k0c6r5cfKMuB3itxUuClsdr10DPUmii1ME0LU4g7MabvCYD+9eBvHWQ4XY55w2h/wAhpu9FSi+1OTQdOh5O2AbmOlUHZBve1IHgYE7VdFVw4xT3kmrFOT1zlLKl7WU2Wiug1OEKMOGvIeePT/UceF4GhiXzEME8L0UUsFjOMhqPbxuvlZ14vH/jhww+8e/w1aauIqYxxB1O1qksM0oy22+x+GTMsU3QY80jJWsKKKD2giyJJUrmp1lEdGN2yOnE0DKWvtK6+s9YMuTZKL3iFimOMcF5/whlhVKsu5N6o9bYbMwNtGFp3ePeE9897gcaREIVSz7xeP6l9oB/w85cPmODDTO36llx8xEuk5X0TFCPpfmOg2t1PP/6O0TZ6K7QRsUNI65162shJX0jnywsi8PHTe9Y1EWP4SqAwexSqlExrTV38+9r7CyK6lPKVIyZDbybRB2rK3OVCnGft9rSWdb1CdDhRzaPXzuXlM94qwrqLShPe6eZKC4k9ZjnQykZdV/rQCjl9ASijq/dB60MtDTZo+3W5k+4bx/lEH0ZHneB5fX1lmWbEKjHh3hrBOSqD6+3GFD3zzjML04SbZubDs3Lm6x3ZC4ixllYqtWRE1KDbS2G0SsWytcJlXXm5rbxcX7ndr6wtkWZFaDuzaN43eO63Sk0NyUkXSNbhrGO73fmLv/hfSdvgz/7Bf89hHLG+0WvXrlMHL+cLJoCfDMfxDLZxu35A6qZE4dEYdmAk71qt0Gvjng1YmLvaQ3od9KTPwJY2ugE/OYJEluOJ+XBAuqX1M+t6J6WN2WtMKgSht8H1fqH2GwdZsCZirPbG9jaoWYV+y0y0C2KhtsKaNsVh1bz76ISByjQV9UH+3gfYaVEAvxFhWxPCsxIXe6WNxNOzoTLY7gmGxRurcRPvFA43Er01rCjx0lklftbqGWKUReTHVx1hmiNlXLikgmVmike8182JPpR+31ZUrFiut5Xb+p7arnz4+FfqIPeO6DXHN2T8/UFIR754uoYC4bwVbFSHvCJMHKUOaldapXBTrn6eSF3wYTfiVWEUxd/2MbiljPGFhahQvlrIJF6vf4cZMzJ0IzUo3NZXvC+0OlFrx4YjVmCeDzg7aZ37wfLzp5/5Of1217MemeJJmV8enLdIA4Nuw/wckAGfXz5R0oWH01vu5wujVQyWYYTv3v7yqxwaQyQEy8eff2SaJl7Pl715uu+3rz3r1gc5Z7VAOOXZe++ptX7VzNZ11cyj96zbyvFwZJ5mLtcz67rSpbF4FagF4fXlhSV6SlqZnNMuztYUMWPVlJlLxXvV52rLGDOYQsRNR3LVNX4YXctkYfcjDiodHzyH+cS4CdfzmY2N4/HEfdXavGVeoHXu21357HvuZ5kXlmXmdn3VOJZXrdRZ7UMAi8FTa9eYl3eKSBajkTKj+l4fjdYqo3emOPN47LzmO+52o6dMk4JdtH5umEYwgzYGqWSGEboILXdlffXB9fLK//bn/4qcM//4H/4z5mnCOch10ziOFdL9igRhMjPh4ZdYUb21pBtmjy8ZhJEGkHDeYHEM8cqiR61P9IrvQ4t2RmVgmOYFH4y+hI32g27bSkpaKXg8HrFisV6Ub8dG61XrCrvQamM0i7OPWHH0BFUMwVuseJxp1KE9riVnrIsglmwarQzqH3KALbNnuzdKvlPCxOV8Yzk84OyJGDIpJ6apKX0hG3I2xBCI3mGsvpVSudNyRby2k4x+wItlLRdaZacz7Mn2XYvpJILzIHfN7VFI28vX7kmpKvb2MVjTjZTvXO8r0yTMROb4K06nX/D6+sL5/kG1ngGC1zWxGV8P1egC1iscbhCYoiEXFW6tazgpmDarBaNUxDTMcEhv6i/ac173+sI8OaY54qw2kedSoG8YiXSzYKxlSxdu24WeA8ZGopvIRTAp4eeIDEfwM0+P3/Dz6w98fv3M6+XG0+kNyzxTm+UwzSzzgimO3mFLV+qmtEsnitlO25XjYUKd1p7j8cjldmOaIp8//Uy47lriPJG2FUFYlgXQ9b61gcNh4fXTK9MU8SFSW+VwPNCaMsS+NHff73ecMfTeWdd1H6eFMTrbeodikdqY/US+XTT4zeC23fBmIEGpByHowqDURq0Zb50aLH3QfhfjmY8LpW7Q9YZde8dZbQOyO6ML8bz55juW04nX9x+orXE8HNhS4n6/M8VIjAHJmZYrVkTJpU54fn7CG4cRlQWcMbvnEBCnf5YRjNsLU0Kg98ooBW8tiGFdr3/vP2ud0QaGTvSGMiyjQnd9D1Z3cBXj0exhafrPtWmO3oXL7ZV//5f/H2rL/KNf/enOT2jkWrTdvUPJagUSC8FNLNMTVTzrWCk10+uGYdAaNAYSNPBtrIXW6SXtU1LHOm36UkSV2kf6KJg9SR9jYJlmLrcXXs/veXp+wLWBDRYngTEytW6I2B1XZBlNEBP3l/vAIIjb6bfoQrBvgSTK+WumYryl1j9AxJ/9hJ0Gt/sLo83cbhsvL47D0ePMAec2rO0si0W8cKVhvc7GQzZEtHyz9UztmdIssm/sGBPreqW3zDx15jlouLN3og8EA7Vr0/AYndwutOvgOEOwlbFHblo1lKy43egss408nd7yzZs/5lfvDvyn3/5b/uo3fw6tKtOoFgaNKXSOy8wyTRyWRVf8vWHMTG6D1q940RCuHZlUBjkNWq1UPHV4clv1LW66fmiHJZuiIDgjSDDksUHT0g7nnK6g73roGBMo7YTxgZozvWSiD3gzMduJ5+V7tvwTl9uF0T5TWuXp9EY7LZeJuHhun15odUOaYG1QqiyDp+dnXIjkorrD6+tnrrdXwrbg3EKbGg9Pz/z86SMOAeeJUcuFv5BGt20FOrVltXYM9cWJKC7nfr9TStJNZk3My8L59Yw9WuIcub2eCcGxrglKZoqiN5Q6tJXGGu7XF3pftLjVWXrrWFF4YqsV8YbRI3bHPY9WlRhbtd/ZuaA3WQajNlLNDI/mJuOBt9/6ry1N8zxRW9cRqVakN2K0GCa2tFJywUlgejwR5gO17h61sIBx9C7axWCDWu4FureIP2geOBesX5mDUK9X0v3Kp9uFW7qTatmLMjy9FVKtiLfYUnFecUS3211vIlVvJCMrtcMMw+1y5s//4/+PNb3y7vlbQvAsISLGsJweaa1T26rAzxqxXfUo2zq1GdVyayKXyhCrPa2iLwqD0ESUACtaLVj23sqcruTpQu0b7PV50R948/AWaxr3ciGVK8NZet/loqGYIZEG4vA2MKSQ0ra/8EGsYRKrz3BrtFxp1VAb2AHFDKyr1PBfPb/+GzaKmrHGM/vAev9AjDP3q4rZLkws0zu2bJHSCc4QnCG1O2Y0pulxR21cGENzZTmvxDjreII2EN22lVQTlcqCjjZqYvN4q+NfSpm1XriuZ4xECB1jj3g3I0yMHhhtIMPwePwjng9/xuwfsSbyj77/H7lfKu9f/zOjKSdp3QrSDcdFHwJrHHGayHtDtzFCpeJtx+KVvOpVl9s2fUu1qu3hvXeMJIzt5DR0LJ4HLur47W3bhfykB21rtAF5U8Nj2TouWpKH2wZPhxNvD98RzEy3jqdw4n6+a3+jVcb8cX4kV0/0njDNXK7bbge4cJxPHOKB2jsf3/+kX1euXC8XfJg5nR65Xm+YYtnuV7ZcmaeJ0/FEaVXHEqcB6G3bWJaZdbvjvfvK+fpy88o5sW0bACKqc8YpklIiZx2rWs14K6w1c82Z0/HIzx9+pG0bw+qBWErDWr+D+QrOWryf6KPSRmUYbU36wunPKeGd0YdPhOPxibzdwQhxjpS0YdKg541pOuKf3rDeb7S0snwJcF9uXK9n3v/8I8u8cDgcEWPY1gS8chgVF2ZsPGBC0GwfQEOb4xmMOnYRf4UwMYwgTogjMuZGf3zi1jI/3x21GtrYGXjDcd82rHTmMCNYekscjo4trWxbZts2TLc7ycFqr2LNvP/wW3Jd+eb5G+BIcIFgHcflyJYH1+tZbUPV0muk1UwrnVYCtQxqY0cBKdK9kclZXe9tNApVt9AitDJobaW2O+fre07HsdOSmy7J7IJLGSMJJwfVm/escqtjh1bKXqM3SEM1PjMgl6z9p3tBzGgKaDRDe1IZQm2VvhODf78DTBrRWJbjjC+DXF44uoM2/FYQO3FcnpHSlKjohHvuiIvM8zvdNK6ey/0DW8r7yV8YVvGycfJseaPTyaVg/WAYqxEh5xCjpsZSC3U0cl95vX2ArkY8K44pHNm2G7kkbvfM+fzKH/96QjC0WqE1lnDA2xlvK84YSu0gnt4WehZ6GuCMFuIa3SB5Y4GEwvg802yJ/pnzObOdz3tzEro5aXU38zZa6XTvaGKQUcA0nPMUGUgelPxlbazCaWpXUho0v+EmA/1O7JY3yzcEOn4MonOsvdDplHGjccHaZ1rxiMxMC5y3jzw8PHCYTvTReH39yJauTPMMYnh4fAIsJauLvtXE5bUwjCMGRSq7oLVbxhqc/WIVaV8zkV/0otbabkDUX7fbjWVRyoPZq+5GK4xR1I5gHHP0pNS4XC544+gMxRi1ofwpo3z0EDylKAffe48ZHiOelBPOWdWOUqdUBQOWkjlfPnM6PlBqp6PNPOu68ng8kbYzbTXUpmNe/vyJw/HI8eGRgVBKUsvBywtODDTN4+XVEuIR471WfQHGBOVaGavev5wY9xVnYASPWB3hqrV4F1ls4tuHJ1o1mBH5cVvZ6Iy4W4XKGXuYObiOsTdaH2w5axluytgxYTCMUfE+6kRkO7f1M84YDocH7TU1Dumd6BxMM5/XTXHjJoJdlTQ7DFaCfo22a3EvKP4pKNa50qho1ydmJ0SIIecbvRlS2nh8eEejYXxh8hYXT9RxU83YWwZKifn6WakFEYuxAxcs9EYMbl986DIIoxjvLntdozM4q7e0/ocgpRFtn/bBMS0TeXS6XWl4DAu1qHM9hpneExStCYvTI85oe/YS33G+f2JLK5XGsJbJLBwmrzgNMrfbVbddqVOyoftINVnLPExnGG1LEXFc1gvOLEwFnLN6C/GBWjPbJvz4/kd+9/Yv+f6bP1MbQq30vuF92QO3g3lx0IVWDKNqdg+xGKc/ALAwDMbvTctOIzEhLMTpWyrvWX/+SWvtPZSqD58zA1qh5PteQNrAa3pBbMO5TrWKIBIjzM5o1ssYVtHfYzWFy/iESR3jI6MXrX0fXeF6deXzy2852CdC9PgY6b1xeDjRb4XtpsbBlDKH5ZHHx3fc7zcu189M4UCMC9Mycfl8ZXl44PV85Xh4wIrRJiE63isnyzsFF87zxLrdv7K+vgL/di+U94VaKmYS2G0jtEx0g20r3Nc73il473rdkKbr+jlGgg24vSwEgTEKMU4Y+0XPUinBOUup2sQj1mqsxQpLOLBtG9u2Mc0HbrfPLHEhxpm1FOIy4XKmJC0LEeB8PjPPM8fDgXSfaTGwpY1aG97afWIYlJTxOWOiY9SGMYXhLcZ4MBY7abfASAnSnWYdJkzY+YhIUad+vXCwhl88LEgwfNga59YUUc4BNw7Mx0iMCSufKbVyzxvr1ujZEKwnhMibp2+I0SBe9bKSV15eXojfHBnNUkbXZUOrxEnUCtGTGrqdwe1UQSuWYTPBK3gy2hnrK9UIIhY7DG2ol9MYXeb03hgmsaVE+nxj9ge6VLwzTDHSJWp5jtHRsST1oNW2IQJGNDdqbWNIxzqjyw/+3vQstuGMIThH8ANvqy7E/uutav+NG1gX5ujBRiROPC0ztWZSfqV0SKXjjOg3xQwSjTQqYUdmgKH1yrapE7eNhAuCC4YQBk/REv2B66z6UsnQq1HEcuwYsXtYujOawfsjzah43yTjxCrxIgbqiNQyuF4T//4//H95ubxgjbam5HbF2k0Laq22W9dSQSbW1qEUUi+MDtbq28W6gXRP8FEf+vArzFgQ4/nV939EKYP3n/4Oba7p0PUD4MKgy4Z1ETGKKBEjeNPBdyRV+tA18TJPLF4QZzmYI6nrWDK65XL/DFiyDMUIC+wpSHJJ/O7zXyPfOuY6a6NyhZEzTdVmvv32ew6HAz9/+Mjr+SPH47Tz7xd+88N/5vH4zO1+20P4jvW+YoPj6fkt9/ttx+boiLeljdaqiqvtC6Z4fKVReB/IaVMzq1MLQC+NNjLBOVJeef/jjzw/nZi8514TNRe21vCHR/wc6a1TatujVGN/g+9gwa7NVfqhVwqEhsYHBAjBU3NnWzdCiKzrneVwZJpmLp8/7Y3WAGNnlg1SWsF3np/f8vPnj3g3Ybyo5WTP2gqCQbB+3ldM7QvHYCdUWEQW2F/EtmurUl/v2DDhHh+R7Uq5XRh2cJgXEpl0vXFvME2PHPxboos8HCrRaudpoxCc5/oxEe0j33zzHTEe9q9fy3CHbLye/46Sz3zz/AuVOqi0vlJJiCkY1/DN0LvRSk5nkKaI+DEKYq0a053TLO8AqboRtdYgJmDGnmzOmu+kVe6l4L3BWANd8PGIdZOSWc1EnAz38YmVG0Y6xhRl29tCLlVvVdZTWsM0gTZYnHrH5tnhXcb6gmDI9Q/wgY1hyK1zWAw2GIwzeDtxvryS0wdGE+L0gGuWXjNZGlvPTK1iTVQGVNq0gkksthnSlglxg2kgxnNYhGiPuu3chFJgrYob0euzom6FgBuBGDzBOaLzhGAxtlHrosn1rnrRzz9fua7/Ae8HPlSmWTejSMe5wTRZzW7mQU4rl5aZ/UFvOZLwvWFKZ5oPNHHcb4XJQwgHeh8EN/jm+Rds6c75+ooZQb0rphKmQfRa+Nu6x3SLFXDSyCT1lFnF7HpjFfbmHdEsTHjsvCAtU/sL620lS1UuuwMnXQ9LY7mlF3738a/4/umPmPsD07Kw3hWsd5jVhf76eua+njkdj0DjcjlzudzprWCcYBK8+/Y7Pvz8gYeHRx0rxmBdV7777hfc76vqF0YNr9aOfXzU+M4YQz1cAM2rnaGrHhqD53q9463lME3kaeH86T2Hw4nDfGB0xSJ1+tcQtQ8Ws/vORAzWeowRQG9FxlrdeBcFKZZWaK3ycDphp0BOmd5gmhbyttJtZp7i3o/o1CE+BsYatvVOXtXsuhwOlFIpOeOs4I3F+x0fBfuhaeliYaeUdGNV/7QWZgfZYEzBdqs0lm3Fu0BYHrG5cPn0nnO+cWuVrWWcmfAyEeMD0U5YyZwOXSeJcmGKhuMSMf3IfAgsexaztUCtiVTuBC/c14+8b4UYZkYvOKe2I9DRS3sr1DrUgW4ag65t7Q1l+jFjrMFYNWaLTHjbEaPpgy+wUi8DUw2VTisVxNFtpKcbi3d4G6E7+miECfVR9q5aeEk4pyP+lio1KTXYiiXYSb14rWDpGNsQk7R+8Q+5gVkmah4whGXHvIizNODDh5+JNuDDRB9ef9hDDY2p3GhoHdptWxHxO0tr0HMlrXfaBMObHYjosLHTRajesPTAZlETXu7Ukhh0gj/yOL/DO7UuWMvXsLQTT58COT7x488/cL1ooe50QFta5oosjv8/aX/WI0uWpdmB68wioqo23MHDIyODlZXJAoluoIEG+Pv5RqABNppkNVksVlVOMbn7ncxMBxE54+6Hre7JpyQ6wgEPwMPHa6Ymcs7e37fWHNVFKN3ik8Fby2UbXPOKN1o12WonJgemc1hmtq3zjU98eHqg90SpGWeFJS33GszA+wXsSrQN760OeodhDKcnxnvYElsx1lB6pRpDs1YR0enAbI7EcIDR2Z1nzz9St4zzFuctweqPPKJX3m3/Qq7vcH4ieEhPB8yW2bZNUdVGsTBbLry8fuLDu98Sg+WwLNxuF+Z0oKw7KagqLYbI2+sbv/6rv1K78pC7/HVgcHqiHZpOR7TpsMwzec/QGtFZctkZXh88yTnW2zf1HiTH6BO5bHer0Yk2dONlhqDjR8M0T5o/y4WU7F27plcOg9U+4his28ZymBTjs648Pz5Ti2Kza92ZY6LsGWOtlsUxuJhUn9Ya3ke29UouGwLM0/F+shzYOJGmpKcS6xCr8Qj9jzR38gi4dEBqwxruQdCC7JumDken9UESOMyRh4dn1rdX2n5lLZlpWTRXNgaLtdQ8sC5xmJ55nB+4mAv+oyVvDeN38LMagYwgUtQ0b4VlmWlFuNwutJaBfqer6tKhtp91aFoVEqssrlKrnrZaup8zNVMoWLw76ZY4aOat94oRnScPGZgGuRVKudxzaQ65NZb5GdhUdELGuwnrInYS9vVFcUsmahNkq1gDMSScaH0ISYy+0btQpejLa/wFSGnvLYuNpAFLCFiveanZn7DjFTEFLw6HfsB8Ew5uume5mn4xnSFZPQpGrLLdW6LtAesWjAk6VOw/Ex47dQhpjnRpbHbjJoPR1Js4z0+8f/o12/6NLX/ByGBZPFNaoE3IUyBNiX/60z9z3Vcul6x1JzGE6HHzRAiDKpoENzaQErSqWvNuDDISplm2remD0hvy+Y1S/onD/I59r4w+CF44ToEm+mAJ/ogPXV133WJ9RMTS6kY3nTI6fRRCMpi9sw2rV5q6coyPRGdI3jCqxYaJ2R/Y2LBYvA1Y8VirMJsQE3YErref8A8zzj4QknoXpRZa23SWUwrbfuP9u+84nU68fPuJIcLj0/tfpCPLsiAIrTeenp7xzvHy7RsxzYQQWFeVqPxcLxIR+tDyrfcBSSpnUGSS1pRC9Izqefm60pqhlZ2H04nrbWWvleANh3nWN3QteOvvM5dAmCKwa+G/KlHVGPNLrCFEh/OWUnTBUHLh9fWVaUpsu5IpTB0sy5FWd2rJpGki74VSMmXfGbXiraHXSmuFsq6EFInTxBgF42am+QEboma+Rldm3BSRaULkLmi26C0hRgyKh+lZ8d5byZzLynXfsQMeD4+sBaLZaF0FR3kryCKUUu9Av4nj4QN53GC8IctNibNN8CRN6tuNEDXsi3QshlodmInb7UK7PzyN1ZdmLYVa9bNunVIgRGBUqK6TnGE0gdbvDZqBdxMxBGwQrAHplbxvWitrwrbr59mQSZOG0p1rd8yWIcYjIgesJHrpBDvxcFTyx22/EaJmw0bXFoY1esrNNVNXwBnG2HHmL+hCHmfLKQROkyWGgo8Hth28LRyXE0NueGORrgnk0e6VGwwhzHRrCDbTq6KdjVNrtDcB0xKtBHw6YI2+NX0fWCMkZ8njLh2NhoinbTDHCW9mop/wy/e0nsn1J4xznI7vkbbgRsDbxG2rtNd/0ohG7/TuaBk22xjNEu5ZrVYNlobxlugsA706WxytJ87XG9bv1NF4vd5Y0leCi/QqWLF46zTtbnQo6vz92C2DIXL34ClOJ9xdksMMlqNjv1ZuvdNGpRvPcR74oEOW2ovOYMQSXCD5hWgdxgrtvi01RihjJ48rMR2Yw4Ttg3XbKTXrlaoNHh6eOR6feTt/otSVx8d3lLzTrHB6XPDOk5aDvhlb4eXlG7dt47fPj1wuF2JMIOGXE/YYg7ZXgveIKCMqBE8rGe90blmMxbqgDY0QGLljbSB6PWE5Z2mtMqVEkXI3q4e7MXwQkzLkeu+0orM0YwV3j9XEoDifWoZmwXrjetuUn98KDUsbYGxgdFGNmtMfFOcco1au60r0ni6DVuq9jwfSGikuHA4WE5PGI8SAQxczPkA6MUrGBI/shbuLljEsNi2kcKRtF+x5UE1nRxg26pA/6YKid9i3jdvmf2Hr+27xaSaVE1ILMnZKrYz+SheHDCV+iIUuA+8guEZMMyXr56ZLown3alPHDIM0dQKEyWOC6gFl6Ly0966VnyrkDqMFYpyJ3t+9jINh4p3JN8h1UBt6OkWzncbD3jZEHJ6EtxPL9J7j8owjcL2eyflCYWVJHmt3xaC3n+dtap6qTdirBTOD2fHmL5iBvXuc8cMzp4QdgxgMuXYwwjInttJopdKbhh3LXhHTEBcJQWccIXimZaaQFZsDjCF6ZO2B3nVNbvzgul8IYonBIEHluUK7J3rPjHHDIpS9Yp2l9c5WNqx3tH5jmQ9QIg+PC7/67tds/Y3bJvRWVCSwR9YONRuWZIgm4gWSH7SeMcOiEx2nSf9hGRIRozx2kY3Wdw24CkQTCX7B2kWvV9y7j6YjFMZQ27h1gBjmdMJwRIApVl65kbdG6Z26faa0Sm4db9Ug0y2IU/vOnI53q3cBU+hmx4UJ6YG321ecSRgeiNIxAdq1gwgPxxPWqii2tcrx9I5t08Lu6RSYp5lSK7OxvL2+EbyjjkaaJq7XM/N81FrKqNSiKO/eldI0BU1Wy2hYNJjKqBqY3DfmO6nCIJxOD+RNMdW1lPuq3bLn+1UBzf2IARc8PngdL1SN2Oxlx0hXqEBM1KK8+toaIEzT8gtbylrLertRcmNKUeMaYyB+sBwWbn3QjAZ897JjbdCH8dAhvVddNK1kVcQdD5jWYV3BWUbtmEmwwTMYjLcd3zbduI3KdtswcSZFz8M0s6ESVxmRx4fAWm98Pb9pwbkXvl0uTJOjSaW1ABR6F0bzjBpoe6Z1reshM2B0O2c6p+PElBZ603hLipa9NRgWJ9DagCzMA7rV/mM3HWM7MWo+S72RlZwre4HkHtm3nWUyeGMZFIUcjEqh09BNaAieabJYo7o7YWDsgjfvWNL3/PZXf4s3iW2rTE8Hcnni6+sPaqjynsaNGLQI7+0dbe09UiO5ql0+/J8VT///PsDmo8e3oFLbXBBzRsTinFpG7BCaDOqeOV9e2G9XYrLkriIEF5Qfbt0g+KDKdlMZUih1U6B/jUqkjAcIjdv1jVsG6TPOwTYMpRm6OF6u33h6fMFvAR8DOW+UnJmSZStvGLdgm6PXzDx5fvX8ntegG6deMiXfTUYjsDiVwHqvcwE/MoPMGHod7k2rKZN/woYTc9+p/Uqrq9YgpCl8TypjXDB+Qkygd6uaqFGBTmuDZZl0CNwV9GetYY6B5CJrKuzrSt4GQwJb0eBeG0XDfdFBSrjgMaYRjNVfQ8uIEcZolFw4SyRMuujo405zEJ2/md7BOI6HJ9b1Qi4rMR55fHzgdr3hY7yffBxPj498evlKTMrmPx5PvHz9RrjPVEreaa1yWI4Eb+61osGoBWu0/+IMRGfu1/sF4xqtdUYX9ADzL1cvkc62FZ1ZtcJyJ2D01vWhGIL2UQ3s+5V1XTkew52AYVSMi7DtGzIGMUamaWKaEt++fWO/iV5re2cdgzRNTCkSk2e9rQSv81sYOO8xztKBNhq1ZsK+Y+sJYegWVoYCDLcViREpFbfMjMuOGV0NP4cTt8uN2/ZKk6HbtXikrBVL4XTwXK4b3e+UfmZUTxm63HE2YbKnroGaD2ybcF31pIJ0xljv0QPL8fhALzPVCN4b4lRoNPKt69zKOqKz4NTtap3oM9BVvDN4A04arQp77lxvmSl95Lj8mndPz5wOia2+cFv/SG1v7LUwTCQkmJzDO93qOqvXwJor1jmOx0cepmeej9/jbaLXz5zPF67blT1vio9u+vNm7MAaXSBMLmgsxSzI3vThK3/BCcwHJWLKprmg8/oF6z2tG5yBZZpgeMqeebvtrNdKzELYOp3CctQfXGP06tF7wSaLjMBeVvZWSFE4TCeiiarSapktZ0y11GrYO+TRMS3Sa+anz7+nPwrGGbZ6pbbCtmtW6+3tE2bcaMVi0IfEWE6UkO5h15U2KrYbRBrGejqiW7khqqSiK+ytT7jgSOmoWS05IpzI5Y1W32AoIqaTwThGE4wbGBPobZBLobaVOSbmsDAts27cWsPOijgJyTBLJPqEOR7oObDXRm2WvaogeATHre1MJPyAWjudRh+FIRmxCWMjl/yZaCeMe4/zWt/AeJZJ+41uipTbesfgHHj34VeAI9fC8vSo7H4fuF1XTssj0SUeTifeXr6qsNY58qZSCSNqBc/7jZQm+n2pIb0jrRB9oCPs64VpjrTm2PMZ52DPl/uJUjVsvRU9HYk+aPZ9J0ZtVtA6EgTvHF3A+wlrDKU0nbl6Q4yB3gXvHbfbjdfXF7y3PJ6eeXp8x/X8VcO4MVL2je16Jt90nlZyuYs5nHZvsWAcISQw9wxUr9h91SjPAH88IK0h9YI1T2Ac1IyNCYb6qZwPnKYj/Zvw6etPrC1TTKbjEXaMueHjSq9faPJGzZ4xmgIk3QIlUorheimct0bJyo8HtVf/fP1zhjueaCCyI2ZljCvG6IZQusaIrDG6SPMgpmJHhWrJw9GroUmni2Wanvj+w7/jw+O/4enhmYfTRIyJ2q98e/0Df/zpP3HefyRS9fuN0887llo9BhXN6In4SK2D77/7yDEc+L0IL28/sW8XQspgd3yo96DqUFS6aMXJB0uSRAj9Dm/4Mx9gtVWM1Q3OPuC67bigZt4QZiZ7oo9Eqw5jXsn1poFG18h957HMxKTVI+sc1g4cMLqndkNuhVc+8asPnhAD0QfM8TtsvGkJ2IKTSL2tjNYJxnO7vhKDu1MwB/aeJymtw9jZzldatQQfCc4zT0diEE0KZ0sbO94pFqeJYoNrE53lOIvrnV4LYpLmWpwjBXc/vk8kv3DbHFv+DE7oWJKftbs3KtiBCxZTBzICjImyQTTgUqIPw7a+4WLHG0cIkRgeoCUaRlfTXHB2h59xxRj2bnFD6GVluAqu40xBuIEN5Oq5bEHdjUxM08ScFrxTS1DrwvV6JYTE48N7DJY97xxPJ6YpKU31XtKe55l5mXl7fSWmyDInLq9npjRxvVx5enqg9c6+r0zTrKDIYDSEeX0jnh4J1pHrpiXmmNjucxuhY42wrzemSTHU8zQzhnoZU9JWwM9J7tE7Jiacc+z7rnNG5xBR3X0IAW8ceV+xBsUy5Z238cLpdOJ0eqKUzHXdVNIiipG27q5QA8peMUmwNun1EUdMMy5Maou39j7rCYhJiPeI63dkecLRoTbE6+dcMBgRnp+/YxhL/vqJ3Deu9Y3SN/b8QufGbX1jqzdK0axkcE/M6YBHSRu1F9Ytq0ORO7IaFdGINazrDe+08O3jDazW07CN4SbcsBSBgcd5fTiUu2wEMThJSNX5bbAJ6xeCS5yOjzw/fc+vvvtAsJbL+RU3HGbAp5fByJ9xVonKfUSMzMzzMylMtLJyuaz8+ruEjxOtNuZ44GF+ZPIT0grV3DBugM0Ild6NSrO7QhSbDM1T0sHUP/8B1iRjiFQT2eSeIrcObxvOWp6Ov2LLgh1HjvMLb+4rdMPohpI9l8vOEQ8SsPcck5eIdxPbsFQaZb9yfntjipF0fGCyE1N65OK+8bp9wXnLFA/kYZisxUmn1JU0J1q3xHDA2sqWM9Ef6DJYy07owhI9U5pwPtCHAWuoXYURziX6QDVe/kSpA4Ymh7mHOEWEkjMWT3Az4LAmEVxltWdEiqKyZSaGpMq0UTCm4bzH1vu63XgogdINbcCwlrzvRN+ZYmCYqvOeAI6Op5MQXY7QoAulWmor9H7FxQFNt0fWZHAr1s/0psXa2U8s8wFvF+pQiel+U0pEH43btnJwypNPMVH2jUmPmfdlBOSiTscpTdwuV7wBMyoyKuaeMwrBkIKj5Kpl/bIiQ2d/zgR62zV2YLz+utwCxnC7rRyWhXW7clgOtKYVJGPtXeM3fnlQ/Vwat1abAj9/X4zRTub1csM5o1b0+2/O6Ob3ci5qZvJOQ62XC1aGjkRKx7moYmaUVWed16iB8icQHzRCIVbxTzEgZmCCBjzFesbtpiFbL5ih7CvBQO9seaWNhksBcqH0ja9vPxK8IfgjUR54e828rJltL7rEeXAYb+9Xp6onFR+wuLunU2F/P8uDSx2INGzdELPRRWenfVQGjeEC3QzEOI7BaUrALxhv7jNeQ60eEYf1g23/iU9frUpIvOP54YDzmprf942+d+wQXbZZEDszpd9gw4khleZgk2/84dN/4uPHDxgvOGN4OC18ePfE+RbZW1O4p5lA0Pl53ihDaNKwTsCJ5sH+khPYkEZuG0MiIS130mIH2YnhwJQWat+xvpFCxHsNrDI8jK7rf5OwOLzr+FaxVZjnhRgCqUeCfaTxjcvthdOsfkJvjszxPUUKl/1HTJyYRmKO/o46EdWhTYowdl6HubWuDGfoAvRBG5bahWnyJFkYsmPdRBtqFscYqB4bLCkc2bJiWuQeyJQxuG031lyY4oklLRjU8hPMghjPnJ5hKO7F2kRyE+Iard7AvJDbzoGEDBV71jHIA5qAczfKXImhMExj37qy3f3M+9P3pOS5XC7s241eO+Dow9P2G94LZYMwj7vJp2NcVl57SAxnWLcXbrc3pA/o+mPpfeJ0fGCaF5yz7PuNp6dnpA227czy8Ezrg8l77Zde9Hrtg0ekkaIl5xVjBsdl4XZ9YbtdeFhmzvsV+3NQsimRd982vQq4u0dRPHkYMMqKKkU7j8YY0jzrdbHpC9I5S++DNM/crhe2fWOa5ztS/P6g87odbV17lN57mrUUJ7TauLyeiSnqieh0UH5Zc5R9pbdNwQQ/87PKrrcFOxOcwzgP1ml63UdN3lsQ78EEfXAET68e4/s9G6d+hdIq1+3Kp7fP7DSGsZQ2uO0rS5rxdmJyj+T1lbe3ipjEcTrgWsJ57tfJjEeoVovxxhpcBE/A/dwTuMceVLTTEWtod8Z8r4bgHadk+Xh84P3hwDx5ra414bZXznvjMgq3kmlr4SovnK+fqOXG7e0bp1PCRsvb7U/88O0fqOUzsy0s1mr0gQAjY8ys/B9XMH7lhx//PR/fPXL4zUwWQylX7TdauTcujS7DelRSTC8wNBc6esFZCM4xTPnzH2C5fMPaE9ZGopuY4gzWUreJ6BUnW4GX7SvFvXE4OcVilK45GVEsNM5pCNFpkjw6YZofyWPiMAvXkqjjK2/Xr/zm8Ey0E8v0yLD/jn/84UI1N3wMxDAze2XPN1QdZeRnDVfgtt6oRRPiTfQNZInQE84MPEGrDcawV23eJx+R5vAhcUyebb9Q5IyYjJBUArut3NZCPyijuw9FUYcwk8JMcEeu28qQjpGA8wvH5REjkdvtG+f1lZ463s0MOrUqNLEaJZ1OAYLtlAzGRZZwYPbvWdLE7D+yT1deX3/Cys6ojfP1Qkx6WjE2YqTjTCWkgowrWa70vSJbJcXE6PprSNPC48M7atUytAi8e/dE74UfP33SStP7jxgGl7evwFB6g9fvnYjOC7fbxuFwoOw7tWbmZUKkkfdMDEoaAEtICUET9H2aCMGxbZXn5ydkdOKdy99rpRujJp1eCdZxPr+RpgnvAsZYnt594HK58PL2ireGw5SISSMZ1ji4X7+tQ7NLGMzILJN6F3strOsFlyaiA5pnH4a8bTQGsVukT2wYQlTPozOCcSAuai/VDD2dxwnjDvTrN6x3+PkIUvm5M2J6waWIMZ5b3vhWLkhciPGB0+FvcFSsdJbYcT7SRWtUxlgc+hzoVlP+0UfMuAufvcM5Q7SBZJXGC0Ij0sRz3QZ7fUOC4Krl4E68n0/89bt3/ObxmePDhPeDtVY+XzalPdw6e9m5XlfyfkWwpHnH+3/AT4XtzVL7zmX7E5/ffgd0jlF/pqeoLDFrnzku72jdcb1VrL3h3Jl//N3/QpQjXhRM2VrBpwUvj3c36tCHlp04BEcbgHV0EzG+MkQ9C3/2A8yZR0rt+JDBNVrfoFuMcTw+fODD829Yjplv1y/g3jg+DHod1B1ybogEWu/44Wm7xaekA8gOyXsO8YG1NrzrFBH2/kLebxxP32Gl87x8ZH/6r/nTt/+AHRZnOtYVXFiUqnq3wCBWi+Gj0lsF1A9Y6sppKlhmgl9o5X7PpukspBSGGNL0qAgf67S+4TxbycrmHwMRRy2NEvQtv+aMsYXHeNK1u40EX1nzmRhOOKJC5Z4PnJb3rNuPDFNYR6E1oZTGVjaMaVprapltZDUad4fnwMPsqZs6Cz88PPA0PfL15Xf8WM73AGZjNKjZcjzpiZQwGO7KNr4xmWem+YDFMai8e/cRY7zWe0JQhM7Dg4ZWXz+x7mf++rf/LaUWtuuZCWE+LQSHCje6YlZK3jgcD8TgaaVgRXN73778qIVnN6mxqulJtbTK7D3zPDFGp5QdUCQyv/QplWNfshbJ8UHltaXip4Cxntw68+GE9Z7zy1cu1wsPzpGiBmuv20YMgRA0mxbjxLbtvFzOPDyeMDHg5MDt9QXpTTlah4UxzbS2I9JoIsyKFdFIRa/6u62IW5DlhBiD5A7mhvMBhr4wyVXJrB4kJpwI7959II9Ce/3EuQ56hTl+5Onk2C9vlAzLdCTYxBC43M6cgg7CjRO6qYq+9gr5815wfmB1gouTiA+R2R3p4xnvJy7XPyBjY3Hw5CIfQuTRGqId0DN7blzXzHnd+Pr2xuVauF1WyrpjxyBEy2wCjsFevuKMpzYNAFvj6SKsdQM7KCRO80dcdOR2xRshTQ27C4e0sN7O/NMf/iPJJkg6hgnpPQ/+yLq+cb5+0b/HJKDqVRdHtY3cOhVtzPzZDzDLd1hZyduOiytlyP1hkXh8+J5ff/xbtnLl5fKNHz7/jxip2D6xe8MeOmXzqpOvFRcmhglYr+iMkgs+daaYqCNiOWBF2NYzbc6atcFiR2LyT2AKpms3rUllDBgCroMb4Kwn+sSGgtR89Bg6nRXDTDAzU/SU28DQVZ1phNHeGALRfSSEiA+OZV64XL9wbhu9NozRZPIYStDo3XC9nvVEGDwigykF9jIY48Y8PWIJDCMcTx+YDpHz7Ud6/kZoHTqsW6UbwCeVo9pBLyslF758/glrJp5PHzlO3+Gw+Bh49/TMev1Evgbe1nafz2g3c15mnZEki9SBHYHgAsFEogvUvVBbZlmWX2Zf3nnO1zdqqXz8+GtEGtt157t3z9wur+Scddaj9gpaqzhrkVZ19b/emJKj7MJ+U47aQMWyQ4S36wVrhroJsNoRHIOSd/rPZXFzt6OnhA8aQelNsFFI6Q65dIYujbpnrIHvPn7HtukJwt/FJk4sue4qkRDDaIpmMcDr1xflz6eAsaqFswCSCDES44kYPS7o4ifGhE0RQgTxSM2Y3jEuaj/QNFhXhHvp3AeGAbuDlAyt0Gtlv950kD8t/Oc//JEbg1Z3RheiT3if7gDLRKeR88aXt0+IGVjvcF5nWXPKyuC/C1+x6mBoxnAMMyEkbHUc3GB5jIz9J2a7MotjlM75fCOXneAstyp8vWU+bxde8pWcK6MMjtFwCJZl8aTFILEi9UY1g70OSilItSCe4TJb1iVDSJUH7zFt4BIEo/GM5tX9+PXrD4gLvHv3V/j5iL/PvqxUHAnqihlD/3+pVKPorCweSyJw+vMfYKMnLIFeBrdyw/lG3YXT8p7T/MDTw3tCnTgcPmBNwNmGE4e1EKPHhAUXDuy90obSKcRECp2t7Zh8xjktYrc6cC4hGL69fSW5B6QJA4uXE4M3hslUKuaudBtYMpsOX50nusTjFFm3DVzX61rfKKzUolkYNWlXrNOtZqNS2w0XFrxRbdsUFp5OT/zp5Qs/fPkTHb3uhDhhXMC7R5ALX759Y4oPeLOTkuPh+MC6vbFuPzDF74jhSTucfubd6Tdca2QvfyTIhvEOcQdi+ECcHGVc2fKqTKh8Y//9P3J9eqH2lafDA4dkGbVgBIyJ0IY6GB20HBjVY8URbMQFQy+ZvXmVkrZBEMu8TMSodaMRVC/WamVaDpRaqfXC97/5ay7XM2MMDj5wvV5ovdF6Y4qO4+HA7Xq+z6A6wSfytuF9YF4S+77i8KQpYC4eZzVMmVJgChN+jSDQS8E5B8Zgg0ecoq/TpLWhIYM2BqY3bNOsV6+V0gqbCIfjkeQXyrYhXcglM2qjSUWcEHwEBOs9vTXyvrFvV3U33rN0GIOxFhcCxlpiSqTDSVPohwNuUlKvSRPjXhUSARsmOEzQbox9xeaCOzzSjg+460b9+ifOr1/4ev7MeRi23jhfr7ztwlZ25umEbZ3adlwahKReyE5nzZnaNTs2pYRx+mt3w+BsArPoUoiBZWct30iTx5kZxoQxDRsiyVTMiFy2yrfLjQ21HtXm2BtsZIqtMCrBWw6T55QGx5PBTUJhUHqnlE7ehdYcjElnm93TrTYJXt6+sfjPPC1PyLBIW7HSqHnQyqLdUzLX8xsHf2LvnUFl3XZ6E2YcplXGqGQau+0073E8qriEvwBouO0r3s4Ed6CWTVe7uTNiJzqnqJTh1LibCymCR2GD0QUMM2IS0+FEH8JA4WbGLwznKBVYb+y9svdMiIEwTeyl0NiwxhHCkXeHwNdzp/adPnasHRjRgq2wUrtjyMIcFg7+wCk88G39ytq/knPV9ffPXHznVFLQq4LouqdTqO0bYEh+IcUnTssRF565bjvfXn4EMchQd9+UDsRt4bp/4tvbC4zO3CwxDmCj1lfKvhHiG4fDE97ORP/Mr949czWWzz/8PbZAPB1Ylifm48y1WV7Xb3QKvWhh+XNbMUOw3/+aNVcMKxIqPhr1AZaiFqa2QD3RC7RwxrlOE33ABQmEO3sfa9lrIU3qfXw7n+8Ri8w0LXz38SO969v2w/v39JJ5O79gjeXx6ZnotRI1zZFadx5ODzrDGoMp6QPj5/iDCBwOB1JMXK9v9AHWJUJadO6XEqUqF80Gi3We1geYQYyJ6KPGHQL6YGoVaQMfAs77Ow7JEqfEermSr2cQURDl8IwhOqe701N70d+NRZshRQfrunW9G9jzTvQOPx/wIWGc189t6ziXMDjoFdMNIy1o/3hC8s5YL7j5AKcT3nyP2Vd+/OHMrTT+6acfeNtWvlxeyZLZ66+ZjeWar+x51cpMVN7bKA07dNuIQK+GHUhyF+CaQe/grOCnnWFvrE2Yw3v2XDBjZ/KWiqUOuAzhmhu3VhGnpf1hBYdltk5pJwghCHFSajRGqLWxbo3bbSMXRymiJ0Pr6aIs/d4t1lS+fPtnRvvKMRxw0nFDXzq9ZIWKGmHfPvH5BSAwRJFcYWzqCnUdpLObTqZRm+dpeiBNMzH9BUFW7q624AN2eLas1Rixhtw7l/WVW9m5rJ81+t8NBBUCJHfA2yPGCZkN47UWYEU55gZHr4ocvt5eWOuOj0dObuGQZhyO5BMWg7ULy/SB/HaDfsWarm9219hrY9TMHHa8m1jSQjjOjFdh/fpCrvqA9PZZed7JEm3H2kEd2v2qIthWqeaG7J95PH2gZoPnwNPhkW37AdNWhBmRSTd5h3cMdq63V5wxrB7muSPsMAb7/pVy+ZE2/iueHv4GK4IzE++e/28YeeB3//C/su+ZhxSZppnWHgg+cus7ay5EiVibuFwuuNAhrmBXpVUeLItA3COlDZVSDE/JHfEb1RV+PoAbeyL4QC4NUyH4wPVyYVt3BM25zcsDh8OCGOHt7Y3D4cToUKuSLZ4enpnmCWcNSNMrlrWEFMnbyugDH72SVI3y3Q2OGNXZGNOMNRaw+JDwfmAYRJcUzUyndHUu1jLItWCsMqcER9kb0qvKLEy8b38D4y6eT9MEpXC9fVHSRFiwaWLfDWE+EAN0LIQ7WdYaXNQrMVYpEzElnEE3azIw0ul7hnnBTwu97bjpCTGGXlfwFnEBLNjlVwzzFfZX3cgFz9Nv/46P52+8/uEfEIEfPv2ec7tgXOP3P12Z7ERrnbfbmdKq/nNsJxwt9KhXxabLCR/MPa2urs/ejJI8JBOcWppu7NQ+Y2Qg3mCYyb2zDsNtGEq3+rMUVLVmzMA5cF4DsSMaVgcd5eOta+fyJqy3yp7z3cLucCFgfUJMo9ZO9I6y71zYcYdMdBOtWlx3uKG2I+scrWferp/oXQPj0UOYoNsd0qA6qHe0dIgW8ZmQPPPhL6BR5PXKPAtDAt4l/UG4l6O3feX1+sJt27mcv2obA8feBtkUst05BkPwHWc7296wJhBtpI0VHw6IWMQMasts+Ur0hlgiwQnRR/1iWk8fVucF8QnJEETDelMytNFoUsi9kvuRD/O/ZQnvMP7EdVv58u0fkdHAZpUXNEuj4eOAINorzJVuO46N29r49O2PHGcN0YlszGmh0xlyY8vg7YT3lsM8se475+tXrPVsuxAj9H4nKZTCHz79R7Y6+PWH/zs9WiyJ+fG3fPivIp++/CO3cuPgj/geMM3RdmG0QO5CjJ7RLH0PzPEd3QRKPePcIJwsfgLZLDGciOEjk7GM/o1uboxxwbYJ5w6w79gqHNJMyRsYQ5oXEOV7zctCqY3lYOmjUJpnnj2fP/+IMYblcMA7j/eWUpUUW3LBt8Fe2/0BWbDOU2shdq229NHvpxulWIyhJ6jRGyJqsNK+q16Z3J3CmrdCaZXaK80ZRgPEoj9pnV4zvVbS5EEGtTRM8CzHEy1Xpjjh08wwjnY3iAsqLa61YY1qzYwzd98m9A4hTaRpwd6v6EhVhdqsfsphwKR3iPXI9gLpAXxEpMH0jh48/sc/IeWGCZbvP3zgy9cz/6X+gUs5U9pOmh23euU1X5EeaN0wzd/p9tTpVTsYjxmOkf2dRSb0nglOLV55dPYq7Haol9MPbKgY84S4iWZ1u1vMyloqrQ5q6Qyj8R3rHM5HjLFMk8Garo0UMZRWGb2TN89669yulVybnghNv/93eoyLtL6z5x0rymlDCkty9BEoY2C9xXeLWId0R2lZC+jGglUIRDedbgzVWvY2aEaIPhPmFZs6uH/d6vGvXyHXK5ZGjAcaTcUEQ0UW63rm2+tP7LVR8o3oLD7ODNEnKVJYe8b0VQmowyFjxsTDHR0846wwZJCWGTeuVFnBzIhxCI3SKtYFZCjv63T4QHATYVzJ7SvDdMVJm04bja1eGayE+IGje8eH97/h7fIDteox1gjUUuh1I2Jxk3K7bOgM03FeYXk/vfyeb+cL1lSC73g8DW3zq7NX32AhRd4v33G7Zt7OFx1AD4szygxr3VFb59OXfwRj+P7df80hfKR3h/UTj4/v9cplA9ZEgj3Q6hu9GnrRnuUcDjACvViIjmGVbOusMHzHzYFpfmB2M5NLVDpbO+PMjT4e9a8dnuAn9rKRc+bh4UnfjNYxzzPn8xvH44mcM7f1zPE48enTn9j2Kx/ff8c8zVinVZ35sCiaxXlqGzifEOn6knCOkGZaN1jbgYK18ReCK/dhfd5FU+Uu6KD9Dke0IZBLx06RNAZ1X9lzJU3xXiR39FqxUu+kUK/D+DHoZuBDYp4etFXhnboQ74HPPnTOVGsjbzdCMAQftIAtkeoEbwfFoQZ2E7UeFNWU9XO1SBBsOIBUyvpCODxjgpI0rEuMxyeuf3olv72yeg/RcTo8cEpP4JRQCgMXAtgjczqQwkQIFudhmjzBW6RHWolYtHs42oWRX7jtLzQaow22yyAmi8QVJ4OUHIekW/CyN2rRIf11LdQxMBYGg+PR69LCCykkxQehWbPWK/0uIPmZPuKswblBmhzWKzLcNIN3ibxl9ltmmQKlV0r3GAalN/2cW0sT0QccPwMSOzU31v7zIVPIMmjW4JLDp0aaM86rf/bPfoCB57ZeqSXjzIy4GUgIlq3c8LcDuew4LE+HX9OtYS8rrTcMK2NAGTdM67jxRCuN4Q0mBjoFwTB8JJpH3j0Kt/JCH7d7z00ltm04ugjWeubwhDRLLh1xRwwbHgHxNOnUtvF6/czp8B2dzrx45tNEv52xw2q+pgrDGIKNdGMZzsAoeGuZ/JESBlvrbNsL0VrcnLSnZizGCM4rebWPwsPpiSkm3p0Mwf7I12+facbfUbvaW/RRZbcv5x8YvfH+tJLsM04qyTuOD+95//CRs7li3AOjfdIqyRi8XVe8dxxOE75pfaZjEWfovuvX2QjN7JigxFdpWZPSHow3d3KmikdarRgx5NrIdeVwWFhvuk2b58Sf/vh7vvv4gZ9+/BFH5zjrD9eQwe120+vXlnl4eNAPo/Rfvjc+zgTvVcVm7uy/e7oeY6hNuV7BBeys6/gB2KCxFemozssWvL8P2a3n67dPfP/r7xADTRqmKyZm8p7ehtIV7uIRYywdIQR7f2mBt4E2Gr011vVM3lakKuLGWCVojGnGtMIoO2OpJGPZjOPwtGB9gBAU0jgKUgsGhwkTfhmQrwgTxuu1k+OJ8OE3vPzTlb0VXteVP3z9ka3vGOP1a3MXZ3ijkhJj/f0B7xEU/jmGIcaoeKBaSZIw/sgqr5jeGa0o8cSjkg1bGCbTxoU+FvZi2PfCtmdu20b/eXRjDWICNlhCUhwUDfpAiS2tIUO7xN4OhV0mw9ZWCFHFxhXKprirbVOr1OhKrUC0xteb0GvD2kUROwPMUJKJFV3I7BZWC1UqLibS0RFT53j0mKhCmLqHP/8B5iz00sm563DPW4zzOJt0OM4bmIFzkY/Pf8PeNm7xjXFeEXNTNAoaZi1bIYQJGkSbiHHSMq0f6jw2Jx5jpI4rte4s6UhwExZDMwaGolJiDLxcKqVWQgITddtobWS0ndfb7zlcDkT/njauWC/4MLA9U5tndM/oBzYRovG6leKkHK/sCd5jpk41FfMzt6qLMsatYPwFa0EGxGnGO4sPjo8fjoyxknO9Dzy9kmpNYJ4i0VtqvvDH7f9gmZ45mQVvDe+e33Oc39OrdujKKBhn9Ws99Kr+8vKZx8eIn6H1Th6FkKyiSyZP66+s3eKI9L4hBmYbSEGQUBhStTbSK8EF8p5JMVD3jVo7j8/v+fz5M1Oa+Pz5s54KnSGmyJYzYZr1Adh0G9v7YJo0TuJcoI9OCJHaKrmpb9N6Q7hDEH/+zfx82qoNaR3nPD4mPXFbEGMU+2wM+3WntM553TjcNh4O8x2ot3OYEnU0gp3oIsQ7X8veT3pqRzLUu+3aOce6bexbRu6xECOGedIQ8KiZYnQB4Xyi1kaUQilXUlbem7UPSPRIvkDeYfLY5QBh0UF13XBdWW5hDEyY+PLllfN200D1cFzrBesszg5qFwZnZHS9nopBbo5gPctskWEJPpPSdG9RwGiNUSqWSvLjzlDTwK2VjsiNOhKSoRRP70arWa3dkUL8Ag2cRGtZiFqy6l5V2jO0hxlt0tOfARO93mCGQYa7L3pQfV6GXMBax5TUXG4YtDLoxejPHp1gB8kPaHoKvjZYd2FvHRcNh6kTvLBER7QGqTpjW7e/oEp0mCOZyr5Waqu6/TON63rm7fKFbTvjoqX2DW+iopG9Z/XurlDr95yPh9bvQ2BH2RWV0nujj5sCCu1CbYNoDrjR6XvDTXpVM6Or1bq80drGlt9Y91dSN8TxM7e84xPU0Phy+YFDyuRcwWhkwpDvV9OIYKmjIrkhZD3eRq925clC73irUglEt2q5bjSzY/uKWMcYhutaeDr8hi6dGIVf//oDZkR++vLC2+3lTqXoPBwSi4t4L7xez/zw499TpiceTx9U0ipKMcj1RngAyV27ownmAEKmyYRp0LuKFIbxuBjxZmJZEqWd2feduhlSXDCLwfKN5I73/JADo0PVIY28NWrOWOu5Xq+KNdHzPdOUVOUVI9Z6xQ8jev3LmXmeiXHicj0zLzN26FXDdsEY/d43NaQoyDHGeyI/UFvXzZ93WKcUVnH6kKitM3D6IKyd67qCseRcuAIxKB5oGLWqD3Tg3kT/nNzx03AfyWB5e3tljE6KgRXl7T8dH9W8Hi3zNBGd8sesMVhnwCjJ1LRO6iDS4faK1CvmOClMcL8RQmT4RckMLast/Jb59uNPXLYdmw6ctzeq3PCTQ+pg25RsMYaAZLxfMQRVu5Wi2rH0L0P74D2HMHHyCW87woa1He+EYAx9NHoBawMydoa74VkQHM54Jd3qd/8O2TTseyZtP3+vhFEbtQ72oreivBckghmOGHTjaDAahwJGN8iwjK5cMhGQIb9s6vetULOCERODJTSWZIl20P1Oo1E3x5ZVWBeixYSdEALOKP66C7Qs7NtfgtNJE10M9DecVX2X9EpeMyWrqWTsN3J+JYZZrcIUrZ6EhHH6RKcawrCUvdG3wl6/cZChNFOLnhrEQHf05hAzyKMx2Blm0HolhcFohj1v5H5l3c/aChh61RTbEWPAd2K83eF0DmN15gCVEBVH4qxn2MCw/Zcveq2K2CF3jNVjPDhKt3QTaNZw3a5s1TAlhzWRl7cbry8XHqYH3j+eeDw8sqT3HE7P/OGH3zNeP1O2rLquw0wwnofZcztfuNzOHE/P3NYVQuCyvVDqNx5mkMmjdcwOm+U4PTAvzwoObAJboVeLjwcMB6yZOMRE2b7SpFEvBZFOnS1zBCExSSL4iJTBGBUZnq1UTqdHStYcW2mF0+nhXhhWDVdMiVqVnFpro5SKD5HcB3VAxNwptoZhDThH6+PeV4xwfwCmlLSjSQMxWqQPERsizgVVy+ddT3RdKaTbtuo8Sjqfv3zm48ePYB3bnpknXRwEd0cgB3/fxirVU+W8eiL68Y+/5/nxgY8ffsXl9oK1hsP0SIpRT4jOEUMA0SG1+AAh4eOMMZFeCozCuFZsO+FOz9im8Q4TIpiE8QYZV7CONho/fP2RC1e6aD91H2faOFN6ZdSIs4rvcfeXixUl0rfRWNeBD4YYdeZsmnZ4U7CYoDJm4yt2WEoxrLfCwQZ1MYjFeoOzFotXYUbYcKbQncE7vVm12mgVfXl25eTXPGgjUXbouXOYNGTsxOJthD6oVWdZP3dVQ3BglNgx+qDkxhiWWgJjKDswLpZpqtBXqjTKUE7/GAOXBj5afFTsVqkgTRcBrTlq+wuAhrgZlwLRdLxzUAN1Hzw8vMcZ3SCu+8rX62esc0zmiPMebw/0asB0nLOAKJtcNra94JjY28rpMBOswxPI+4UxnJLrGKxbxcUGVokMxlxJMVJrpkvDeUuXQWlDryXdovYoIfqOnUUDtT7hD4OyFQYDN2n5tnBv8TtHcGqV9tHhjJphxhCGCMlp/yzYwNE/MapBWsAtjzA2fvz0ibd0xrjvOT58x+npxJN9wntLHYUydYJRs7Qn4qOwHB8oWUO5636mu866XwjRqzQjgrWDYC2cEsfpPd480Eth9Ey3ETEWstXqjkTClDiEI2V945LfGPe3rZiKNwU7NgJBZbW10orKVowRDJV9z8xpwtvAaEMfONZhrVElfO94F0mzbvdqR//5/S45VowH4S7BFeNw4WeZRtCAkQm4++ncDh1HGKvEh363c5txr28N7lLdmeu6spdK7dBKp+Yb1h6wMbC3TdV+zjGHQIxRU/qt/UK1WJaZ9XpmSpHj8QlrA9Y28A43T/i4qHXcwLRMxOnAPM3EdGSIR7Hs/S6vuCKAe35mBI+0K8afwB4RClYKp+Mzxv4znz79M7VcOU5Hat0Z/Y0hmb0bBEsIymAzojEY2yIMPSWKKPZZh+0zXRz5Dl5siEITBKRbejPkm8FOM2ImulPL9hg7QlMAghPS1Emzu3P+lD4syibHB0uI0LNGU67XG8FMiBNdlNwf8GM0ncWORoyKnFb3o5KWaxV6VQKttSgw1FWKGTRp7GVw3hvbGNjkWI6eKXU12A+U4d8ahvu4Z//XH1H/FzgdAy7iw4x3A2cDHjjOM9E5+rD0bClbx8VOiBXfI86eqH1Q1otmOgTa0G0brlDaDamO0CtwYgyd7dRSFK7WdDA7xj2mc/8AxQDWdXJV7I3mHgWLyjHGGNRckUtGMMQw4SxEH5keA/k2GC5APNHdoEsg1wvewjIfmacFKRvn84VS+/0hIHjnWfyMsQlrZ7x7phE5TJXhDdftD9xule1mMB9njkvg4+Mz53cf+enbFw7TgZgSiEOqJU4TNg78ZOgms+8CI/B8+jd8LoPRr6RgcVawIRKXmYhFfKLXiSCq391y41qvzPPMqB4hEeyJGBt1ZHK2ROsgGJzVq+oYXU8sXp2L66YSiKen9xgXsHfETG2Do4+8vr4ioqRT5x3L4UQfRt/gd45aa3r1VJzzHQcdAoLcv4d6krDe0EQUr5PusEvrlQHVNCWR70uF0hpYT26Vba8Ijtya+iardm4lDFrtYIViHclpn9Vay77vlKy028fHJ1pOiBrp8NHC0Plnbw1hJbpZ4x4+kGJEjKhAw2j2IDiH2IjtO1KuSD1inGNsL2DPGL/g4oSYKyVveGfUoVorXhJP0/dYU8F9xhpBhsVawfqGI2HthBFPb+ApvzwYnPMYH+jdsFcdbBvTcPeTJ0Ot36NYhouIi7S7dKXLjhmNYAzGCCFZUgJr1EvqbMD5SZdmTuEE9nalt5Vb6dyuhcfTDOKxBKIX+qh0EZz16hmNqqQbIzOaSnBqHbQmxKBYnuE6W6vkfXC5Dl5vBnGew9ERFkOc1EsxmlqU2ghYEUbv2H99BPZ/pVUDkPsmTYerrRldzfsHWqlse6ZnITg1yrgYFFPTHHkXes14Z6ilIAycN3h/n8eIUMhYKYh12OAV2jeELqIzkVoYrSPWKq0xVR32OndXkxuSRGQ4RAreTSCWfbW42TNNEykq6+hxjszT9xxOvya6wHV948ev/8z18sKcFAFkLPeQpcUNB21gvMWmoFajocID4aCbtHcbcr3izET0R2yfAUVqz3PE2wYBfFBJhZ9gYtYfWG/oYye6hVILfkx488BtPeO65ZAC+EGpFwUYiuGYIBye2ZtBbhtvbyvX28rhOJHikWEsKRyY7aJX4gYEq3GFYcE1XLW0lrltK6015uXnpoQHG7lur3z88JFvX95oY+Ph8ZGBx3iHv2e3zBh4awk+qDnIWfayM6qGjJ01TCmx5pX0M5NN7pERp74D6zUFX7LO5lpr5NoRFEX+8wC6lMrxcOJ6/oaTRnCBXjt53XCo9Tn3DX8/1ez7hvRC2/SzJwOW46MWoo0jGoOLXqOLo0KurHkjHRa9aQAxTcqai0fydqGWnfj4DoiQN2R9xYRn7OEdXL+yf/kdzUesOL5db7x9e+P2+oVrFjoH5rCQ0geWfmGMC0MmxZbbijXahz04h3OVdXfqSHQR72awgTo6e+sMURpqih5j7ix5o4Hh0QK9KEV20BRLbhshFgKD6TBhYsWiRyYxMKVEihPBedrUka63lTDpHLJWBQUs3pKMIZvBkMYUF5xdsOI5TJbSrpRyoXadl49mkdAZeJoIuVbWvXO+dPpwLMfANBnmSYh+YICchdYdTTzRWOzo2PEXDPGNrfRm8M7eqQw6H9pbod1eyftG743j4ZHlqFkhH1Uv7gP4qijqEXRFbpzFetWtDRFaXxlSSW4mTBGHWmx81nlVK46260Nka4MuhVIGwwjWDJ3pCIhtxASHMPO3f/Xv+Lt/+9/y9PxA65Xb7Yzx7T5P2DjMj3z34TdYn/j0+Sut71zPF25XJXbK6EjrBAl4gdKHln9Fw3/eCrWdiVNgzztCYfaJ4BZiPAIJMzyjGsLdpbfvK3NKeOOZ04EpPtDaDTE7tV5xLjGsBjUNliAHbHbUHewMzV0Q14kSWJxlThNhcWzGkIvler5yPW+0w8BHXXUz5I5fiYwewQXFzliHDUJrBUSDkSKiEZep8tPn3zFPM5+/fKL3zuFwonULBpyN9+hCv2/AFMvjvVp1eu+UVpA+EOMRMxjG03FEnyhFiRatKhbZWqWBtpbVPtT6fQa20ZrQh2XbKq9vr5wOJ263G0t0LNNCaYVRd5wRgrOUkql5xTnhdjlDKzgTab2wLMv9OqOnM91Umvu20t7tSnot66JNEuNnOgEjg3R6Zlu/4faMnRIsM2ZkTGlISsjD3+Hb7/jn//A/87sfvnJtN356+z3n9cLaoI5Bq5VuG10sJq0k1+llRkbC2YERD81jzKyd3XzDmnj3M3R6H4zeqaPTrAU61hqMjwSjXDRntNlQ+q5zMjdIs8HPUdV0U8BFjba0MRAMLkaOx2cejh+Y4pHH4xei/0d6+YFt7boRd4bcGmLqXXZS8CaRZg/dE7xDSOz7lVqE2oyKenD3HJ4jZ2HPlWEgRsuSOscEyWk2rDbBiLsj8NVyH5xGSf7sB1iXphu72hBpjA70gAkT6U7AnEdFUIMMRiFyihfXD0bv0PpOFc1S/fzWFXTO4QDvItHDFBeiP7Hnlcv1lR4GEu5vqtroY6GORB2ZUjO1gRmebag84/t3H/ntx3/L3/7q7/ir337Hmjeu+yuX6ze27UJ16r7qJdMa7Hth3yr7vrPvK1INvWWmblmSsqkGwu4q0U3kVqBF+rhQRmYfhSYrta5IP5H3zHa7EMOENZZjfGIK33jdzry5wWl64OHwV1gzM8bKLf+RYTb2/g2xE+IqIRr8OBDEw50CsN0azlfEWu0C9oIPgdODnna9N2yXK71VQhCccTqTEouBX1blmotKDCtKOxiBUjIYaDXz5dOPLIcDPSTO5zPH03LvKKJXotrZd/Va1lo4zDOtN0Xf1HK/wndSTLQ2uF5XnPPEpHOAMSrIwDmNVogYSinkrFy0MfRKuN8XH9uu13nnPOvtRmuNZoRWKrfLizo5a2YKivlxnHj9On5RtsUpMob9JVah/w7B2gD32AXoiiemhAs61A9xpsqgbBdC3ZgfP5AeHrHXDVN25JjAH5HRMW2jh4nw+JF33/+G/+k//yP/+z/9v6n+jVt54VLUNHTzE2C4lQthMYRjI0xK9ZDe8P6goV8rxDbTWrvDCoUmnVIbuWRKq/iYdIiOuluN1xM2CKW3+zZRr3DeRlJITIsissUaRu9I3TUa4SLL8R0fPv4NpkUsD3g7s98ar/ZVv1bGknvFWEG6RbphWzeCO4CoT3SIcu17U7Wc81rUxwxahbwLpci9NG8IYeCdwdhBHR2xTj0OztAkYbvFWkVM/9kPsHW9YSVgEJy19KbdOYtn1IGzVu1EeIyxlLpT+k7onmkKHA8HvBuse2PrKx2LbYrk8B6dATBwdsf5meNyZArPTOnAnlf2eib4hGPB+a7k0er1dNI6pe4Ec7xvYxqfv77x+vbC5XbhfFnIfed8vXHLWkx9e/tGriufT9/ozfHjp2/84Yd/Zt8LxkUVbVg1YLfRuWw3Vho1KC/fGIfphonGyKteYVsljcD2euFy+IoXQcyBUwocD+94enjlut9opVL8QGwkTQ/0YblVS+8FA7TaqT3jkzDEQ9ONnpegHkLv2YZovWbAYWgL4XC4b4H8RMsrmELpnRQPKi2pjRoa3aruTsTR0Su6DO3+jd6o65UhBrccWS/rfc4ErQs+mHu5N9PWDRcCDw8P7LWSS+Hx8ZFxJ4ogagNf1/U+GPc66K8K1RExv1i/W+t6bcz5F3ltLjvCoPXGbXuj9o0pTZxvF+Yg1NzYgJZvNL/QayWXG0ij50gehuPz8x1HDNN0VAqFqFvACIze8PeIgnMeGwPBT8QpYj3IMExRrd/0Qbm8kpYTBKe1p9qxk8O4hEjFyBVxgcPpPc/Pj/TfW76+fWJvO5ctU7sKMPKufcK/Pv0Nk9uxIbOTGcYpnMAmrG93s7bKhgdC70IuO+ueqb1hyqCEivcWFxyHJWKiV1R2rWxbATMAT5gTdjqBm+nD3Q3bDTMyw1Zah8vtjYfDRV2oZEQKU7KcHhOl3g1RDrz1yEjEAK01rtcXnJ8UR323THmjVidr9ed7dEtuhbxVRrH4YHGuEUPCOIuYjnMwJc2XuR5pw5DboHRVwP3ZD7CcV5JbmIPHyICuWOTmm4YBbUNKvVtyFL9c7303GTMxOeY003uhtp1cdu1RoXd3I/o2XWVHDis8dFLSk5wPjtzONGOxXGnDwphwRrX11nicNXgbVaNuBtfbxv/0//1f+d0f/8ThQUOeznrSPIEZnK8/cFs/abbpBi+vN/a+saSZJc1MYcKMAr3dDTu6kbPWUNumCemysyyROXhmYwgYdueotfP27Sd6q5jwPQ/hmWVe+PD4HZfzK2/7C9UVDHIXFdxPHG0oxnokeu2UvN8zU1ZRLnfirAlebUpd2PpglEEURdsZJzAJLkbM0Dd6bhvWZIb1eHGs/YYTR0JNNWZoQZreqb3SXSWlA/u2ar7KOLZ1Y3l4otZKbZ28bVy3jY8fP6rx+6qnoxjjnXi6sizLXdHm6V2wVkmutVbGGL/YwPd9U69jbfTef5GK6Ikss643eldaxbpuHNKEtYaUtP/Ym7714+SQAtHpvzemoVvUmKh1J19uGCNI06qMtQbpjmEd4lV2AY3JQ7ABMWClIbpapY+Or5lxq9ijvhQMdyns7DWh3wu0nZcff+DL1xdaE9Zr47I1tn3QuZ9ShuGvfvt3PJjvCObMsD9S3Tesb4pWbvp5dqERhtbsWm303qg9k9uuDxOjnVNrHd55xUsFHeqXfafkTDOVWvXBHHJGRKtEbWRENrpcGGMnb2+Udqb3Vx4Pv2KI4ba94DwECSr8CAbnjRqajNaeRQx71v82a0CsYDA4PMEZvPHamtkHfRT2rHkuR8V7pwHXIMi9eWGdQUqjiEY1rnsjuESQvyBGcXAn7UAFbawXsZSa2fMbznjSNDHqUI4VlVwyuVSGVcjfgaCbNLHYYZBqKLXrg7AZjFhadpTeqG1n9q/M4VEFp2zg1vs2aANxdCkM0SGyN5HBvZ4SE95ZZAy+XS/88w9/oPWK9ZbvPn7gt7/9K06HAwNPH5Ey6l2nNZisI/iBper8I3hMdPRecRhSURnoMODEK5DPNSafmMKkZFTxalUpV245cNwe2OuJZTnydHjH0+E95/UrraycLz/grGJw6JaRJywCo+NxRL/gw0xIM6VXWr4ye4szARMdZUApm1qo2cA5Bvctk3eIWPpd39boDOnYZnDFMU0Lwag52wr6Nh71joxpROfZ2/gl/jAtJ3qrbLeV5CPrvgHCvu20pvWckCIpRvKeKaUS48BaHciLQEq65Wqt03snhMDttlLuPLDe+y8WonovXm+b/nvg/ve1xvL8jBl6UpO6E6L6IlNQKkXyE0O0WtV7wYiSenvb2bYr1sK8LDiXaMbQWyVNicM8U0Yme4jRESaL+M6WNwRtWXQZ2D4wmwE3YZYEMdCHfrZNyZw/feF3P/6Rb/uZUdVXsN8Go+vYxAPvP37gw8MTS/DImFgz9wdLwdgbw0REZsJ9EVarXq0Vu910FtY67Z7bGq0RQ8fUQZs6DkOtmbJnuu8YP8j5hg8dcXr6LOOqpx7TEWn0UjmvF/re6I+eZZ6JaWI5PLC9vuB9V+9EUP5bD5bWHdu2/aLXc9bq1wH1U3hjMc4hxtz1gmpCStFymD3LwWB91zmdVZZ/bSu9GUrxlNIBvbpqqOPPfIAdQ8C5hguC9wmDkO8+veP8kXcP39PmD/zw5Z/48vV395I25H3jS93ZN888J1orbPugNkMTFVb22LHDMaolZyW0JvvGHF9UCtQhGIfzldY3ShNKy9gxY8dJwYlUzXLZWYeGTjieIiEkXl9fOJ8vfBqf+dWHB2wMd7idZ6+ZKUSODxEpBjsg4ejG0IboW9Ym4mTxUXBOh759gMSGtSiaxYE3nSiVLujvpnIrZ972A6dlZnGRp8MTn79F3vYzYRPmqLC66Ax+PuI40WrBTZ4wnUjziW4s1/WF6/lHXCs4HNZ69rrSS+FyVpaUtZYwTTw+HEgiVCO0Aa0UnB8EfyC6gAxhyxnnLL5rgHe0htQCxuqpdNtoZiP0hTDPMBq3txd8mNils+43ktd51/l8JkY9Db++vNC6Ckr3nNmL6tHkfuLa872Qey90r7ebnobu0Zl1Xeldrwrrqu5Ka5XOKqMTgydFT9tXGDqLCZMOuK2NiBsMZzDiKLkgvGHShLdKdDV7peYVI/nuMZgQ6Wy3Hak7x+MDo+s1Pk4BGxakD/Z1w9XBYdZAq4K4KiKaw3PpgMmFdr7x9nrm8+uFH//0I6+vb3g346nEpNSN4+nE6eGR4KHKxreXz2zjjD8IS+yIvWF9ZHSjCHSUYVfywPSBNMGLwwj0YaEIVDAdmmkMq2FQuct1Y3BE1+j2RqGo6NY0rKsE6wk2QQ+I6TTR+ZOzltHU4HR8tGz1TKcyLYLzFYaQV7WDGxncLjd6Lfd+8AxEkE4wluYtlaqWJqMvh8PBME/ggqP0ihFhVD3SiQh9REa3YGGaIRgh/Osz/H/9AVZ2hRTGqMNgY61+oKzh6fSIEa92lfiOPf+OGA1zUJ72rWSu10LNAzFQG2AUS2JA07+AGE0dM4Scd86Xb1rM7R5XPHbsRKfQu1GEPgpIxbmAbULZC95k/KxDQ+8c4XhkXmaOpxv79o3z6wsP08TiHziGA60OxnCE0OgUkiR6gfpzihm11Ogg0gINZx21CUUalcG5F8wYWOfpztGTDlFj1Fb+ul9Yt4loAtZblmVhM28Mblhf8P7Aw/EjvRuMjb/0+Kb5wDyfaB1gJV+8nih8VbyL2RGTqWOn1EzwFt8qrlYeDkc1Ho1CWTfFpRw983RgTgfcSEgzdAzWe+J0oBiDHRljNLvjvMeZTi+ZsJzYtytHH/TDWgru4Ynb9cr5euP7X/2K1gbr+ob3jhgjl0vn+d2zPgxSup+m1DUpIuRSKLXqB7Z3Wq3U+x//7H7UP9fupxNldY0xqK2yjY3np2eQoMl7Y/Fx4TirYPd2ORNFUdtdKjFEjodnqtMYi7WCsUMdhQy8i3TjGMbh04xfjrg0M4meSnvLjGGwQY3qJk5IStyLuIjpfP78I//ld7/nuhdKbZzXbwyzEWeLDGE5zLz/8Iw1WpN6PX/j6/kr4hvzSLgDmFAI/gpGtX/GDARFcPfacQK+CU6MmrecIFhiDIToiF4/s947itEyf7eK4m5dzzFOLD5G7R47r5q9ZqhWkVfeV9JkwGSsLzw+R0qv5PblvrFeNEZrhThrnq9mtG4nyvmyRpcjHVXwQcN5ldXa4CFY2v0h2/q4i2m07ORtwDt7D0V3gvU4+xcw8bdboxXBuwlHo3f9h+d9o+SV4B5pMqh5x5nAITmW2dC7xa3C2hpdDNFE5nnCOEs6RL16SqO1TN42yn5FRqeNC1UCViKCx9lA2S3ei65zm6HsYE1WuJ2WIKm10qO+wUWEmBJzTExxodUDZuzkLEzBcUzvce6Bb9cv1OLYV4NbEj4e8Rasy6SUmSaHGw4ZhkJlqy+6hvfQTEHpowHjIiE6+i60NqhDbStfz0I0hmU+UMxgen7ku8d2Z5E1xL1h3YSzHmz9RW7RyextY8udry+fuF0vhN6wWEIUoq3sdWc+elxMeDuYg2OaDCZ0bC14m3EMyh4ovuJOltkdiSkyqqE7i69NU51GcMPhjA7WBcGaTkgTteW7rDRT71cqg3odpzQpWcANXl5feHp8JISADI23uHue6ufTVM5ZN2pN51w/h02NMXrdbTq8XtcVkUHeNoxoCBaEfV81AIqGO0fXpL0NSv4QY/Ex8fj0rA8c71Tt5+EYH+nTcqfUNqxRqa0YbQ48PL7j+PhEnE/4u3mr7VmtSf7nHyCnaXR//10MpnekK//93//93/P3f/zPXG5/wvpBa5297njrOT3M903t4PX8xuuXV5w4pB2oIqy9Eg8dUiGGyhgbQxRfPYb6JsMQjPf0rjO14YXoLTE65mkiRUPtjToUXSNYgnPY4am7pVthCV57q0HT93h9sSyLQcSS2wvJdEI84IZhSECqIxc4X270rjhpOzzJDVyEZB29O2rXTejsvVJPMFgL1RvE6AkaYxSXzdD1xF0C7awShr3TPGDOejLzNt5f5H/uA6wUchuECLEHKsK+N3Je+dOPf8/7p8Fedm63CxFhcobJa/K79Yl9FDCaEznFyOk0EZaJMB8RZ9m2ldVf2e3Cni8IhTLesBLpYmn2ChF8VNJB74Nt1/t/DBYZOgOZ7lZv6yzX9UouG4f5QIg6p3P28Eu5t+ZO9JEwFta9MHpA7AE/PUJ/o7YLXi4InuAfsTKDbWzyShsrMXqCVUaSdxHDo1JaTdPrj1R6d+ScGG8bx+2IjR7rLdPyDuMKYQIZg5xfKDnjnK7Ye995vegsLlfYbwbbAk60goPfSL7zFCI2OPZNQ4tiA904uvHEaEhxEGehjMYlX7iWjXkB5y3BWryZsLbQjDDZA2NMtFzANNqoKuU1HaQS/JHbtjP6IM4LeS/MaWYrurgZrZBrZs+ZlNI9pKoPnrzv9weQ/noB9n3TH0pjaHf/4/V2xVnHGF2jAiUjY2BR9+AYg9Erc0oE63R7WXYsQ+UbVtjzpr5Q1DuKQJwnLCp8CfGA8TDuPxHe6TJhShPP777j9P4jcXnA+YVcG+YujQlpJlmnp4wQkWVBonom2/rK7fMr17Xy8PTM+T+98fX1250rN9i3ysfvHrQvKEKphcvrFd896S7x3fbOuVYOw929pW94f0LuWGxjO8HDcKJVKnsvqntLs4PkRUW5nbvMRXBSCXZmEu4IqUCaEnarSK6MBFvtlNyYp0GICRcmrNUXhYi2V1LyODcj/R2un7i8raznN4ILOBEC/v7icND1v8O1QghwioFtDFYZVAmM0RllKNXEF/wkTMkSosUjGMm0Nig75NUi2etM5i/pQh4OJ3JZuZVM7kVPPMOoGn5744f97xm90VE2UKmGqUWtX5iAt1BFw4PBNoJtOFcwJhPsgkkLNMvIllaFUnau55XRtR7STcY5vSYIuukYIvRWKd0SQ6J37UuGaEnTAZzh85efKH0jTRPH6cSUJsoYvG5nrPGY4hhyV8Pbe0/RJ0r33PYbt3IjF4+kwGQMwzQ8MFzHOsFbc2/4V5xr5F5xtjC7HWMzaxdyM7xcX7n4I3N84LBMRBLzskAzpDBxOs2sZuX19Q8MbrRWudx2Su00gWAmFvfAyS6IQO0DuYdyH44LRgb7tsEQBpE8HGYIJkwsJ0tYtO5za9/w+Qj2me4i0Wkq3zmn9Iaa9Xok7hcKhwWit4x8JYCW3ZqeDn2wSpnwjuvljDdyF8Mq5mj0rioGow0M5z19KD20tX85KddaMSHocFoGIrDeVqWUNN1aWqMbv2k+4d3AmqFDeYa+nJyjtca2XTFDEceejjWWOjxTVAu1DVG3XUYfHM5ZUtTvx/TwBH4CG7heb2x1wzvBu6A1tZAw1oELSJwwITI6uDHI+85//If/wpevnzgcHjivB87rC2XbOc4Lzw9HvNeS+eV6RoZjmQ5YqeSh1aC9VFxQL2YLBTet2AA+OUztNN/oHlo3lC60MpBg8MPgrMc3aLUwpOPGYAkG5zuTiwwbcWnGicO3Qqkb+62BG4Rk4DjYXaH2DMYyh8h8WDmeJuZpIrjA02FBguHkJvIU2bdKcJFpOeHmk9qoRFV6fW+M1rjuO/2q/oA+DLUXrG94pxtN6wTpQtuGzpb7oO+OfIOxRoJMII7R/4Ih/vund7zuga28MZp+sTUpq1cepUho5qMj3HJGjMc5y5YrbejMqmFo1vK2NZxxxJEJIWPRvzZEi1kDUi29wjYaxquKXlylS6c2wxCLT+CjhvHGyESfiN5wXCLYyPPjEZHGl9dPvJxfuVzOPJ8emabEy+WrXkGM493zR+3ItcKH579mnh/56XPj82Z4eb0S3MTb9MK7uRFiYCRD90dqsUQakwlgIPvGNhrSGslZEonhd3qvVDq9Q10HOS/MS0T6Aw/HR5blex6nX/FvPz5xub3wX37//+LHz/8HpThuV7Bx4Dw4O5gCDLTovXdBRIvej4eZFBLbvqmhqYAMrVj5aTA5MN3SqiGXC3uYsUAwyrR3IdFKZoyhyOfeyduKd4qVqesZaz1GLMkH2uj4KeJs43A4Eiz0thG9YY4ztXW6NGXaV0OckmapUBLuGMpfaONf5l65lF8iFq3rD+HoRSMedIwIKQXMfV4laHwgWv9Ljenn0x2jaeH9PmfpPRKD1oOs93eLu5a2y2j3apsnTNMdG5RpvbKtZ6wZPD9/hw9B8ccx/Wy8QPDY3ri9feFtPfN2PfO7z//INZ+xMWJqZDbw/ccPpKiW95e3b5zfbkCkO6vl7FF0O9gdNRvKbvCh4oM+QGOcaMlQNkHcPVslDSmCGY4UHUGM9gVFqSDe3v/HW4wJmtC3Bl9QnnSBXDImgbGR7WqpverNqqis592HB2qt5OONx4dHjilinfLz3PE7YjgxLc9MpyMuWKCQy87lcuVyvrLdNmzSr/mX80reVtzsmGdH8CqdLmslN6B5GI5aLPVmCN1zIJLCjMeR5S+wEiV7IE2WW2002dRfaCqNgkQ9gzh7H242fbve8o4RT+13e7Y13HKjloHNjrDCdCikeVe+U/eUDkM0yU03IAHTPN0IxRpcEAyWFOWO6LH0bmi54iNgDXV4DtMJQ+S7939Fq40vbz+w3W6slyuHZWGg5dp3Tx94PAXevfuOd48fOBxOxGlivV0J9kRwB25nYf26cfaFh8PM6Tng5kC2nlsxPMSZlAziPRWhN7UlxTgj5sBWX3Do5sfYQmmCr8J2y0i9YWXn+7954lcf/i1/+5v/J3/7b/4f/H/+t/+e//l/+++5mS/kzSK+cfWFg9VhtBmRunuKFObYSc7iDw9YHyj7Rt4KLQvROY7zA3MwYKFYR62B1jNteIgBn4Kig4YwTxN536mt8vB4IljLvt10i6gDJuY03ekWcFoSh2Ui18wcAQzONG77leV04vz2ig9R56P/pweURioMJRfG6MQYOZ/POui/BxaFrol9tHg+hYg3BhmdKWgif0me2+WN6C17bTw+PqiHczSsDEQU9aJZNDW4O4PmlIzFWENyVmkAgJFBbztN7L3Q7dm2K7VUloNDgsMuEyNETBOM64gZ/OnzT/yP//v/wj/86T/z5fwTe9bQ6mk+8HhaOMwW7y1v5zN/+vGFvXhSuOP9rachNNEHM3dGm3UDYwV3KFgG3gZidBTTCQxwBhsctlkOYggyNLMmIHbgk2V4B8bTCbjh8CKaZWMwTQE7TRB1abXnQtkGtwvk6rC+0uSVZi3Kb7lRYuTgT5ym9xzjM9Yk/GHGzTAnR4gLcOD9+/esa+brtzd+/OEzl3XncLBMDzMSGt5HpBlu68r1Orie1Z85amc0g22D5xjxKd4lMOjS7s99gKk2KWE5EeKMtQ2hYdx6D9hZXAoaTOuJWhqja+JYDLjhqG1QZHDdG/ZS8cEwrZ3jw+BwiBgjbFmoo4Oz92HiwDSLNzAIlDxwXh2IIYIxd855ELwvxKMlj89M4wlnAqMPpjirLJNKbYWXl1fGgN6Fka/8+r3B9MDj8Vd8eP+Rve6ksGDM4Hg80Evh07fM217ZHwqze8di9W18roO3kTmEBxb3jmQNW/l2P3UF/PTEEhJbeyOEoPMeUzQbFQw1w5/++AdiOPDw+IF3794T5vf8N3/337Hnlf/w9/8D314ulFy5tcweMjFFkjsQauW6V2rJHFJiniac0XyaiIXhYNzJFcbRR6He6zPBBbyPtGC5tYyrulHe1hsiwtPTk258EcZod/KB4TgftIw9LMY6DjFCz9hemY1u6fr+gpOuyKXNUnaVApdSVFlW6y/XvdbqL7gb/eNC61mXAPeVuqJw/P1qa0je0+qKGZm31xdaLcxTQO4ZOBccUg3RKXhgDIjWIvdTmblb4QG8C8QQdXnmnIZVY0TEU0vHmsAy/4tQ9Wc+nPHqLR37imLNJq6XjW8vb/S9MMrG8ZB4fpqIaeC947ze+Px65tvbRtsdy0GJGT4aNSXZgHcFYw2jB/p+pPpKCV2XV6bciaYKDQzWk44Bsl7xnREYDYYimO41eQTVy1lAegEZxMligyO4ib1V1r2x74P6Koxs6abSfcMmfUhiJrY8kNYRL/S9soWVEISx7XRXidayHCaW5ZF5OpLSie8+HDkuH3h8/szn15/Y8868nLBY1usO+04eRxAhThNhCVh0kXQInuM0E5JnyGAu25//ANu3N3YTKKXhnbt/kxu9WczY8UZx0N4mfJrwVsmO0OhVqxxpVKQn1hEotXC77ey7/nW9CSkZ6JpuxjpwetJVAkVnDpaG1W2EaKgPi86ukiFEoZsV4zzn7Y8sztEyGLE8Hz+AGGp/Id8atSgW5C3f+P0f/4nj/Mi+7RiUO75ur/igm5nkJvLauX3JGCyj7SQmPJGXFrmthSGZaZ6Y0wOmCrfrH+nWEXPHETmm7zgtCy5Ytv2N7d5nayOQS+Y//pd/z14K6+3K89MzORcejh/4zfd/izH/xNcvN7pA7kVlvBKZ0zsupVLbTveNdb9Rh9C7zq6MU9JmN3DNBf3CGUwo+OFxY6fmQhhCHINRdNYzTRPLlHB0jWfEQC1CmhLzdERk4ERnUt7ceVC2gK30YVS5R2K/nDXTZSNb6fRekaEVtOleMcKYO6K6UUrWMUNrOuccQu39LuQYdAaTMxgytXV92LVMqzt5T5xOD+TtxvH0oGHpvmGsoZeqElagZMXHiAHnj9rLFGGZj+DcHTVk2fesG1Jj8MERvQU0SqNmoI7xESuWrz/+yNdvrzjjWdyByUKcDQ9HR7IVQ2DdB7dbZiuNYTxiBGMEUJihBloiYocCFZ2+hGozsG6kaO4jG42Y4P5/7f1Hr2RZuqaJPUtuZeoIFyFS161bRDfJKjY5IEgOOCAa4IBTDvhrOe1Jg8WubtZVmRmREeHyHDtmtvUSHHzLLCKbrLzovJMuIAxIAUeE+3Ez22t94n2fFySMV2GLxMBkhbVCjVVGRMoxawhG3BxZka2hbhzWCQkmAXkamZaJNQaSBrfJ7LuM2xq6vbR7Ggcxk1JDSgei3RNNzXgOXIYL0zLgrGJ3yNw/dBzuFF1XlUVXS+03PNx9KXGLuZIM1nvN9BBZvgavDLWXtDNrDc56nFbCECwHdkj/ghbyPBxZdcO69ixzIkaJ4pqnuQRyBhH5ZVOCCGSYSBQER1Kg0GiV6XyN0wIrHJeVeYoMbiEFhVUarVwxAVtSmlgQJLH2gU1l6UfkYFphJKNtQrsAzkjSdEqocGFY/oBTe1L2aGuETaUdWiXCEmWb1Vk+ff7EH9s/0G33GGtZ0sTx8kRkpTGGTW15PGxwi8XkjKYihkDtRw4kvpsjc2VpqwO77SM+a8bzJ8K0MCsRipqqgspTVxU4yPTEpJmGhaQzyzTxP/z9v+cP3/4drx4f2W86tA00jef141um5YPw15HZo1aGxra83jech4/UXqit3mgGq3FGoWxmWSNzzkVvJzYSXMCwognY7PEI1qZuKmxEoHfKME8DqvjajHW07Ya6qrmcj+K9UxKw67QW3Is2MoyPgdMwMo4ryorHLbByOb+gjKbtNry8aC7DQFNXhTxh6PsL8QoxXNfbzAwteiLvK5yNVE6Ro2aOI9M4kdaJxY1MVkvl1mxo2g0hKHIMuEqxrAu1ceQUGPozKYFVFqMN1sh8zluxmkXlcalgzmOiLUBHpcRyJBhrYJ4YXi5888P3/Lf/w/+Lb9//R4b4ke3Bos1WNrlrZOlnzmugD4sQXxtLMoFuW7NpawljCaKt9O0OVxnQUrWtMeJcYNeWXIiuRj1aSEI6yWlhHi9M/YkUc4EvumJXCmTjUNpifU3Tbmiaisoqal/hfUvKmnkNjMPIMi2C4jaSimRqg69VIa0aSAaTHV431L7FOdFtkTMpinQInWQpkA0bu2HXbWicZCfM01yYcVlAAlFDFCeN9RpfOYyR2aRVqtBCICfBzv+LzNxzWhHX/sK0zkzzxDon4rKwzBJykcZV2OpOrBvSpkVAk7Unh4xKGZc1TmUqa0hG45CNxRoDxq8Yb4AoiuDGYc2M95m6zdTOURsY80haZ9ISmJbIqCEsQh8QUWIgrS/MRJTZEpEcQ2sNddVIyskSMBqc13x8/p6/+6Nm6I9EFTmev2MOFywWmzpqY1BdQ1wiS7BchkXgfkpR60zlNjwefsGbt1/zSdf0T3/iPHxgXSbmNdFuHanSzAZctaFRiZhWsVTNsjlFZY7nd7x7/3d0rePNmzsO+weMabi7u2d8+YiKMzGNeNuRdMDqDZudReVntJJdeeW1JFgvEYyAIVNOqJzQwFKyOZe0UJkWYzsMFQkh2krSsth/wroSwsrDwyNaay79mb4/8fbxkWWdsbqh8h2uEqpDDJlxnsj5zLK8sEwXlmRYYmIdF5LWKAND35NRxHUhpFgIuyshypxDay1VTqkytl3Lq8OOigh55vj8hLWeu7tXxGUgpwXiCloRlglrHbZqICVUDGQUIUoFSg5M8QRpLbIEhW9bmqYWaxSRNUTJjdRKbDFGUqSt2ZB8C8vK/HTk/bv3/On9Rz58+sBlOGMq+dlzSqwR5h7mIeGqmr3raFRib2uc8dzvD+y6beHnWbyv2e43uKq6iXtTjtSuYrfZSkq5a6isUCVUqRRPlyOXy5kQV2HeVTVhTfTD5ZaU3tQNvq5p6gajZLmjnWj5SBDmtQiLpep0tqPplFBbjZF4tZDLZyOBIN5WOGdF4mK0YLFIBbclglkrjki0VXQdxLBIApVxVL6mshXOiAsgKSEEx5gw2hb9oAKEIKLtv0BGEWOkrSA7C0SmJTKMgh9e5ijAwxAEVqijRL+RMU6jtBMIXrTEaRXCeuXxlZAYlQFdEmEME6qw6UPMWFuz3Rzw3tHWDqszWS20SWNXR6UX3veBcQms64JSK1kljFnQyhDDicRISEa2bsbS7T2mmgjTjDbQtJ5usyXlxDfv/4lpvUC6gJ6Yxszeyo1vlMF5R44TMXkuIeNcZLtx3L965PXDWx63r1hPJ37IBhul8qnrGo3HqgaVLTkHfNWyzgOtSyyzIgLoyKZ2sBhenp6I8USMM129p3MZKgOTgA+X+RnjIcYGV23IBOCEVhoTLEM4YZ3mbuNJOdNfJi7rIpKMbAkxsswJnEMZy6bd4ULEG4czhqHvxZeZI3eHB5yp+PDxe4zR1N5hjXCzvJOgWu8qrLOkmKiaGudqlHI8ny4sy8QwLUzDgrYW7TTLtOK9Y4oRZQ0pVYV6W6xNKZKSGLubquawv6OtHDqvjP0EaIxVdHXLMmnW+YIp9BCdIzpHrBJyR0xZhvjrRIgrbd0SwsI6zpyfA1Z/TUwbjJOtZE6gjKGqqoL8sXIYegveo6oDOb/Q92fefX4mZ80v3v6Sw7ZjTWLp8rZmjZlpN5GzbLtjnFljQuHoqo6u63BW0N5t29F1G7Ztiy+IIcnL1PgiBs5KciqbuhGdndas68Kl39NfekKUjbD83Eq4aKt4hOu6QmlNXOMNp2SscPmsMqQ1Ms4T8zyhUFR1Q9O0+LoSz2mUwycE2RiLeFvaUOcc2lihkETR7+WUhJQcY3kPK4mG0x1kGftYZyWHIIueMy4r6zqxLEEOSO9xTrJEk8qy8firD7BVtlR1U5MJaCeLG4OhsgvrVMSMi6T6ylIn4VzGVhllC77Witq2biTj0FXF0b5CCAadE+s8EvJMTJZsM95sUKmCZEgZpiXS5oTXE/ebxHlN9LNCZSu9v0qEPAvmOGhSWojZiCTAi8Bzs6mgVigLTdfxt7/7L3i4+5Ljy2f+P7//DxyfTljnqZ1mnoXe2W02uGwIE4zDTAqJepOxXS22CyJDP/NyHHh5uqBjJmmJ+mq2Lcokau+JKpGTxuCoW0/XOpacucwvnNeVyjlU2xDmiZfnJ8xBvriVM6xrQ06KrBem8ZNgSNZXNM1bQjKEfMQ6i3MNKq00tcdVDY1ZWMORdVpxqqb2rgD7arSW7D6vNAnFZZyofIVRueCVLR8/vielyGEnLoWYhVFfVQ3WOeHmW0sMEatEKHwfswhx04lhXJiXBbWGYkMTKGYMEd/UhCDMemMK3z1HkUh4y7brUDnz9PwRayiXQsc89SwhoF2FKVIItASy5BQwOJRx+MYSFsdSdGRKazbbPcssaUKkVWQg84Q1FTFnQlYCfcyId9A5VN0KnWOZyUsgm4q23bFZE8Z8QVd3aJPZ7rYcDvcY5emPJ5ZlBQ3DPBXHAfjCyIqr6OW0UqSwMpxOLM7RtS3WerxxGCt0CY3kLHZth1KaaRZRcVvXNF6CVuTBFxeENRqaRhZHMRKWUIzzWg4H6yR5SZgatLalrmtQ4J2XDAwnPtNsBUEEcugZbSRXMiZSRryqKhdJTCyUZFHkZwoSW8sYxxhTQJKCfUflgjiSpHBrM2hT4A2y8NEY9F+Wgf0zB9gi/HHtwRiFzorKO/SayFoxa8OwhjJoRA45Y9BWbmvjZJVdbYUW4bXgf5c1yq2wJOIEKUpAZ8iQomGyiSVkmmbDurhCal0YwoLymUSP8YZ1VaSc8U5ubGsrprAyzlkOVhLRaJZc1vitpnIJV2ceHu/417/7t7y+e+Dp6TNPx2eePj2JT9JoIpa2bqmrBrMm0uyYh4F5TiQirTesYeDp+Qc+qyN/+Pgf+fDyno3W2MoxTxOmGsjGoCYRw5JmtIm0+47d5oE5Rn74FEh1T2UsofZixI4Xphm02VKZLa5rISCpNVNP4oWmvcO2DZvt15wHxToPaPYY3VNVDbXfMKkThIRVUoUe2pbsHQQRASu0tG3AZrtH54yKgbBOXC5PxDTTdRv2+wPzdBFGuTNo7al8DQXtrZSIpp1XNG1mE8KNZd9byziN5EHIFOuaSgSXxbkKX9cSwScnGBoZ+Fe+Iq0DQ3/GWamCdc5gHMoWvJPSJKUIqDJHWxmHRLMDrSus81hvSSHhjMZXnt3+HqVhd/eKZv8AumZcEmMUL2pVIIBOgHXgGiDD8gIho01LUzW0fiAsM9u2o+ta9rs9u/0BhabWVjy53rOUNPBMEi1wCPRDzzgORdgbGMaBcI6M48Rm09G2KzbJ9rptWrabjeQepMSyzChjSupSLD5HsWhprYslLRFjFAzSKpIVpSw5u4IyWsXUby21cxht5d8FGfRp+T1SWOU7YsBaOYRCTsSYCWFFG4VSMoN0SvIERLQcSTEVbV8gpli+J9JloeT3o6RYaa1JTgSlV26btSKyLj/VX3mAjTOxaUhjIBhDyonayC2SlUdVkQXxbSmt8boWU6+TdB90wd6K0o4pZcI5MAwLl3FCBzBRYVFkl8AWRffYc770WO1p6pq68jgE/zJvpOJKZsE1mfFUETVYJSJS7zNpuZCDFnhdTkxTxKyQWIj1ShNr7u4e+OLNW/btHf1loKs31N4zLxdS8kQryn/tNbMKnPPIcR4Aj1aiBeo3Z977b5lC5NPL76EJTBE8Bm8alpDI88g4XdDWsa49Rit27gFVJ7ymCAFX6tahdE2KunwpVyIrUWexLCmYp8g4Wk79C4fdC137SKoN7eYNA08YVuLkSLNiXDMpaIgWmypq11LbFoxH55rabOm0x+GovWKZzoDkEAxDT9N2VPWGti23cQZnK3zV3G5awRqXPMCccCazlrCQqq6pm4W6GlmDIsaVGCWaSylxAWw2W5pugzVWtn/IBkoRUQVXrTKQ5PbXJCpfY7RG5bXkYoJNGZIS7tyyoG3E1HuUluEwVg5Pbxztbsf+cE+z32N8RY4alzNTnAnzTHCGatuhXS0AxrQIyHJJzFOQBzILy81ZS9d2WGeZp4UP03tKp8Z2u0EbzaYSF4VzjrZtSSkxDAPjOLAsMmscx5G+7wFwzuK8wB2V0gU3tOKcaONAZk0pRaZpIoRV0oDKM3s1xaci7vW+AlIRnkfSPEnwidJYY6l8VdwJgrwxxalAlIDbnAS7nVL5PXIgJGGxSdhH/nFumcWhonJCpUyOkZBWqchyxChzM2fHZUHdskGVkGWNFjO4FoS5/H3+4vn1zxBZF6hDxBqJB8phLd4yzZwjiYC3Wk5lQmHeq3IyS18sHjdLQrPOifF54fgycR4XnDFsmgpnMwYJFA05Ms2RCMxqYp4GKldTuQpvDdMyUu8sWWc2m4iKkTBLTJV2laTirBnFgI5BSJdACpmxF/d7XRleP/6SfXePNobPl48s4YX9rubdxxfCmohaoZSk61ArdDQ0pkGtHp0TUTv6UWMuF+b1zDo/gRWB75oXatuKoVivhHAhTIopRNY5YsL39P0L1itWVpqmwRojuN+sqHVFygFyhcoN5AZQKK1p/QEdJpg8eta42ZGTpctaQlDyQpMdOYIxC+b+a4xuqLymNl7sZSHjc0YbXzaBL+S4oHNkHAe6XYd3rSBcrCmBDBO7rqZppCIJIRDTTOU9CkMMYtbOaaV2hsYLa+vFniRhCl0kELEEmEobY42R+DgrKnetdRFmZuZpZLfdyRffadZ5JC4zaNhvO1Yd8d6gY8IUqUEyimVacWbFeS2hykgmg6sl4UlZqeIwhhyEc6VSEBLvMhGDo97tSCaj8kpaIipopnHgfD6zrrLcqqqatu3IOXE8vjBNI9Za2dYaxbKuAiDUms1mg/eeupYNpNbQbcBZkSn1/YVxnITAUTjgIUSxNo0TVVVRVTKb+rMH2Io39IrMlipMy0HoHMboW2UWYyrJUXKBXDWK67pKqC6qhBnr4kVV5GK2X1ICJeMRox3WOmF+JTnMZQETmAonDso8z8kBBRDDSkiJWNJw1XV8UETG1lmskYDhVP7+awzcvf0rDzCzCSQV0LmTvYCzqBjAavpTL0hiLRsjYmReAmFNBOsgS3lsbULpVVqDyWKCxiWPDblkyUFWXjAdEcI1aXtNJCwpKNZpYdQBrQzeaDarZbdzbIzCtJlTWhimga7b4KuOprH0w0oKYkcxSqGMhGUu84jRNffdHSomns8vfP/xG87TDxiXyCkwTUqIGJ2kX/umQbuEq19kzrcsJOVJ1jAvEynMGO/QTStU0XmmT2dsrtBJePRhXWCpcKnlMoiAdJcb2nov20gdpPK4hpIqC6shpwqyw1c1qnbMdWBtLhDBLBY71fhoMKpjtYmFGYtQUl3rsI8OnQ0xSfUwjAP9OKCNBuWY1pmstOT5LReslYN7DYnKeTCGYewlHsz7G1IpxiC3dlIs68wyjRgrMbdGF1KCVrRNQ9dtOF0ujMNFMgR1Zg0r0zyTz2d0oU5oI21RZUGTStsUhM67LBJAEhYgElZFY2tBwhCxURhr1gZU4R2EuFJpj7VeNFEKqraVllxZsjKscWGYZ8IiftfKSxoOypK3Wxkiv/QML2fOw4VxmgnXTWUhZcSyfABKC5cZx5HU96xBUEzLPDFNI03ToJT8/d1toF3aqCQHllKKqvKlFcxMU880TX92gCmlaJqm4KelNXTO4311M9CDVH7WOrTWhBAKNFL+/WUR98s8z6yLbILtKr9HSgLFjDFK5OG6SpCLUXhXUdcNjRNUlCtC5RgCYVlvF8aN42F08fIK/y1lCT42ZEiJaRpZFrGXaa0ltQxVLsnEr/72rzzAmlcRExc0G1IC17SkNOCJaJd5fpppvKWqHTqmMuQ2WLUI5lhrYiVSiEimdhZTQb0o1mQwXiqcmLM4763GVQq7RIYxC2vfK4LKrDGhc2RdpZzVwdK0mqxnfJWIKmKdZ1PdEZeBd9M75mHGWFVmBhGDJWUx8r4Mz/xwfMf5eOKH7//Ey/kjZNAusUwjU1A8jxW7ZcNj2+FqwxQjx/5Fbg7rxaOnxfrR2AptPUuaUXpGZdG6GDwbt0UbjWkavNvQ1jta12CsLhDIBEnSnzSGHCNOO1GVO1VK/ZpMZBpm5rpBK7lVc0boAEphqohSCZUVXlvRUGkxMMeIhInmckAa+XVlasZ5YU0JRJaMzqVqUTKTOF9OdPXmNqMwRjOOI1Zn4jqQQmRdJRxEKSVfTi1K8hwnNBGjIMRQwkakDboy8mOK5aFL1M6VtqRnWQZ0XmmavSStJwkjTgnGaaLqHCQRQ2KSzOGaCms1MUfImZASjfNUdYOyFcPlIrQKW8ngLomVhSShrVVVo50iM6PcGxQtQUU+H3/gu/ff8dwPEj7sjFRP80TKciAprVhikAOizJRMVdNWNUbB6XzifD6VwFovW0K1EothWWZVuRyGDucs3u+Y50nCXUqFVtc1xthSCSLVkBJCjjYyP7qSbtd1vVVauhA/lkUq4ZwTKSeWdZHEo3VFG/nemOJQSEmQ5nNYOJ3PQKRtNmitaOoaqyzjMnJ8euJ0PjMtM3Vds9ls2G+3qJRJoUgsUpSFnhIjvS1E3ku/8nJ6urk2XNlyK6sF/fMXXn/xAHONhnnBmYT3exYj5V/tEyjLMj4zDRmnhdDQuYopB/kw0agomONay1bFJsGSbI3FdDVLiKQkq1+IVNnSeAdOVMDLIv29woimJwUCMEwZYmAJGr9V+EoOQZVXSXgpSdpRVWglEU1GGZyG7X7DY/dIf3nih/d/x/FpYBxmhovMyOqqIocFRWSeR16OPZVtsZXG0dGqjK8bmmbHrtvgjIIk+GmxykiYKypjdYNOFqsNJsu63rmKtt3gfFUY8LJVqpzHV55hGJjGwBx7vFtx3knblhPOWWhAW31jWonPMEqpH69hwLLcGPpeZkLG4EouZSZLmIU1BemcJJGHHcSK2ipiymzaBmsN7z98z7om3KYiZ0XKEJeVYbyw33Qsy3Ug7EAGAahyKBtrqJua+HySYUaR2Vgt70lV1Rzu7slxIUZpUTSKqT8Tw8DYn9l1G/p+hCgjh5QzlXeFBxdZw4xxGqcNGQmgMNpIqLExKBRrCNRK4wzUdSvoGF8DBqMcGxOJYUbpTFW32KYiNxXgUbpGtweodwzpA6fzQFhXNpstqjaEsJSZX3Nr1Yx1NLbCGEmL95UnJ8ksiCWGrq6us6ckOHDn6LpOAAMxAPL5WmNomkYqnJIdcGWtLctSKpdAeXMxfY+rKhnwh0CKUu20Tc12t2e7uyOsK+M4yYWiRbw7jiPraWUYLszzKJWb87jKE1Nknif64XLVm+DHGqVe0DkzjCOXy4W5gAGukouYAvYWlZZxvlyMWtra68Dfecfhbl9aZ2lFswLrKoz5FyRzj6PGBkVbW2KSfjilTFtlqrsGFeH3f//EcIbdxlK5jPaGZCDOQR5clclLpKlbtFby4CyKeZik7YyZZV6oPdQJ6mzBZdQmYmfNOgXmMZKy+LuSgpwD06oxs8K2yMPgWhlUVx0P25qv9g+ihjcyEwCIKtO2Fbt9y3bjicHgVcfX97+h0S1KBzZNg1Nebnbr8day3W6o60ZEnxi80jRtQ1s3WOsLqzwBWXIlU5QY9SAxYU5ZjBKyagZSWAnIYaLI1I2nqqoS7DqzFFvNPMvhk1ICRZlhaGIITIUfD/y4dQrhpkG7Hm7jMqOtZU0RlWULtob1Vi1llbGuhiSUhXkdqL1mWRfeffhQsDkt/jYbMRxfPhPTxDiqIknIaCMkUaIEFl8rMWtlLX++jCQy3nnqpqXbbmmaWjIsjWZeJpTSpBCY5oF5PGI1+MLb79qaHFa6rsNbeQ98VbMui2yrSsUQY2CdI2HVdF2H9XXBUweMkpZNuYq82YBp0EvCRY+NC8aCcg34ulBRPeQCUXSW/X6HQtTlRkuboxQlYObHmVI2oFzGai0QSGXQtaZxlhQjzkoi0jIvheYiGzfrftTaXRHb8rmuImitPG3T4pxlmuaC4g6lopJlwBKW20zqdlDqDAZ87dhsN4BiW/SbCWnhcsqs9UKKInWRwX8k58iyzIxTzzhcys9rMUozDZei54pI/oHDNzWV94IUJ7EsE6bkcZLTbUNpnbg8tNble2BQKrMsQWCVEZYcsOYvnl//jJXomLBZo5YTXW2JzAz5GW1a9lXFm8cN4xj4/htZ+9eto91BVAnXOWzSOMoMKGRsJShcqZYyaU2s/SIhrBis0liV5QvpzG1GplMkrxEbYV0irjbiuq8TGMehfcPd5i2Phzccto9s6i3eWqyTg8FbJwuCMpR23qCMIsTAm+3CV7s7punXBSPtUEg1UzmPdbqke4tsQCOiyUzCWYcvAsIQg8yCZnEmZJXxa2BWkoWYUsZVTh5ukMorJapKVvYhBIb+wjiJebXynsZXkgNZUMuCghF9T9/3rDHKCj1G1mVFWyPQv+yxSgiYykriuTYiOHXWoQoeWRuD1Ya4rmS9sKSFGCfyongZB4nSi5GcZmS17RnGM+uyUDc11tWsIeO9xjvPGqO0bpQvdZIwCmcNdeVJiLjUNQ1V3ZJTor9cBC1MpvaWeZbNmsKyabakJDOUZVnpGnmI13mm9g1JWZKKhLId19cWF0mqnucZ66RVkbbfYWIEX8zvEYzp0E1D1halSqp80UhBIvQnXj58ZOwHFNBWLUYpSZ9OGXdrBSVzdFlm5nFhcoYcE8NloNtuuL9/oPZegI9ZLqOYRLm+llmPNoZ1naQaKjOpECNoxWazpepaNpsdVkvFW9d1+fwD0zRxuVx4fn5mmiYAclZYX9HuOuqmISvF+XSS90MJ0y3FVbR0Rtrv+66WvMyChl7LheecY7PZsE6S2pXCKmQMrUT0mkW6FOJCpQwxzZC1BBcvYs5fl/WWAVpXFaH8vvM8M08zYZUAX1lkyPczu38BD4y5onI18yXB/AHjM8YbxpOivUs0tee3v3qFN4bPzz3WiS8qZqEz7qylcZYlwek8kUMZ3OmEWWdsWNhaRTYejAD7LuOIWy3OeZld7AyxaxnniWVcCcmQXKTuFO2m4YvDl3y1/zVv779gs9nhfS1Vl5U1cV15uk1XbgCZf6WYC4hR4Zyi2ztCtxDWsh7WYmNw1tE07U3DYo25aW5CiKB02fIYTJLNT+UjoalvM4i6ogxDA2sUiKL8OWVZgSobIBH0ee+lTa/rcsvLK6ZELtjlnDPLuvJ8PLKElaqqaFsRJFpriTmzFLqHNxXO1ThX3eZmcmPmG/VBto2WaA0qOeb5TAyj0BJQ7PYHjLaMY48zmq5rUVouGhTEFKmslgcxBCEhILoqshzWSoH3NU23xVUN536gHxequmMYekJI7LYtVklL40wmhoXKNfTjyKbzGGDsB7ZdBykzDL2w5DSs60JTtzKk1hrn5WJo6pq6adDaUNUb7OEO2h3KbciTku05CqXrIoH4c91RDPD09My7Dx9lhDGNHI+fSTnSdTILWldVBuUK753Mv5SWNCMTIRqI6ubzvCaCX+UJuVSs1xTqeZ6Zl6VUVCvddstjU1PVNcu6MBVxKgrRx2m5ZLXWosiHm0BYleQmFEz9yMt0LEp6mXMpLZ1A5Sy+tKa5fDdFqCopQ433WKXJHbff33tX9FplkzkMJVJuYC6zwcp7mqaRMUYJ6jXGlA5jZllX5mliLgsFgUZYvPPCcfuXMPEb42lcjdaQw0BnHabr0AbWNVB7y7b2/OKXrzHNC8fTi5TAS2CdJtLBUm9rmmRRaKZxxCiwLhI34CtHSkYwJtmA8bS1pakqtk1H23ZUVYsyiiVOTNPKskioQFKSLPTm8Gvu96/ougZXW5IO5KzQUZNSAJVQQ5LBscp432B1Ybd74fPnJKSKcZzIccGgyYUymovaW+mSNlzKfWNSGZzKMDyrJJyp2tPq5saCV/x4M58vPafzib7v8d7SNI1giZJIC+SrRNkQqrLB0lhn/+wLD7Jd2u/3xJzwZU6Ws2IcB/phoG5qNpst23aD9wLsE8O9HF4pC8omXkm7QEqKdYmMQ4/OK5WXLVe32QlOJwTqupI0oyztWtXUKBQhRJRWBRUd0cbIe2bEEB3jgvcddbXB21ogl0GwLE9Pz4zjxDLt2W9bvDMs44wzirkQD7qqgRSZx4ld12GMZhgHwLJaLer7GGUpNM/gxa5z9ehZ7aibBrU7kGyLshVmuyetq5xSeUS5mp+6h5UyuKphd/fAnCLzNPI0TQzTyDj0PB9fIEuQr69kNlW5Csg0VY2rKurKYWzidH4CRI9mneNKmLjOqYCbpssYw8PDQzkYFpQWkkdYV/rzhWmQKr2uaxKp2Itk62+KCNg6K6OFdSWsi8Aj50UumShzKaUkkf36nRL1fmBdrssBK6TclNhuNmWwz00TllJCkSWiUBvxjpbDTGvLtMxkZcBY6qaldg5XZB2goCgYtLX4FK+/JDPFqsJpe9vu/lUHmNNekCIqkJKjqXa02y1WJ/p5ZprAbzRt0/DFo8JluFx6QsgQIvMSmaNiYx2H1rKaGrKEWD7ceZSp0Hi0cVjdYEyDdZbKVdJHl3Wt8x6MzAXiKluTzy8fiHFl3+3wXiwIEseubjecUoqwzizTUA4Gw64DXyvB4LTtrZ1zVmMMLMt1NS4gv3FaUAqapoaidpZWJd9mTwDWebSWt3MNsnoep+m2GnZOHqBu2RBCwjWOdrMlL7KqXmNgmGdiWEvLtBRtTWIdZEvnvei2jHe02w11klI+lUpHAmEHadNMS1PvpSp1jhjTbWMmmGZYpplxnEhxJYWFsK4sMWJsRWVqKt+w7TaFErKSY2aaJrZtxzQN4mctVpelBMc6Y4kkpjVIyENjabuWYYm0TYtxjuN5IMSANb58vhXDOJUMQY3KMyqBzoplnukqCUvVyglbLARwUvGEZUD5ipBgNiPb7Z5FKVQhHIjAMsul4Crs5jWxOoj4VkUhb/QTIZwxm1Y8j0okADkrVNXy+td/w5tf/pbL0wf+Yxy5xBPpY+bTp4+sYWG32zMnwzzPpBBpqobdZksms4YZpTVLudC+ePs1VdsyzoN4A5XBGUflPaeXJ5YlUNc1h8NBZBPlkLu8nMp2UeahSl8zWieuQb/Xi22dJSzFGAtKUO3X76HWuszm5OAc+p6+F3il0GdlC1jXFVUlnURTq7L5vPLchB7y0++/1hqMRRdUtzYCCFD8+O+m4pUMhfhxlXMYY7FGWHuypJbPS5fFzF99gMntFyT70DaoWOOSI+eROEE/zaSYqeuGHA1tvWWZYEovNLah03sq7uiae5pNW0Slhsp7EZ0aX75IYI3HGgmvyDmRYsQYTdO0bDYbrHfyoC+hcNMz4yT5ghKiKoGq6qrkRUrhq0Ulxoh3Xgbw1hJWzTJRIu2hqry0bzH+2Qc0zWJSNUawH1oVB36I0gKWFbpPmcpTNqfxZqqdSybiVVntnKWuPdaUMANniEYRJvGTXT9QmSdEFNyGstcb9jpzWJaZpcw7ktLMk9y01rob871tWgBinP6sXbmu2JWSoI15PKNJGO1wdce29mVWE+XhI2Ft0fOsAbLCmpo1RLSOLMuIr2qZX+WMVRGdA1onrDN0nVTUAc3T82eGYWCZAw+Pr3jz+hFnJZHIMLNtNbtGTNW5bGhjzFR1TdPIIFgFhfMVXntSkHSbdV1Y16lUF4qmazHWYOuKpu3QTmLQyFEG9OvIfOx5+fgZCDSnhvphwW0fyLoqG1WR4WAs3asv+PI3f8vTywuTn3n1+AYNxWQtVfY0jmQyy7IQk4RYaCMVzzhOHI9HDlkxT0PRSQrg0JQ4sXmeMcZzOp1RpRq5ar+8r3DOCGwyRax1LLPE1F1nYiGsYo4nC9VUwTSJH/P63ZHBeSPPoVZsNp1U5zmRlaGu62KqFvfA9ZmyZVb702dsHEdJIdLXOLSSgl68jtZ7nJELXFsjs0dy+R4Ka01rmdHirDTw1/+SdfZff4CFNHMZRpwCV29IwTJdIiELEXQZZ87nE/v9ju32QOVatg2kGXbdnjcPX9NtDrT1hta2GK1QWuYt3jUoIw9lKCk1solIRTgnG1uKIJFSaucsOZL77Zauq0lJyu4YI/M8y6yqfODzPKOuK90YmdVIZQzbusEZQ0BueVv/GAMW1ijDRyMOfhm6ZowyrCUq61od+boiqYzSBmv9zfoRY7gdWm3bFrVx5nLuhfigxGY3j6KhmmMgZinHs5IvUgrS9hprMGWgmoIM61XxqskiQLaLa0y3z1ppcN7TNO3tsLumAF0fmquGCLjddmkNQMJ6xxxkuVIZh1ZGgmXXmZQlv/PqpVsWAVtqMnGRmPi8Sqaft6nYjjLOG6xzhCAH4LwElmXl48cPOGtucENxZ9RoownrKu2z9eSsmJcgX/4YWWPCO42vakzWsqzQkptQls4yxHYVSluq/R08viH7DvqFuFwwtmZdV7779I7aKB7inQz9qy2qrn7yAF3xLpbHt7/k7eN3qFAOqRhxzmC0tE6x2zKvC+M0Ulm59JRSxLRyPr9IZY+kIU15vlUcqliwUsrM848XX9u2dF3L+Xyh73umeWSah9tQ3WhN127Y7fdM08jp/MI4XoosY4NzVhZhSt0qqFiouEuxV3nvMU4UBiHlItnIRQ7Bn83q5CD1fzZzs1bmfkYrmS2HUDqUSJgX+hJ6fO1crjqvpJaSKi/zamZdNvPys14DXezu7q87wBQVMYiTXXpZxTwJwH+cz2JUDYFPc8TScTg8YLYtTf3I3faRV7vXVJVEhimEiim3gJNIQiXBFaQf5w6mbPauD93NoBqzSBVUpm4cm62ENcQkGXz90BdvWLgNFUMIJdlZ3rBpHLn0PYfdnna7kbW1s8VrKH69lCPaIIPoYnkIITDOo8w+xpFxnGjbhjrWLMsiEghjUDjJEsyyWUlpvt1UUmr/9AshMVthDSxhxRoJzkjlg1Y24cryQLxkoq0KMRbOVSAGW2iWlqyENb+uAWMNm62E+zrvBM88m1sldx3wxpjEaJyyeNTK8HlZZlonv/eubWi8pR+PzNPKw+GRGM74qpYMv/HMxt9jnCcEkVA4ayUAxFY411LZgWHspSoNFcY46qrBGkdYV86XC5X3OO9QRJwVEaj1Uqn6umFdJlKccTqS0oKwMS0UcGVIEWP8zdcnl5FkQLZ1Tcpg3QalOlTrGIfPrOcj0zQSxpXZZOY2soGbZe56I8hDJf/P1y11t5MLrPKM08AaF0IS7V232dCVrbH8K2XWmCzLKvKYc99jnQUjgceVr4VG6m25mJZba3Ydlntf4b3jdNLiBVWidey6lnbT4pzkbPaXng+fPqKU4ssvv+TVwz37YgZX5sdZWT/0nM9nnPF0mw3b7Za2rn9c7JSuZRxHrujv6/fnGlJsnS28NYNVckEpV3HFgss2Ot26D+D258cYxWqkNSonkskkpZjXstBSSmRYWmN3/+kz6i8eYIfNRrLmFFjtSEHSduMq6UKahF5BJ8faZ9ZKs+kObPZb9pu9qMczrOss1VNSonYvmhDRC0HGlnLZ4X3hoJcbI+csg8QYJV+xiDJ/3FAo1rDcHtRhGEo4asY5SwxS4iulmJeZzy9HYko8n09suk5CQcttch3KX//c6wc3TtNtQ1J5Lyvr85nLRXhUKSXO57NwrJqGEAKn04l1la2h0qXdm9eb0PJaJZhinVAIdshrK+9R8apppUvFJB+60VLiO+dZ41r+Wc0cVvp+YFkCu8Oe7WaD1kh8XLlxr3M0XdbwMofIpV2Wkj6ESSogEl3XyvwsiM5Hfua1XEI1wziyhpUQI64oz4dhpKoE55LLA1w7T2UXhmmS2ZjKGCv4JaVhmSfJx9SZ9r6VEI8YcU0rARvTRFhGIUSklbG/8HB3xzrNrE4i71EQyESr5M+nEFQ0wgNrO7KOxOVCzobq/oCfKtLHj3x+/oRKK5uuk8gvp8tlUzaSIZB1AC1Awfs3b/nu239gHUfGYeB8ecFay+FwT9t11FXN2soh1A+9vM/F4DwOI/Xesdm11HUj86Gf+Bvruub+/v5WMf84Z8p0XcerV6+YC6KnbVseHx5RRjI4jRmJD4qE4nh84dOnZ6Zx5tWrBx7tKypd4apKks6zVKg5IRvA0/mmhL/+Zy2p6UDpkDzeC0YpF02hZFcK/VYhMzgZsch2MaVYvq9VucgDy/Kj7eqqEat9La12ST7LiFFCK0X7F86ov3iAbbeeeZhYxsAyj+QxSmx4CmgvMxafK5TdYKmxOBrXsq03eCcQvzWEwn0STEcmywNhjaxpyaCS+NW06EqkQpB5S0rCqpY5wfXXk2wjs8gLUtniNXVNjollnslOotNzEQ0mJLHbGMs8z7wcj4R1Zarrm+FVQiSknUJxu+mGYSDlhDdWyv1S4cUYqRvZOM7zTIyB08mgtWENcpPkWYB9OUuVY/yPfw+Qm8pm+fOscfiygeTajmb5sK8zOVXkCSlGmrLsWFaZg2hj2O067u9e45wnZ8ktvFwupBBk61pVMrjP4uFTSmOcYxkHrFbU9ZbKw7a2EqQRF4ySC6frOmJMWJNZ5pnL+UJdGdEsaV0OLGkdjJH8BFW2Xd4YzkGit/ppoh9l+5VikrlNTNRe8+rwBVor2q7BaM06D2A1cQ3U3jMN18tQWvdpnrHeoaJkMSyjCGJ3u71sEW1Ft3+N8nsoc8w0Rvrje3Ja8A52245//Lu/4/WrV8zTgF1mbCUHmMxrImnssa0h49g9vuLNl1/x/bff4CtPEzuGYeB4PJJzYtNuqLxnu9thrGEcR4ahRyV5fy7nM9vtRrSK5Zm4HlQxBiG6eNlm6pLTea30vd8wjgOmVG91U+N9TV2M5dvtjt3hjvP5xPPTM8/HzwzfjZyGnseHR/bbPbvNnjevd4QQeHl5IRaL17X6A26WHq2lNXdFj3j9zk7TRCbRdB1N3bAE8QBfq99hGOn7S/n37M36JDM+U/6+K8zSnZy1oa6r8jwnwhpKUeO5/2sPsGCgzxNTHqhzoFor8gTGLlgU1tWoqsX4HW3d0rgKbzSJlXmRNz9nSYT23pYeP5EAtCh6r7qQui7WizKgBEfO8qFe3wAZPq+sqwzIdZHprmvElzDUaZoE3WGuymapZGKUsFOLwiCLg6xkdT1N0rNvNqJSnudZkpbKwHOeJoZhBDK+qkW8WNWEFFjCQsyRuqrJyTGOA5FYNDI/InJzTjL7KxWQIEi4OfXFBpNut1+Mwn5K5fDWOgILxliWdaUfBhEcZpnFKKvIpsg8UMVeIlu8/nKGnGiajmWZ5D+zVJGgqHzN5nWNs5quq9l3jtpapn7EOAjLwj5vSPPAMovqew2TjABMTTZWDhJrRKm+TPjGymzIlJY5wxJW1jixplAWK2tBKGe0TmzairapsJVwpcLU41Rk6cXIrLNsXetKWF/YxFIS2+t6hzWatnYkEiEl7vY7Ui7vye4R1J6sDLYz+HHl+PEJbRRvv/iSb779Ey+XE8PQw+mJbb1FuVbqMGNRUyTygu4e8LZis73jMvz3TPOMNxX+UIkwNAUu4wuX3tBPYwETRpqmwm0t8bOQH4zVpCxZjGsKN+qpta6MQQSts93uxP5UFPpycQa89+Sc+eGHd6SU2G637PZ7mrbl4dWjfEcuF56PT1zOL1hnudvvqVyDUrpIawJVNTPPk2Rmlq5GFlsaaz3GSpj0VfwqyWNRCpTbdzehtaClQ1jKIkwOwr7vCy1Dl7b0OqsVy12ModAwZp6P0i1M00ROYuuq64Zf/rUH2HQeGMeZJQZwK1QyTA7zTBgVbW7QbYU1UiJaJ1FI87LiHILGKC3S9WH+s4Gg0TjjqKvqNhQcx6F4ovJNn3KdI10/WLFeyHxD2tDEusyEZRF/WBn+Vt5LlVh7whrow+W21YurDM5voLUsBAEZJi9/9vNecw2lPJYtmAFZgVfSNtZ1DWTqurotE5qmYbPZsq4r5/P5z8SmQh/9kQGfUir+tPRnK2bBqPhyGAWGcaSqpI289D392BfdTsJUNcau9EOPdY46JcZhYOilzXl8eCsH8jxx6c9M80BT1VTOc7/f0bWNbEkrj/WG11/v2G53xHXl47t/pP/0jjW+0DQbdIqMqsdWLZNQGIkhY7QcviEElLb4tsXMkaxEzCrEiBa1ExO3aJ8CKkUOWwFItr5mmRZqW5OXXj4jozldRA5jtSeZDSpnam+wviJpzZoyU4Btt8V5T1w1+1cHrHFlpipjkLgeIUaUMozzSNe2/OLrXzBNJy79gGk2tOsoiOysS+5oZH33zOYXO/Ceqt3KFi1npmXCVZ79fotzlpeXE8fjZ6Z3E1XVcHe4k4soShV7FZteN97aa6q6IsWItb6If5eyLHGiHbPiUpCZkuRNXk3gMcpFP08Tz0+fy3sqlUxdO3Zv3pKRQyjEhTxH6Y5Ki+ic5WpLuxYKQnB1pLyyzpK2rrVkTJhSbedYtvwhiVSl/CzrGsshKDq2aRo5neQZTinfLndXUDvXS3oeZYbrfUNdCWnEFNT2X3WAredAnmWgrayRFONKM7OwhsSYE1s8zlYyCC83rf6Jovm6uRCjcSobOYVWUDlL23Z4729VR9PUP1Y+848iu+vvIx/mCmUVm7I45mMIRWxqsWVwaKwR8WmSVfI4DhLvZQXfSzE9XytDOcgSVRmmGi3+PqOFCGqt2HNylk2YVurGZFdZVuZN7enaFl8JD/wa3npFp1wRKCDl+nXWZrVm03WcLz3n84mMVIeibE8Ij9UyDgNKybA4ZWEq+bYsHLS+VaFVlgXEWFbol9NZbl+tmKaRcerR1hbcrwIS3snXYQmJANAPeN/QtS1f/erfMD1+xTKPjMOZT9//gDcVL6cnOTxToqkqmdcUQWJclxJZqMjagHZ0bUXMCmvkvRcSwopWif22o+tq4UatCyZ7kddYD8rTtI6YA9Y5dndfoJzDVTWVd7BOxHkSUWXdUlUeU7esOVM1DeqKOS4zRkXi/v6Rp5dPzONE359JceWbP37Db3zF41e/4Lp0ub7COMG6gPdY67g73NFUBoPheH5hKSSGumqYhoFx7mV/mfci9M0yxrh+t6VDycRZY7QThFJWdF3L/f09yyIVat/3t0KgqjqstVwuJ56Pn9HKst1uqSrPOI58/vSJ4/NHtBX93dvqDXd3B+q65un5mZfPnyURSpvbWOb6/KVU5EE5433GOsM4TIQ1UDdNOTDl0B2GlXmZMChh8je1cMPWUIz9ooEzVrRufX+5yYBi9BhjGceBp6cn5nmm7TrRLFZSDEhQiaeu/F9/gLX7HXqomOeVvEZhdWWxF2Sj0b7CVhXWKozOGKVxpsYV0sG1HbqaXIHb4QS5PODupkUSQ6i//fPXyiRnSFGGKd5bUioo5CxDe0DayXIYXbeTqQj85nVkvFxoy02mkfW0dUK+cMYIdC8lwjKTvUSOVU42qL7y4vsy9qYpW9eVFIM8qD+RTmQxb7KuCyEsjKP8HbpOTLgyLxOmlbSqojYeFrk1X04nhl70bcZZ5mXh5Xym7wfZcJbPZloX2ra92V+cFwN3TivTfCbTkbMS21KSCvX4/J4lRDFCO0fKEHJGW3m/zpczTdux3dZsNgfxkGrLx+MTlYU4DZAD292G3eZfczy9sL98yadP70jLhaRVSVsPYspNqixuDDFlctaM04qpNF5XkBLOV1jf4ZSmqeSAGiLkEPGVIWvH/v6e7faett4REPbWl7/+DZvdoWisEms/osm03QajIE09WQcqW6FMXXRJUdhoIRf91IBznmnoWePKNE70l5FfrUvZCxhQieX8wnp5pnu1A5fLwbby5u7Auu0Yp4GkZNDunOH1qwPe/i3ffPMHTv2FYRw57A9Yq8qAWv65dV3wvhzoWi6TdQlFs+WoqoYQFpk1FR+oQnE+X3h+/sz5ItXw/f0rtNYMw4RxEjaTkowW/vinb8la8dvf/pa2bXk5npmGnkDEVxJvl3Ki0Q3WKOZl5HI5sy4z2kjVv9ttqetKXDYkhqnn+emZZV1p2pppnXG9KwswTaXqMgsVx0ZYAzHEIhoX9tjV8tZtthzu7qialqq5jusz2ll8/ZcPr3/2ANs/3rNLkCKczj2n44lzfwE945SjqYrhssyUqrouqSsWY1WBqV1bsULgLKVlCMvtgLpqla7SibWQFmSIKFoxX8mpnVIo7Vl7WwiklEqycbr+/YmL6FGa7YbKGirnsNqURKWJy9hzqPbcbTaogjT+/PkzmsRuv8Nox8PDo5T1ziHZ9Eb0M/MiKJLCOLqKQq8toTYyXI4xstvJDjgEaQ2naRTtUImcX5aFrGRWsC5SqttSpaoy6Fc50lYiVFTGsKzhJkz03t/AdNf3UZKv448SkHHieHwhrDPaWYyzbKoGnRJdlPndpmvZtA3bwz3tZivVYmmZjd3z8vSB4eWZ4fJM4y2vH+558+oBXjfcv/2C09MnwjLe9FBDf2Y5nYk5ETLEdF18QO02UoUbWeQ4Y6iLcd55i7eezf0dVe3Z7TZ8+dWvUUYekFzi2Lb7O9qupa7EHxiqDl9JZVQ1nVwyQ493CoyImZUS9HlYVrzznPqBp48fcJVlnUXT+Pr+FVVVkdYFUwdyzDy9/xPL8TN31Vc4J3TcTdcy77pSyVbY6uGmdA/Lwpu3r6mbmr//x99L/Nk6s9sdUEpRV2IAjzESlpVxXQnrym6/wzkRU5/PL+QbPjqxLB11XTPPE6fTmb7vURiqqr2JYK3VbLoW6zas68rx+EyMke+++47n52ceHx/LxlBGHGL3ysSYufRBNrbrUgzX3FBN2lvCx/c4I3QL5yx396LNkjl1ulVNOcMa1kKHNWUQr2lbL77fdaFtWnmetb4tK5QSpLQq+i8Aby3pmhP61xxgS8zs9zu6ZkM9TKjqB8KnwDDMpUWTH8pYS9dt2Gw3bLpNEaqZ27YhpfgTaYQuoDZ7mxNdP3gZ2JvbXOg6+AshEHNmU3tMdiyLfBGFHiEPe4qptHhikVmmQcrknEVLVAnMsG0bqsoyzgOn05FlGlDWsywzQz/y9s0b9rsDxmi6Tcel7xmniaZrqCuH23ScX070w1nmMd7jvC9ugHxrlZeypZznqyJfbtuqqgpbaRAz67Kw5sQcV5QGbzzLurImad9TTIzDhZQS87TQtIKiuUQRfl7b7M1mc5stzsvMuvZFtJg5vjzzcj5xPp5Fgd3WuAfPvqq4f7hjs9thnWWz39N2HdvtFmcdJIkXG+eJ3d0Br1Z0WojzhcvLD4TljGsegIRxFa7Zo23GrStYz7SuvBwvTMvCNAeWdeXldGFaRh4fHm9oH+s0uqrwVU27qamrjrbZ0m5a7h/u2R7umGcZSsvhvLI/7NFGQibEWiOG6rrbQNthrMcdXhNCj8oTSjcoaukgjCIsIr04vpy5jCeej0cJzL3PrDGwDGcBFk4z4+VMyopxXmiXSUzxUciiT09Hco6lEymPk1bsDjvaTQfa8qc/fcPz82ecG0QwHIUaW5XLZ5omEakuE19++RX7w45huBTel9jDzudnlJJMyPP5QoyZu8M9fX/mcn4BtGi52oo1rHJoFurr589P/PGPf+TNmzdY5xmHkbZpCMsMKorv0FXUTYO1Nda0t1/b7XYYbzm9nJimSQz5dQ0t8muz5FJaZzHWye+T5GBeQkBpIaC0XUdVS/qTOBwksCPlJIJlawu9ZEU7V5LZIf6PzPX/kw6wNWhStISo2TQd6XCPVpmno8EmizMNbdtyf3fP/eGB/WEnrZkRWqVUT+vNQCw9sKh3m6b5yQF3NbNKj+ycu2mwrsr6eV3EPFxVBUGysi4L0zwLQDGD0YpYDKabtiUhhmzv3G0Qv9vtcc6WzcfKGhZOlxNDf8G5mpAzT8cjd/sdWWXevH3DPM+czy+czy+0TQMExuEswsXt/qZ9maaJeZoLlyreGF9XDde6LkUCIirrqhIiQKVgZ7aiZ0qZOa6M00Tt5EYbhobj8fkGCPTOsd/tWFYZ6l8tKNM0CeHy0su2clqYlpmPHz/y6fkzz5+OeOvYbHco43n1+ise7l/RNIJ5WVNmXWbG/oTptnTdRr6ck8PZzMZrvFEMx8zaf+bD5z+yuwuCxzEO7TRRWXTt8DHg2y36PIpbIgbB5/SX4pErn7eI1WSYbRxVs0Oh8HWhcippjaZRwjXq1mG04jKMtJuNoHXqim63x9YNytdlE2vJxmJ1RVYZhZWbXhuq9pHz0+95eveOjObDD5/oTxeU1gzjzKfPT/iqZTgLnubl9IK3hvGHgeeP77nfb5mmC6dzT123KAVLXMoSamQcB0k8t463bx+La8GXbWQjKVrFH2itZ4tgc6Z5Zg4rv3544O5uz6dPH3h5ES1hCJHz+cz5fCEEWQaEuPDtt39gGAbevPmK16/f0nUtnz9/ZF3XMttUbDbC3Xp+fgZlaOqaeZpYodBZIDiFUp794VC0jKv4VJ3Y514/Pt6WDleJTlUFNtsD1hr64cKyRjCC4Nq0O7KCNYmOVJctpjOSTOR9xboukk6GzGqdMTdsdgiFDPOXz69/honvnaRrZ4sns2vvMLkhp5Y0z2ybO96+ecuru9dsNzuaxmOtQRuNc1dI3zXaKf3ZTOwqFr0eWleUyDW886qqv1kPbqiRqSRHy7ZwnWfBJBvRCk3jSFPXtJ0MO5u6vs3TlDH0lwtKKR7vH+naDm0NH58+8+7dD4SYOI8DqzX86te/vM2nfOUYPvY8Pz/jtMZZx1pM5YlngfplxTStXC7CfTfGiLxBqRtj/MdZXwHWFWN1ipGSCoz1Fb6tadsNRhnmeSLGwG63L/PB6jY7c86xLQsImcNJ2b7ZbIFCF0gRbcSq4X3NEpZC0DyTk7TtwyjWFKcVp2XmcjlLW9BU9MPK6eWMt5a62bM9BNJy4dO7P/Dhw2ce3v5rXNWincZUimxaYpZEZ+9mtHouVi+hru52O5x3PD8/MQzj7QJs6g1tu8FqscjUBZst32AZNRh7NQrL1k5psSlZV+HbDuVrsnEoVcvfH8is5RmQ0JBMJivLn96/5z/8h//AH7/7jmEYyWGh61r0Z0WIC0/Pn6ibWgb+SuF2e9EThpEf3j3z/fvvcdrzxRdfkJWw7fb7Ox4eHvjuuz/x/PxcNF2Wtt3yi1/8gqenTwL6u6nTFW3X8fj4CuMs379/x6enJ7777p2kc4UFpSxkGaZfLhMhpJt+S3y3Ae8rqYqQDf12eyDEyPl8ZJ7lO7jZbORCcg2+qtG2eIhJqAybzZ7NdsPd4YD3nr4fCGFlGGRO2HWidfv0+T05pYK/yez2d7x9+wV1u+F8euHl+YhVmtlJO99uusKUEwRVKCMLyuY0FxnQOI7oDLvdjq7rZEPaiyRj/9eGevTjkwgnmx04gzWWbbeByhP6ib0/8OruDff7u8IcSqzjhDZiAQiLECtVhlxaKwnBzLeQAGMMnetu7eR1BmCMxdpc2k8xg64xsEwzoUTTxyjBu1YV8mEZaC/TzLZtOewO9P1FCArbHXVbcT5LfNhmt6VrGkKMfPH2C7abDeM08unzE+s8c7lcqKuKdZnp+wvHpydZEhQihHfQjz3904XdbkddNxijUCpjrBE7TfkCy+xOZoE/VerXdUNKYte43m4hZaosrfl5HOj7npSizBadI6aMqyrImf4ijoOrWn79id0jFgX0/d2B7XbDL778iqEfGaaBVNJpYhpZZmlrrPGEmFEpEuNM006M4yR2miwzRpSh6jZ0hzumxeC6V/jtK6qupW5bUJGIIkQBEOqmIWRpF8QGlW5Av3mZufL1RRws7oCu3XHYH26z0OPTiZKQjDM/OhPiKvTZ3W6LRrH0PSYlVN2JPQ0RgIp94EdVvcJgvKF6eM3LvNDPF9ptC1kuq8t4JJvAy3Dk1eMDX3/5NfvDHV3Xlfdpz/uP7zgNI68fOhJgjOP+YUeIcuAopRgH8cyKIFd0bI+Pj4I7Op9Zpom27fjdb/+GeZ05Hk/st3u8q+jPZ2yxiFkD+33N4fDA4XzHhw/veXr6fNNPGW1o2g3WujJfDTJPwrLd3PH6VVOM5otcUraiaVq6TUci0/cXcox0nVx6OYkINUYxZG93O7zzTOPAMo2QMp8+fcZayxdffsXd4ZHt/h7jK6Z55vTue7quY1oES3XNmFRwm3tVVSHdxkztalRSWGVZlwI2DCs5B6Zh+f/hs/1POsD+4R/+yONhYtM9YStH7Sta32C9WH62bUdTV1TOQCoPUExoI/6mGKS3rbwXY2aJy7qq0nUGleVHvG4nr+JOa8Ugm5EhYQxrObxWUpBI9BgCOUYotqOqqXn75k3JuhM6QAI2uy1aaYa+Z7vdcnd3R9e1N0/Ysiy8vLyQUmK/6chdyzzNnIqyehhHvJE2MSdY4jVxyaBNwzCMTFORfKjMNC1cwllKfyszghClZe26DmMsu90eELuIDQlfiWYmxsw8yTysaWo2m45UUm+uLahAEFfcT1bgMQuZdRgGpkmY5uuyoJUS47rR7A9b7s2hVIbQNg3LvOKczBDXNeKMIuaE844QI1ZJxanLBhjVsYwbvvzV35C0o+p2+LrBODGPz8vMZRgIGRnU24plyfTDQD9cRJENssovyu5hvFA3ov1pmvomt7lud0OUZCLvxDCskBHCvMx46ySxKIkwWntR5GfEnGyUL5TVn1iDgF/99m/5v/zX/zf+2//m/8nx5Ymq9ijybWHUNhtev37D/f091kr71TRt0fZt+Nf/6t+w3WxEs6cV6zrz/PyZaZrQylFVLW3blRQiReUrdtst57OV2aI1HPb3LOvK0/MT4zTS5gaVM20nnLi+728diCzADIfDPdZ6oVxoTVXVfPHFFxwOB67JRNfxzBW/JKk/S+l65Pma54LYUQrjK5qmLl3DyvPzEy8vL4SwcOkvNHWNt47L+cIf/ukPGO/47e++Yrvbo5VmPJ9BX6it4as3X3I8v3A+nQGY5qL/q2vqVUYil8vlJpsK4Uc0+uVy5nj8fJsfq6xpu47f/bUHWH+OxPkTp6Z4BJWm8o673YY3d1+Di6LyVRmrDAlxqnfdhqr2t+H8dbjsKvkyretK5TyuscVqJJs7AENCJVAF47KusvFL6zUSSiw/tXOMMWLrmk3XlT+34/HxkZQSHz99Yo4BW8ucKcfE4+Mr6rq+5eB9+vTptjAwRjA38sEuIr8YB6qmZhrHYnviZiEC5MObhtsD54ryP6eCoSkD2nEcpf+3otUxxqFU5nh8uclIrBXOuTgTtAy3S6s5TePN1dD3g8wxmpqm8vKzZbE0AThvaWbxfl4uYnC/Dpevle91K7wsK+M0EkIJi9CaHBKuMgU/5ASelzN1syXEIEz77p6HLxPH0wXXtmjnyNqwxkxC9F0xrMzrgrYNwxR4fr4wjQsxjTdXhC847Zwj+/3dbXxwleAoBceXFzaHHebqHTWi9Xt4fKDxFes0icF4t4OqRhkLWaNIEGZimlHeorWEeFwPshxl4/2r3/wG/Y26LZr2+8MNJ9M0TcHUCFwghsi6CLOtqxq8MUxrZOwvTKuMPOZ5xjsx618uF+Z5Fja/EUS6yhljHJehJ+YnLuOZqnI4p4lJPpsUZARx9fBedYpaK7puy5s30lPJd769jWTks7W3av0qzp6m6SbWvsqSvJet9nWW/Hw8StdUqvjD4cC7D+/44YfvgCw+0ZLCrY1h6Af2+zumdeDzhx9IMbHZ7HDOUBvHS4jEHHjqTxyPJ/peRjdtKxo9sXoVbn8tgdjihtljbSXLjpzQ9l9QgblkiWtktRGvDPOauTy9sPYjh/qRoR2In2TGddgf8E7mE85anPHFQhII6wo5s2k7UPByOYsHErkT12kilYpmXWZCiDff1FqG91IBiH1iHkcRBCLRTtvtttzY0poNw8A4i8zgsN+zhoUUM03TEGPk22+/5bDfY42Rcj6uHPYH7u52fP4cWFYJ3wg5kqaxoIEFSrfMgvKtylxrXcTWMQ4jujPc3z9SVSKhmGYp6evao4uE4+PHT8QQaRph8768nEqFFhiGgcvlwna7JaXENM1C0ZhGIRnkzOn8IsrpvCOGimmaSCnD5VzaWFNmh9xutnmeb6yz6yG2hpW6kupxs/lRiQ3IBqm4CdYYeP78VAiwGzIZYyvqzYFOWZKCdz/8wJs3r0k53WgD67oyjDP92MsWclnK3EQOr6giMUj48bwM/OY3v8N7L5uqGMlRoJLGWppazNnjvJK1Zu179rs9zeHAh7//R5zObO925MqBaQBLWi/kOGH8n1NWr1XY1J95/+6PvLx8ZNM2t8DVYRjouo7Xr9/cJD99f0FrQ0A+E9HyiU2orhpiyjcdo7T5oXw/6ht6aSnZi13bcbi748tf/IJ377+nLwfMMs+M04Qxlk23paoqNpvtTcgKP2okU5IDbo2RlLnpAetaKBFa61uVbYxmv98zjmMBDIg5+xr80o8DzskGcF3lc1+Wlbu7O149vEY93HM8PvP9998zTzNZiRD4+HJh6CdePd5xPr8Q5sjx5QVbOSFrWINBcTq90PeXm0i9rhv2+91tZis/M/T9yBoi1jia1tNWNbkM///qA+zOKeYUqa0gcIY4oRvPdrOnto6wjoR1EfZ1XQm9oPagERMwPxIcr/oYpWSUGkPkMs23KuU66wph+TPrUEqJthFPlKsqSWA5nbnyxZWSzZmuKiSlWOwRpuAellm8XqB4fn6+6c2enp9lyJsSwywM79P5haGfytA5YJ2+8bNAvpxVU9+qBIXBGF8MqgHnKsIaWVfJ3BuHiXmeqXxNVTdYU4twMCXaRmYOn58+c+l7rgGjP93Myq0faJuWqpLZ1nYjZt5hGLl8erpZk0xRUk9jz5XkcX0ABSc833R2V1uUtdJeGGupZ7FuhGVmDT8eeK9eveL4+YlxGGi7GqdAm8LBr8VEvM4Lx+cj+/1eDsdJpCPjNHM8vTD0PVZRDP72tuTw3pNi4O0Xr9hud6ScmOaBz5+fZOBcVRzuDkJa0Ia7uzs+f/7EOA386te/hgxrDNS+Eq+C1kCUQypHVI7yw/LnD0HOiW//+I/M40XmhatQUJ21fPz06fZ9BW7b5GEYWJaV7XYjoly4ibpRcH//irlsgVPilse43+8LymkmZYWyjlePr0AJjWOaFzZNx6tXD8zzxNPTEynB4XDH3d0dSimOx2PxFIqmTxuL0pqmEzH2GiJhlWwEqdinIlXyN3P2sizF7ibt9FW7mNZAP04YrfBVRb07FBvTwtPTEzkJ1GC/f8A+OoYpst/v2e92rOvM5+cj8zJSVx3OtbTdj9Tgl+Mzd3f3fPHF1wWUoMp7GrjSLa7dzHa7xTpfZl4ZbR3DMrLZ7v8FB9jjhmgV9U6CXOdpZV4CzrcsSbaA85oIITHfiS0mp8RU/Iw/RdPElDidThIJllaImZx+dOHnnIqZVf/E3iDG1qqq8E4wIHVVCb1zHMhJMDOzWbDOMl5kY/fVV78ABJ+zTBPWWD59/sTLi2jHfNlMXi4XrDFsNptSmYQ/m8MpDVWlS/UkMwPvJTH55eUF7yvx7rUth8O+6HbE7HzLbVwCH95/wvmKpm2wumKNa6F1RqyxtG2Lc14YXyXrL+VEU8u8pdt00kYnoVyu6ypc/dOJdQmEGKlKuELTtlSVrN1DCHjlsau0p+ttKyqV7LKurCEzTwtaF0fCNBLiwm67ZRhGnPXs9nuGYaDpRQ3uvccayzz2LMbwxZdf8PHjJ6wbmIaBYehFLX488v7DR06nM1oZnFXoAh6Ulttibc1utyvBqoElzMS0st/vmKaZqsz4KudLS9Rz//AgqeFJwih2+x3atmStUSxc41LWKVA1/Ej4LC+lFA8P9/zdf/ff8HI68urxdcHgVDSl/Zrn+bYhryrPFc8krooO5+8x2jAMPafTkXmKeFdzWaVaSlmcKZcSpHvpe3a7HVVd0/cD3377DaZ2vHr9mta33B0ORag6M41ygVReOoau3WK04+n5GeAmQ3JNhXdygTprb1tvmZf+yPWCa/Re4qeBLusqB+Llcin4IcswSgUqI40J72rJOjCOcVq5f/1WtpTzwt3uQEwZW9V437JtDrx59cC5fyGmmbvDHcuy0PeDxElqmGeZF++2O4wxtG1bED01Vd0SQiw020CdHGEe//oD7PDbR3xdSUUVwCyBZkqYZMmFeKCxeK3JV/Swks3jlT6Rc2QcJ6FMTpMEVThLXdXF4a5uhIZ1nQty+UekLQBZsa6R4+k9VVNj0NSVZMmJyXXm/v4Bax0vL0eenp5YxokQRXnuK4+vLOM446wjhBW0Zg4yo7neXMYk8Iq+76WE3zal1M6cz5fbOvpyOeOcYZlWrNZo4NPHjyzlFgzrSttU1LUnK4VWmmVaOL+8EJXi49NnOi/VxWaz4c12c4vFuiwS1Fo3HUYbNpuOeZaWebPvIAuor7KWpqp5fn6RQavz3D8+siyz5FTGgEqZpq5oq5rKWNhICyjBDlLRJTLrGnn3/geG4UwOEqbb7fe8nF6Y5oEcA5+Pn6maiipGplkOiTVm+qeTVCUq8U//9I/oZPj09Inz5cI//v3f86c//F6gf8uKqxx17SSuLV0lNBXjGLGuIqZAypG7wz3GGOZlxlUWpZFEoLiSAV/JP6usoe46EbO6jAoTSksgi7IK01SwzGSnxAN59ZTGhV1b433Fss5YqxmHnrRKxqTJMNMLjMBolMpAkqG4Ba0Mm9uc6YrpFpJuVaCAOYlfdRxH5nni9WMrs9qq5ptvv+X1mzdY79hut7csgz/84fdst1v2+wMpJdmE13KA1HXNdit6rvv7O5yXeaozEmRzlR1dZ4fXtvNajf+PZUlS8QtW6eHxUQTlSoJkKJV6ziKpqKqKZV2Zxpmmauk2G9Z1ESH2LBCBSOTp/AFtVn744Xu+//5PPDy+KjAAmXNXxU2jlWWaFkJcb8buaZq4DLLR996z2RSabPwXhHp0+x05JuKScChyFrNzZ2o8HlsSs5VSxCBbE6VUGZRH5nllGHqhLBQmWNO2VMXkfH1dTdTeu9ubex04LovwqFKCOaycn3qcsgJyU9IaVYXguq6By0WkBzEFjM54A8djzzROeN/cBINN21K5irDIrO10Oomko+sEqmgUQz/x+fPzzRXw0zCPnEFpwziv9KPc1muMGOclGTlTbqcKlTJ1VeB6OXO32zP0PcfjEeDGgFIKus2mVJzSAoQYaG3LdruRga6xxJgx1rKNmfv7B+Z55vhyZBz6kmEpw3FdiAIhBDYbkaqM41BW6p4UEzEn1hg4nc58+83vGceJ7W5LRtqOrqnZbjq+f/eOcVo4HA6FryvD8HVdOF/OvFxG3r97JswL37//jm+//5bnz59IyILA1xXKKqwuSU0GXEESV03FMPVcLieU1rRNx/lyYVxmwjkRlpXDfi8bycKd6i89rjqxaVvOpyO+Ot/oJ8rMKGPJcySEgWQH/O4tSltC6DF5JMSZ3/zyl/TjpYR/aEKKeG/JWjMuAWMyrbHokNntDqzrwsvpib6f8O6JpmluuqVpHKkb2Wz70jrWdV2ExWJknqeJp+cjbdfRbeTSmsbptlH89a9+fYNnKpWZxgltFOfzmXmZ2W53VN4zjjIrdc7dHvDrd/NyuQhFIqWClvrRnvdT+knOme12d7P63QJjyvb3KiS/VknzLGG+8zTRnyXPtKk9T59HPn5+RhlNf+n5JifaboP3nmmeqNsKRWaZB3Jc6bZ72rbl6emJNcy0rXQwLy9HPn7+xIcPH9jtdjR1w+fPH9kf7vj1f/l/+usOsNPnM6wBkzLGefIC62VhdArfNSjriGGV8Muy2RuGoeg+xKw6TkOBDVZl41BxDcO8RnDlQnKo6+rWt4cQuFzk8Nt0DdvtDjNowhy4u9/f5jjOedq24Xg8yoE59tRNyzAPksE4jOSU2O33pJBZlpXaV1ilMM6hy21+/QJcZwPrcmFdBfhnnRb18rqwLBI7FWNiWhZiEo7Y/Z3cin0/cD6fqCrPq1dvRO0/z0Q1o3Kk9p66bpl329uf5bwl5yt1Q0SOQr+45vxJu7ssC29ev6VtG/GfliwAa6XK/Pz0xPl8lpAGYsH/epzboJSw1qrKMo9FCGtFGrLEmf2m5u6w5Yd37xmGnn/4p7/j0/NH9t2e2tW8vLyw6T6x3x+AInEgEAt7LQbFhx9+YJl7jqcjl/MJ5y2tbyQM2BpsLSRdkljArHViMVWZvj/L56AM87Sy3e8Y54mUAml/oKllYfF8fJbtW9sRwkLbdozTzPL+g1Tt3pUH0XF8euF4PuKc5c3XgcPrX5LDTJjPxGXg7nDHf/Xv/ivWdaVp24KryTjrUcrx+OqRazSZc4aX8wuvHr7gyy8rrPcMlwthXQUmeN/d7DDGGoZxYJ4FifPyciKTqZqKX/zm15Azx+fPMh5oO+qmKwQIz+VyIcYFYy19+cy7rqPbdMQQ2W63gFzWxhgupWjw3t/wODGGm8XsyvdyznPFQR8Ohxs66uo3HsfxVvFd57DXsJx5lqq+qWtSnJimgWkcyDkSw8Rhu6XuOuxbQ+UM2lhxnsRYhLSZuha1vS8RbzlnLmfZ0h4Od3z99S843D0A3A783X5fUFL/6ddfPMDe/9M3+KTJ2tI1HUEr+mnEMgjTXIluJMXEOM00TUeKRUvjBWVT1zVt26K1YRx6ILPdHshJDpPrgFnwOL4AzeZbdHpVVYRY83w8sS4Lrx9fYYCpH1hmSal+elJUlS9YnVUMsGu6UR+WZUZboanOy8xut2Wz6ZimiWEcMVrftjLXljQlWNaFL7/44qaPcc6zLGv5QojxdF4XUZc7xzCOnC8nvPe8evVaNqnLKIe1lqpKYsdgb+9YFhHM9r1EZrVNQ9ceio5HlxTuwDwvaGcJ88S7D+9Zl4DSirawyau6whtXfg7Dy8sLbSt5ntdD0Rj5O7ZNw/f9O9Z14fXre+ZlQqmKunlDiInf/uoXxJi4DD2fn54YhoF3H54YxpmX8wvvP76ncmID8W1FP470ZSs4DuWy2mx47StQEOPC4U4U6jlDWIJIY1IkpkAMiZxgjZEPHz8S5oWn5pk3b17z9CxLimmc+fTpCZUzy5pYgwiKt4O0/0+fnnC1KwSHCoWi7wex9cwryzzydHzif/XvFEN/YRkHYf1vWjqzQaUEyjCOA2uIxCiHyVp8j5JC7uhCIKYFUmIoguiUEv3Qo9TAtt1wOZ3QGV76M9lomqpj024xVmMLqaPyshSwxgqUIOVikYtUlUGpjuPxWBhfM7pw1D5++MA//P3fsT/sqaqapmmL0Fk+334YOL2c6LqWruvY72UALiDNiqenJ56fn3n//j273Y5Xr4Ri8fIiSOzrlj7GTFM3VL7m+PLENIkyoK4bTGVxWhh+x+cjdbshZ+jaDXFdaH3NvAZ0zKzDyNiPfHz/kffvP7DGlVevX/HmzRseHh746qsvePfuA7//p2+w1tBtO6m+mobtZktdu9sS4j/1Utch3/+/17/9X99nlTRBG5wDnKBA9vWWX739NRu3IRrYbXY87O/Y76Rsrupr3LoSYV9pv7z3t5imvu/LilnddFTyo0g1tiziIVRK1sTTNLPZdII1TteUkxL8qX7UOEmLJ2nYqChImrIdqsoWU0BvMmebpukWRJFz5nh8ZlkiD/cPRUxaWGdK37aDV43N8XQiRtHMZCBEgc1dFxh1XdG09U0GoqEMgTccT2eGsb/ZlU6nF1LRu7Vty7ZQLJxzrEGCP9Z15XQ8c3w+0TYNv/jlV7x586rckstN89X3PT/88MNND1TXbZFywNCPfP+dUDx/969+RVXJ3w3g0vfC/a9qQa6UeUhWBus967JwPl0Kn6vwylLi48cnxnHgcHcnW8kcoVAKhsuZ56cn7u7veXx4lAHzsrKShYW/rKxLZl1npllCU9ZZ3q+n41ORBQhW6PX9A/eHhxL04Tgc7vBVLW3xOlHXNZtuw6btbrPHt1+8Zrs9YKzlbr/DV3L45iCtc4wrVick4kvGJLIIqUs2KDdaqvwvt+rEOakOvvvuO2KMvL5/lNBfAKU4jz3eee7v74kp8v79e7TW/PZ3v+VcMhUeHx7JORZr0ELTNNzdP/Dp40f+6Z9+j9aKqm5kKxwCh7t7Ti8n1nW6sfOqumEaR+mElkXSiko72DQNp9MJayXod10Cn58+o5Vmt9thnblp6+q65nIWE7j8eRFbkE5k0V9+/PSON2/e8nI8ih6uaRmnkZxkThlT5PPnJ+ZlLva1imUOWOvZbDc8Ptyz2+2Y55kPHz/w8JPZdeJHVr410q1Ny8L/8b/+v/8nxWB/ORdyCqSsCTbhMFRaDphQTmm/8dw9PkiJWzW0dUPTdPjKkYnFA5VwxbQsHkcRTkIuVdOPnijZwowlSsrcZA/CA5dh4vlyZrPbMU4jXdsxXWbWReZg0yQ99bou9GPP4bDBVzXeeUJYbz+DKO9P3N/f3aqduqplCGo07z98ll+3qthHXEGMhJto8Fq2j+N022j6quZw2GGtmJBTjlirCVHCZ72X1JxlmTEW7u8PEjwSI1XlOB6PLHmi2wjRQ7aeFUpr+nEo27M91uTCzA/F7C1lvdIK6zyHw11pwS80TcO6rjw99TirGIeZZZ5588WbgtDOxeqU8M7x/v077u8OtF1LiDJTqnwlkEGXaeoDVd0UK4t8D7q2Iayyma0rx932INvoacYpRV1au/1ekN3ruoK1xdAdZT5a+G6isVpv5vuuE5DeMEzENZCSDH595WlaaauqrqZWsrHddhu2bcduv2e33QlaJ6/UlUXFTI4rlXesOtG4FqNElT6NkmtpnacuA20bI3MZZ1zR59MkW+YYU2kRZ+7v71lniVKLMaKM4auvvmQfD/zxj3/k9388stvucdbhK8fnT59JpRv44YfvOZ1ONG1LCpmnzy/EBNvtji+//BLnPc/HE3XbociczmesNbx+9SXdZi/Sk2kkRUmPb+qab775hqqqeP36NX3fc7lcWNeVX//6N9RVB5gyH1NMo4Q6H+7uOJ9eJD8hR2Iqo4tBLtnD7o7T6UxcI58/fOQf/+EfWEJguz/Qdh3GFj+z0jy+eUNMsTx3oWxRLX1/KbkKYp/TSqq/h/sHDocDaxC+WIhC6j1feqHN/IXXXzzAYuXIKaBVIAHzWuywPmKdYftw4PXd61u0unMWX5kygBSbx36/L1uvQIwUk7fcXJfLhXVZCyGhY5kXUoJrSKcMIOWBl/Ww+N/WeSatsfgThTwaw4rRmbBMhHXGmFx8gpZhHfn44SOv37yh6zb4KpCyImWBFBqS2JySVGlv3rxmHEfBvCiFt7JcWEJknQQBo5A49WtGnkS0Fd59WnDOoFIiJXsb8q5RDlHjDBZ7CyIVTInn7du3t+3Rue+pfS3t9Dhi0MR1QVcVXdtJfmSKnM8nmqZFWIvSAissTbPheDzfKsdh6Knrho+fZfgcyaXyNFyj69q25Te/+bWEmRjDGkNZRsghk1EiAxh6IFLXHctlRGvBem+2LdM8cef2TOPIcDkBsNttboe+1pqwSo5k5T3JWYJQ/liXmcZv6V43dJstSotyPubIMk84LUsjbSUFOyxysNhizWrqms12wzIvGC3yncpXnPuBdx/fc393T7vZUHU7Kldhw1RSuy3b7V5IJkaLwHZd0IDVhtrVnM4v4ISHFYvMwlpHUwunK9YT/TDSbXY8HT/z4dNH1mXFGE9tK169+YLhIrmOT8czVVXz9suv+Ye//zuO5x7XdHz9q1+iy2LE+ZpXr1/RjwN1W9NuWrqu4fs/TcQ18v0P73n1KnN3f0fXNey2HUM/kHJmu91yPp/55ptvuLu743CQrebT0xNv37xhs6loGs+lv9BWDafjkeenI1pD3TjG/sL5+VkIJTFx7ke+/irQNjXb/QFF5n/zv/3f8f7TZ5pug9GGt2/fYrTkP6Qw8+799+Qo28fxfOL1m7c09YGX81E0ZHHBeflOJCUG73EZSSisr3j34SNaKw53h794gP3FFvJf/dtDVipjrSRgpJhY58i/+urX/B/+3f+er99+RbuR01eXzYYxmmVe0MZgtSGlyDCNGGNuK1IR28231Gt5kAPjMLDd7co8KpZhYmQstISrg/94PFL5CufdbSA5zdPttqy843C3FxJkXMgJpinStE0xEf8YpiEtbS7tkqjWrfEF8nbCaYOzhs2+I6SMdp5NIUemLEykqhIyQiz8LKncKDM1yZ501hLj+hNrjyoyk3RbWlw3qvLBSM6jzICmQsiEpm6IZcO0rCu/+93v6LoNaxhvm1iAjx+eeHp6YbvtqGvPMAzEmOgvPU3b0nQt9wc5WHQBKEqArrRM/TAWaKRsuBSqZBwKbRZKPuUq0L3LS08gYSvPPIxU3ss2OApKyPuqWHNkZV43jZBByvzx1atXtyH0biu8+Wmepe1JK2GdUFFcAMZ6vHM0dUvbdex3uxuC5eoE0UrT9z3jNPD8dOR0OvOv/+Z/wf3r17imkwDWdWGZLqQY2Gw2txGEMaZIYrygoBEz+uXyUuQA1a1VXxbx8v3w3bfs7+5BaXJKxce65Xi8oKzBOsO7d+/48uuvRNhc1VwugqFWaEKIXPrLzbXSdg0hlhQf7fjNb/6WGGbWIIEX/eXCPEu02rIsxDXQtR3LuqCV5od3737062rN3d0dWms+fXqP0ord9oArHcG3337Lt3/6E1+8/YLN7iCfV1XRdR2Xc8+333xLXbX8m3/zt3hviSnQNsLK01Zw0U3T8HB/z/HpmUt/EcLv6YWcEufTiRgjX371Jc55yS3dbFhj4NL3vHp8VbytUsW2TcvQ9zw/P2Gt4f/8f/1//HUtZNN5GT4aIEWmIfFweM2/+fXf8OXrt2x2e7RKxU/1Y5DAdXVrtSblhC2pQDlLFPm6CmsIpTi9vPDx0yeB6DknuJywCqUA+QCMVVxz8q7K3eu2SciTR3KJB7PaMS8jLy8vOOtELZ0Vd/d3rGtkuxGCZ4ySfjQMA+12I2t9YzHWYzCFN6/QWfHq4R7rNeO6YLwv5vKAs+62Jq+bttgeMinC+dTTtg3GSJjCVZ+z31c30OFm093mcvO8MI6ytEgpSwURJV7KGoeiAyUp5fe7B9ni9P1NupLyii5VBypR1YaqMjj3Ywm+hpGHxwNffvU1S0w4hcyAKi9V1howpertuo6qKLeNNnKBpci6RJyrmKa+bM4M5/OZ0+Ukhuy65unjJ2kFy+zxq6++un1+1/i3eZ7Z7oQfd02LfvX4wPly5vPzR7QS9LNS4OqGw+41Tlu+/PoruQByxpZ23jmHQrRPxogQ+uV4kg3g6YTznt/97m9wjQOrmKcL3imst1i/JS4zSYlKzF3dC2Rh7jtbLsCRZtOyTJLFeT5feHk5sq4Sd7bdNFSVY1mFY6VQjOPM508fWOLKL3/7G1xVsYbA8fk9X335FeuyFBlE4uPHT7x585qHh0fmOTJOAyEGmnplv7/jcHfPPA3E1BCXmU3Xcnp5YX+4Y55WTscjWlu+/uVX5BB5fHwtwbXOorTYizabLYfDnn//7//f5Hjmv/hf/pecL2f+/vf/JM/k6YWqalnXwMO9hJmlGNhuW37327/hcNhzPB5FdL0mjHVUjed4PPLw0GGMoq4tKTX0/cRud8/zyzMBxWa3ZwmJ3b7j7u5OLut2w+7wgHMO7x1D3zNNQ3G5iNPhuYh3/1Ovv1iB/fz6+fXz6+fX/5xf+p//R35+/fz6+fXz63+er58PsJ9fP79+fv1n+/r5APv59fPr59d/tq+fD7CfXz+/fn79Z/v6+QD7+fXz6+fXf7avnw+wn18/v35+/Wf7+v8CK1/xhgJ8LMUAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbcAAADnCAYAAACdbhioAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAwCklEQVR4nO3deXxU5b0G8OfMmkkyyUxmJgnZIUBCQkhCIKwGCVgQESrUKmi19QLtvSgauNparlZbbXFpFHqFqvTSXkVFUbwC0gKBslkFIqRAMLITspGErGSfmfvHSWzIvszMmeX5fj7zEWfOzPllm2fe97yLYLVaQURE5E5kUhdARERkaww3IiJyOww3IiJyOww3IiJyOww3IiJyO4qeHhQEgUMpiUhSVqtVkLoGcj1suRERkdthuBERkdthuBERkdthuBERkdthuBERkdthuBERkdvpcSoAuQE1AJ/WfzcCuClhLUREDsJwc1cyAKMAJAOIbr2vGMB5AN8CuAaAsxiJyE0JPW15w0ncLkoP4AcATABUXTxeD+ASgKMALjuuLKKB4CRuGgiGm7vRAbgfQHAfjm0E8DmAf4KtOHJavYVbdnZ2lFwuXyaTye60Wq16R9VF0hAEocJisewym81vpaSkXO72OIabG9FBDLYgAH39rNsC4DOIAUfkhHoKt+zs7CilUvlJUFCQTqfT1ahUqmZBYEPPXVmtVjQ1NSkrKyu1JSUllc3NzQu6CziOlnQnCyC22Przt60AMBfitTkiFyOXy5cFBQXpgoKCbqjVagabmxMEAWq1ujkoKOhGUFCQTi6XL+vuWIabu4iF2GIbCBWA2QDGoH/BSCQxmUx2p06nq5G6DnI8nU5XI5PJ7uzucYabO1BCbHmpB/EaagDzAIy2SUVEDmG1WvUqlapZ6jrI8VQqVXNP11gZbu4gBMBwG7yOAkAKAC8bvBaRg7Ar0jP19nNnuLmDCQDkNnqtKAB3Q2wNEhG5KIabq/MHEGDj14wDkGrj1yQiciCGm6szoG9z2vpDgNga9OntQCIi58Tlt1xdkp1eVwtgHIBDACx2OgeRnRmNxsTy8nKnfZ8zGAwtZWVlOY4+77p16wyPP/541Nq1ay+vWLGi3NHndwS23FyZACDMjq89DeI1OCIX5czBBjh/ffaQmpoaIwhCir3P43HfWLeSBMDPjq8vgxhwl+AZy3NpW29tGgG45WdaIvfHcHNlOtj/JxgMcXeBXDufRypeAG5r/e8QiNMq2lQDOAcx3PMAcDYVkctgt6Sr0gKY6IDzqAF8H/ZtITqaAHGy+hQAjwOYDHF+X0iH4/xa7/8+gJ9A3G2ByAnl5eWpBEFIWbhwYdSJEye8Zs6cGe3v75+k0WiSU1JSYj755JM+/QVv375du2jRosjo6Oh4X1/fZC8vr7EjRoyIX7Vq1ZC6urpOE8tWrlwZIghCyo4dO7SbNm3SJyQkjNJoNMn+/v5Jc+fOHXbp0qXvJhW11Xjs2DFfABAEIaXtlpqaGmO774aILTdX5QfHzUVTQgyCXQ46nz2FA5gFsZXW17mBCojB90MAHwCosk9pRIN19epV9bRp02JHjhxZ/+CDD5YWFxcrd+7cGXDvvfeO+OMf/3hx6dKlFT09/+WXXw6+ePGi19ixY2tnzpxZ1dDQIBw7dsw3MzMz5PDhw9ojR458q1B0jo3169ebsrKydDNmzKicPHlyTXZ2ts/OnTv1ubm5mjNnzuRqNBqrwWAwZ2RkFG3ZssVQWFioysjIKGp7flRUVKOtvxcMN1d1Gxy3DqQAYCSAbADXHXROW1MCSAOQiIG3QoMh7rqwGUCtjeoisqHjx4/7Llu2rOTNN9+81nbfwYMHr6enp8euWrUqcuHChVUBAQHdjn9+8803r8TGxjbJZLd26j3++OMh69atG7Jp0yZ9VwF58OBB/0OHDp1NTU2tb7vv7rvvHrpjx46AzZs365YsWVJhNBrNmZmZhYcPH9YWFhaqMjMzC230ZXeJ3ZKuzJGrDunhuutORkPcMWEqBte9KkAMuBG2KIrI9nx9fc1r1qy5JTTS0tLq5s+ff6Ompka+efPmHjvX4+LiOgUbAPzyl78sAYDdu3f7d/W8Rx55pKR9sAHAsmXLygDg6NGjksyYZbi5ojAAERKcdzKAsRKcdzASAdwLcVCMLT4MCOD2QOS04uPj6/R6faeW2bRp02oA4MSJE949Pb+6ulr2i1/8Inj06NGjtFptkkwmSxEEISU4ODgJAIqKirq8GDJ+/Pi6jvcNHTq0CQAqKytttThgv7Bb0hVpAPT4K2onCohv7GcB1PdyrDNIgLhXna2vTeogXrMr6uU4IgczmUxdjukNCQlpBoDq6upug6axsVGYOnXqyFOnTvmMGDGifu7cuRVGo7FFqVRaAeC1114b0tTU1GWDSK/Xmzve1/Y8s9ksycrWDDfqnzAAQ+H8UwMSAdwF+wy68QMQCIYbOZ3S0tIuf+MLCwuVAODn59cphNq89957ulOnTvksXLiwfOvWrZfbP3blyhXla6+9NsSmxdoZuyVdjQBpFzUWILaGYiWsoSf+EAd93AlxE1Z7GQd+NCSnc+bMGe+KiopO7+sHDhzQAkBycnKn7sM2586dUwPAggULOg0Y+dvf/uZrqxrlcrkVAFpaWmz1kl1iuLkik8Tn94bYKnK2N/e2YIuB/fekM4K7lpPTqa2tlf/iF7+4ZcbmwYMHvf/v//4vwNfX1/zAAw90OxWgbTj+/v3726/Tg9zcXNVzzz1ns4X+9Hp9CwCcP3/enh8/ne7tiVyFD4A5EOe+Sb1yhwzAMADpEK+FEXmocePG1b7//vvG7OxsnwkTJtS2zXOzWq1CZmbmlZ6mAdx///1VL774YuPGjRuDcnNzNWPGjKm7du2aat++fbrp06dX7dy50yZhNH369Opdu3bp77nnnuEzZ86s0mg0lsjIyMbly5ffsMXrt2HLzdWo4Rw/NRnEwSWzcOt6jI4mhzh/bTE6rzBCHs9gMNi372uQbF1fRERE49///vdv/P39W9555x3T559/ro+Li6v78MMPz/U2gdvPz8+SlZWVd/fdd984d+6cZtOmTUFnz571zsjIKNy2bdtFW9WYkZFRtnz58uKamhr5hg0bgl555ZWQv/zlLzbvjxKs1u5XxBUEwROWy3Ut0wDcDufpErNCXGD4SwDHANTBcYssh0KcmjAWjv9+1AN4DUCTg8/rgaxWa7c/3ZycnMuJiYlljqzHGeXl5aliY2MTFixYUP7xxx9flroeR8nJyTEmJiZGdfUYuyVdjQDnCTZArMULYugmA2gAkAXADHE1kxo7nTcMwH2QrtWohrgk2X6Jzk9EPWK4kW0IEAd0+EPsIgSAQgCVrf9ugRgEbdfnmiG2+PpLBWAmxEEjUnaHyiDONyQip8RwI/sJwb+ug1kBxLV7rATAmXb/fwY9L0isa31+BMRgc6bWKxE5HYabK9FCnF/ligTc+tsW2nprMwE9X79SQWwVElEnMTExTVarNVvqOpwJw82VyCHNsluOwOAiIhtyhkHlRERENsVwIyIit8NwcyVjwIEUzqIJtw6IISKnwnBzJeFguDkLM8SpDkTklBhuRETkdhhuRETkdhhuRETkdhhuRANRA8ctEE1E/cZJ3K4iEOIGmeQcjkBcL5Oc2stGY2J9ebnTvs9pDIaWp8rKcqSuwx2x5eYq9K03IuozZw42wPnrc2UMN6L+GuiOBkTkMAw3ov4qAvCN1EUQUU8YbkREbsBiseA3v/lNYHR0dLxarR4bGBg45qGHHoooLy+Xh4aGJoSGhia0Hbtu3TqDIAgp69atM2zdutUvNTU1RqvVJgmCkNJ2zDvvvKObP3/+0KioqNEajSZZo9Ekx8fHj3rhhRcCzWZzp/Pn5+crli1bFtZ2vFarTYqKihq9cOHCqNzcXFX7Ov/whz8YkpOTY/V6faJarR4bHBw8ZurUqSPefvttm118YX8vUX9YAXwldRFEnT300EMRmzdvNplMpuZFixaVqlQq6+7du3W33367T3Nzs6BUKjuN7922bZv+0KFD/mlpaVUPPvhgaX5+vrrtsWeffTZUJpMhKSnpZkhISFNVVZX8yJEjfs8880z48ePHfT799NNLbcfW1NTIpkyZEpufn6+ePHly9fe+971Kq9WK/Px81Z49e3Q/+MEPKuLi4poAYMWKFaFvvPFGcGhoaNPcuXMr/P39zcXFxcqcnByfjz/+WL906dIKW3w/GG5E/XVd6gKIbvXXv/7Vd/PmzabIyMjG48ePnzUajWYAaGhoKJg6derI0tJSZUhISKcdEw8cOOD/4YcfnvvBD35Q3fGxHTt2nI+Pj7/l6rLZbMa9994btW3bNsO+ffuup6en3wSA7du3a/Pz89WPPPLI9T/96U/57Z/T0NAg1NfXf7dw4LvvvmsKDAxsPnv27BmtVmtpf2xRUZHNMondkkT9UYmeN1UlksCmTZsMAPCf//mfRW3BBgBeXl7W3/72t9e6e96MGTMquwo2AOgYbAAgl8uxcuXK6wDw+eef+3V8XKPRWDre5+XlZdXr9bfcr1AorAqFolNLcsiQITabYMNwI+qPPABVUhdBdKvTp097A0B6enpNx8fS09NvyuXyLpccGDdu3M3uXrO4uFj+H//xH6EjR46M8/b2ThYEIUUQhJTbbrttFAAUFhZ+dx1t9uzZNYGBgc3r168Pvu2220a88MILgYcOHfJuaemcVd///vfLCwsLVSNHjoxfvnx56NatW/3Ky8vlA/iye8RuSSIiF1dTUyMHgLCwsE5polAooNPpOo8AARAcHNzc1f1lZWXycePGxRUUFKgSEhJuLliwoDwgIKBFoVCgsrJSvmnTpsDGxsbvuhoDAgIsX3zxxdmnn346ZM+ePbrDhw/7AYBOp2v58Y9/XLpmzZoitVptBYCNGzfmDxs2rHHz5s3G9evXB69fvz5YLpdbp02bVrV27dpro0ePtslEG4YbUV+ZAdyQugiiznx9fc0AcO3aNUXbwI02LS0tqKyslAcFBXXqMhSErvfQWrdunbGgoECVkZFRlJmZecvmTnv37vXZtGlTYMfnREdHN3/44YdXLBbLla+//trrb3/7m9/GjRtNr7/++hCLxYK1a9cWAmLYPvvss9efffbZ6wUFBYq9e/f6btmyJWDXrl36OXPmaPLy8s5oNJpBL27HbklXIAAYK3URhAYAX0tdBFFno0ePrgOAffv2aTs+tm/fPh+z2dyvnSDPnz+vBoD777+/08jFrs7Rnkwmw7hx4xpWr159fc+ePd8CwK5du3RdHRsaGtry8MMPV37++ecXJ06cWJOfn68+fvy4pj+1dluHLV6E7EwAMETqIojIWT388MPlAPDqq68OaX/9qqGhQVi9enVYf18vKiqqCQD27NlzS5AdOXJE84c//CG44/HHjx/3ys/P79QTWFhYqAQALy8vCwDU19cLu3fv9ul4XGNjo1BZWakAAB8fn04tzIFgtyRRXx2G2y+WrAYgB1DXy3F6AO3f9UIBJLf+uwhANsQpgYUQe3PJvu66667aRYsWlb3//vvGuLi4+Dlz5lQolUrr7t27dVqt1mwymZq764LsyrJly8rXr18f/Oyzz4YfOHBAGx0d3XDhwgWv/fv3+8+aNaty586dt0y2/vzzz/1+/etfhyUlJd2Mjo5uMJlMLQUFBcq9e/fqZDIZMjIySgDg5s2bslmzZsVGREQ0JiQk1IWHhzc2NDTIDh486Hfx4kWv9PT0yrFjxzbY4nvCcCPqq07j0FyDCmJgtRneeuuKD8RQ+hSABYASwO0dng+IYdbdJhWBABJbX+csgH8AKIW4HKejdwnSGAwtzrw4scZgsNnHpXfeeedKTExM/Z///GfT5s2bTTqdrmXWrFmVr7/+ekFERMSYgICAPg/UiIqKas7KyvrmySefDDt+/LjvoUOH/IYNG9bw0ksvXb3rrruqO4bb3Llzq69evXr9yy+/1O7evVt38+ZNuclkap4yZUr1qlWrSu64446bAKDVai2rV6++dvDgQb/s7GyfPXv26Ly9vc0RERGNL7300tUVK1aU2er7IVit3f+6CYLAHaucgQzAEwA6zSohh6kG8AHEpogLiMW/WlYpAALaPSZH57Bqzwpxbeg2Sog94wPVArH1dhjAKYhTBfvDarV2e/qcnJzLiYmJNntDdEenTp1SjxkzZvTcuXNvbN++/VLvz3AdOTk5xsTExKiuHnPaTzTUzjiIH6lJOuVw+mAzQhx3NBRimKl7PrxbAsTWnq0oWm8zINZXD2A/gHyIY3TINq5evaoIDQ1tkcv/9dGlpqZG9thjj4UDwPz58yulqk0KDDdX4IueP2qTx1K23tIAjIb4q+LM2rYlfADARQAnAHwLsaVok1EEHmzNmjVBn376acDEiRNrgoODm0tKSpRHjhzxKykpUaalpVU98sgjNlmz0VUw3IhckAxAAoDJAPwhttIG03UohWEAIiB2W/6z9dbtOlHUq1mzZlWfPn3a+9ChQ35VVVUKuVxujYqKaly6dGnJf/3Xf12XyTxrcLzHh5sfAO8O901B/za9bgCQhe4vlutbX7MrpwBc6XBfOW695kESswI40vVDBogtpzqIl+XsSQBggtjlmAYgGK4/l6etyzIVwCgAlyBemyuF4wefuLr58+fXzJ8/30WHPdmey4WbDJ176JLR/cit3gzt5rn9+RRsBRDdyzHdvV5oF/d9g38NzKsDcAgcTi251g6dtt+/eAAhrf/1BnASwGfo2xtyV7/DPYkAENP6nKTW57taK60vtBBbo3EAtkNsyTHgaKCcPty8AIxs9/8xEAOpvba5OVIZzBtNV88d1e7fFgD+JcDOZqBFOYgT0aBFQBziPgrigIv2fzwJELd5k0NszfVkJMQuub5SwLYDPJyZAPHrvRvi95ILwtBAOTzc9B1OmgSgp+nzSoiLc7jjJ9W+kAEYcxbYNwuoYbhJZjiAe9D9oFUFgPkQB3T0uDYR9YkCwCy47NRCcgIOCTc5xE+2gRDDrOPCYZ4aXP0h41AySc1C77MxuEKabakBTJC6CHJZdr8ebQLwUwBzIY7s8oYYZu1v1DOZBUjfJ3UVno2/p9LobiUVot7YreWmBDAd4koJAb0cSz0TAKhtssMRDdRNH8BYLnUVRNRXPbbc0ltvYa0H9qWZJ0C8WD4fwCQw2GzFWAYYuMiQZPZPl7oCIuqPHltuaa3/HQ+gCeLF3UOt9zVBnJMCiK20Ye2ODYM4ypFsx1gO6CuA8oHOeaCBE4A6b6BaC/hxhAORS+hTt6Sm9eYPYFHrfU0ALrf+WwkgCrwuQe7rehBwJRJIOC11JUTUFwO+5qbCrfPPiNzd+eFAXC4g58hVl2F82ZhYXu+8W94YNIaWsqfKcqSuwx25+uo9RA7zTSxQapK6CuoPZw42wPnrA4DU1NQYQRBSpK6jvxhuRH3U6AUcTQXM/Kshcnr8MyXqhxPJQHlv62sRkeQYbkT9YBWAv84Garl5LDmh/fv3e991113DAgMDx6hUqrEmk2nMlClTRmzcuPG7jU7WrVtnmDVrVnRYWFiCl5fXWF9f3+SxY8fGrl+//paZW3l5eSpBEFKOHTvmCwCCIKS03VJTU2Mc/bX1l9P39xI5FQG4GA3kxgHjj3GEMDmP3//+98af//znkTKZzDpjxozK6OjoxtLSUkVOTo7PW2+9FbhkyZIKAHjqqacihw8fXj9hwoSa4ODg5hs3bij27dvnv3z58qF5eXlea9euLQQAg8FgzsjIKNqyZYuhsLBQlZGRUdR2rqioKKdfVoLh5mqs4DuqEzgwDUg6Cai48R45gezsbK+f//znET4+PuasrKxvxo0b19D+8QsXLijbHXsmPj7+lnBqaGgQpk+fPuKNN94IfuKJJ0qHDh3abDQazZmZmYWHDx/WFhYWqjIzMwsd9fXYArslXUjaQUDgBldOoc4b+MckqasgEq1bt85kNpuFlStXFnYMNgCIjo7+7mNYx2ADAC8vL+vPfvaz62azWdi5c6efvet1BLbcXIiWq2M4DasMOJUgznszcVk0klh2drYvAMybN6/XDeHPnTun+vWvfx18+PBhbXFxsaqhoeGWRk5BQYFbbK7FcCMaoDITsP1u4JFNUldCnq6mpkYOAFFRUU09HZebm6uaMmXKqOrqakVKSkrttGnTqv39/c1yuRxXrlxRffLJJ4bGxka36NFjuBENQtEQ4JsYIDZP6krIk2m1WjMAXL58WaXX6zt1S7ZZs2ZNcGVlpWLt2rWXV6xYccs+F2+++WbAJ5984jYTXdwioYmk0qwCjo8DmvkxkSSUkpJSCwCfffZZj9fLLl26pAaAH/3oRxUdHztw4IBvV8+Ry+VWAGhpaRl8oQ7EcCMapMtRQH641FWQJ1uxYkWpXC63ZmZmhmRnZ3falKVttGR4eHgjAOzatUvb/vGPP/7Y78MPP+xycTm9Xt8CAOfPn1fZvnL74edNF1Kj7f0YcrwWJXBsPBCeDyhd68MtuYmUlJSGl1566epTTz0VOWnSpLiZM2dWRkdHN5aXl8tzcnJ8fH19zV999dW3jz/+eOnWrVuNP/nJT6K3bNlSMWTIkOazZ896HTp0yH/OnDkVO3fu1Hd87enTp1fv2rVLf8899wyfOXNmlUajsURGRjYuX778hhRfa18x3FzIwTRxhQxyPmdHAdcDgVCXmgnk/gwaQ4szL05s0Bhs9nFo1apVZYmJifWvvPJK8Jdffqnds2ePTq/Xt8TExNQ/8sgjZQAwYcKE+p07d+Y988wzofv37/c3m81CbGxs3V/+8pcLer3e3FW4ZWRklF25ckX96aefBmzYsCHIbDYL48ePr3X2cBOs1u4nTj0vcFaVM3n3AeD8CKmroO7M2clVS+zhV9buP9Ll5ORcTkxM5GQMD5WTk2NMTEyM6uoxXnMjspFj44EGbkFP5BQYbkQ2UmoSJ3azu4NIegw3IlsRxOuijWqpCyEihhuRDd30EbsniUhaDDciG7LKgDIjuyaJpMZwI7Kx3DhxWS4ikg7DzYV4NYBNAhfQrBK3w+GPikg6DDcXkr6P+7m5ipIgqSsg8mwMNxfCYHMd5QYgJ1HqKog8F8ONyA7MCk7oJpISw43ITq5EciscIqkw3Ijs5NwIoMmlNgkhch8MNxficxOI/UbqKqivzHLgn2OkroLIM7HTxIWomoHA68DZOKkrob6wyoDCEKmr8Gwvv2xMrHfiLW80GkPLU0+V5Uhdhztiy42I3JYzBxvg+PpSU1NjBEFIceQ5pcJwI7Kja2FAmUHqKog8D8PNhTSqgOJgqaug/qgIAGp9pa6CyPMw3FxInTfw7Uipq6D+qvLnUlxkf5s3b/afNGnSSJPJNEalUo0NDAwcM378+Jg1a9aY8vLyVIIgpBw7dswXAARBSGm7paamxrR/nUOHDnnPmjUrOiAgIFGlUo0NCQlJePDBByOuXLmi7HjOhQsXRgmCkJKXl6d65ZVXjCNHjoxTq9VjDQZD4qJFiyLLy8vlXdV64cIF5UMPPRQRFhaWoFKpxup0uqT09PThBw4c8LbV98Op+6OJ3MGh24Ax/5S6CnJnr776qvHJJ5+MNBqNzTNnzqwyGAwtpaWlitzcXO93333XuGTJkhsZGRlFW7ZsMRQWFqoyMjKK2p4bFRXV2Pbv999/3//hhx+OtlqtmD17dkVERETTyZMnvTdv3mzavXu37uDBg9/ExsY2dTx/RkZG2KFDh/zS09Orpk2bVn3kyBHtBx98YLx06ZL6yy+//Lb9sYcPH/a+++67R1RVVSmmTp1aPWfOnIry8nLF7t27dXfccUfsO++8c+G+++6rGuz3hOFGZGcNXuIu3YGlUldC7mrTpk0mpVJpPXnyZG5oaGhL+8eKiooURqPRnJmZWXj48GFtYWGhKjMzs7Dja1RVVcn+/d//fajZbBZ27tyZN3v27Nq2x1avXh3829/+NvTf/u3fIo8cOXKu43NPnDjh8/XXX+eOGDGiCQCam5sxadKkmK+++kq7f/9+7+nTp9e13b948eJhdXV18u3bt+fddddd353j8uXLytTU1FGPPvpo5Lx5805pNJpBdXiwW5LIzmq1wPnhUldB7k4ul1tVKlWnQBgyZEhLV8d39N577+mqqqrkc+bMudE+2ADgueeeKw4JCWn64osv/M6dO9dpaYInn3yyqC3YAECpVOJHP/pRGQD84x//8Gm7f8uWLbr8/Hz1j3/84+vtgw0AoqKimh977LHisrIy5WeffebXl5p7wpYbkQNYZOJ1N0HqQsgt3XvvvTeef/75sLi4uPh58+ZV3H777TUzZsyoDQkJ6VOwAcDXX3/tDQDTp0+v6fiYUqnEhAkTarZt22b46quvvNsHGQBMnDjxZsfnREZGNgFARUXFdznzxRdf+ABAfn6+auXKlZ1mgZ4/f14NALm5uV4ABtU1yXAjcoAjU4DkE4BPndSVkDt67rnnSoxGY8vbb79t+vOf/xz4P//zP4GCIGD8+PE1r7zyyrW0tLRef/Oqq6vlABASEtLc1ePBwcHNAFBRUdFpkIjBYDB3vE+hEOPFbDZ/95nuxo0bCgDYtWuXfteuXd3WUltbO+heRYYbkQM0qQArm21kR48++mj5o48+Wl5WVibPysry3bZtm+6jjz4yzps3b2Rubu7p3lpxfn5+ZgAoKirqNCoSAIqLi5UAoNPpOgVZX7Wd49133z3/wAMPDHrQSE94zY2IyI0YjUbzfffdV/XBBx9cWbhwYVlVVZV89+7dvoB4XQ4AWlo651xycnIdABw4cEDb8bHm5mYcPXpUCwATJ04ccP/DpEmTbgLAwYMHO53D1hhuRA5gFbg7N9nP9u3btRaLpdP9ZWVlSgDw9va2AIBer28BgPPnz3caFPLAAw9U+vv7m3fs2BGQlZXl0/6x3/zmN0EFBQWqSZMmVXe83tYfixcvrgwPD2/83//9X9OWLVv8uzpm7969PjU1NeyWJHIFFjmQnQJEX5S6EnJHixcvjvb29rYkJyfXRkRENFmtVnz55Zfa06dPe8fHx9fNnz+/BgCmT59evWvXLv0999wzfObMmVUajcYSGRnZuHz58hv+/v6W//7v/778k5/8ZNidd94Zc+edd1aEh4c3nTx50vvIkSN+RqOxeePGjVcGU6darbZ+9NFHF+bOnTvi/vvvH/7SSy/djI+Pr/P29rYUFBSocnJyvK9du6a+cuVKjlar7ZzW/cBwIyK3pdEYWpx58WSNxtDn0Yw9eeaZZwr27Nnjd/r0aZ8DBw74q1Qqa0hISNPq1auvrVq1qlStVlsBICMjo+zKlSvqTz/9NGDDhg1BZrNZGD9+fO3y5ctvAMCDDz5YGR4e/s0LL7ww5ODBg361tbVyo9HYvHjx4tIXX3yxKCoqqsvBJv0xYcKE+pMnT+a++OKLQXv27NFt3brVIAgCTCZTc3x8fN3TTz9d2NfpCz0RrNbu58k9LwhcNciJVOiAdSvErVTI9cSdAX74kdRVuJ5fWbsfipOTk3M5MTGxzJH1kPPIyckxJiYmRnX1GN8miYjI7TDciIjI7TDciIjI7TDciIjI7TDciByk3ABUdjmzh4hsjeHmQvyqgXHHpa6CBqokWAw4IrI/hpsLkVsAby68S0TUK4abiwkpBFSNvR9HROTJGG4uZsQ5QFMvdRVERM6N4eaCFDZZsIeIyH0x3FyMYAVmZEldBRGRc2O4uRgBgM9NQN0gdSVERM6L4eaCIq4C0RcAcFlrIqIuOe1WENQ9AUD6PuCbWHGfMHIdBaHAsIviz5Dsz2h8ObG8vN5p3+cMBk1LWdlTOVLX0ZEgCCnjx4+vPXr0aJ7UtQwUW24uSlcJJJ2Uugrqr5NJUlfgWZw52ADp6gsNDU0IDQ1NkOLcjsJwc1EKM5B8AtBwUjcRUScMNxcWdg0YeknqKoiInA/DzYUJAFKypa6CiJzFxo0b9ePGjYvRarVJXl5eY0eOHBn39NNPB9fX1wsAsGPHDq0gCCmFhYWqwsJClSAIKW23hQsXRnV8vaKiIsWiRYsiTSbTGJVKNXb48OHxa9eu7XaF1I8//thv2rRpw/V6faJKpRobHh4++qc//WlYWVlZp9EBbV2jN27ckC1ZsiQsNDQ0QaFQjF25cmWILb4XTt0fTb0LuAGYrgOlgVJXQkRSevTRR0PfeOONYJ1O1zJv3rwbvr6+ln379vmvWbMmNCsry//gwYPfjhgxojEjI6Po7bffDgSApUuXXm97fnJy8i0XOaqrq+WTJk2KValUljlz5lQ0NTXJdu7cqX/iiSeiZDIZHnvssfL2x69atWpIZmZmiL+/vzk9Pb3SZDK1nDlzRvPWW28FZWVl+R89evRsQECApf1zmpubhbS0tJjKykpFWlpatVarNQ8dOtQmCwwy3FycvhKIPwP83QQOwSPyUHv37vV54403goODg5u++uqrsxERES0A0NzcfG3WrFnD9+/f7//cc88FrVmzpjgzM7Nwy5YtBgDIzMws7O418/LyND/84Q/LNm/efEWhEKMiOzu7ZMKECfGvv/56cPtw2759uzYzMzMkKSnp5p49e84ZjUZz22Pr1q0zPP7441FPPvlk6J/+9Kf89ucoLS1VDh8+vOGLL77I8/PzuyX4Bovdkm4gJRuQm3s/jojc08aNG40AsGrVqqK2YAMApVKJ119/PV8mk+Hdd9819ec1vby8LBs2bMhvCzYASElJaUhOTq69ePGiV1VV1Xf5sW7dukAAePvtty+3DzYAWLFiRXlsbGz9tm3bAro6z+9///t8WwcbwJabW1A1AVGXgQvDpa6EiKRw6tQpbwCYPXt2TcfHxowZ0xgUFNRUUFCgKi8vlxsMhj59FI6MjGzs2I0IACEhIU0AUFpaKvf397cAwIkTJ3wVCoX1vffeC3jvvfc6vVZzc7NQUVGhKC4ulgcHB393frVabZ0wYYJdloJnuLkBdZM4MZjhRuSZampq5AAQERHR3NXjJpOpuaioqF/h5ufn1+VxCoXCCgBms/m7CyGVlZVys9ksvPbaa0N6es3q6upbwi0gIKBZJrNPByLDzU3E5AFfTgRq/KSuhHrC0a1kD1qt1gwA+fn5yvj4+E4DMkpLS5UAEBAQYJcLGFqt1myxWISqqqqT/XmeINhvoACvubkJQzkw6qzUVVBvgos57odsb/To0XUAsHv3bm3Hx06fPq0uKSlRhYaGNrVdD5PJZNb2La/BSkpKulldXS0/fvy4l61ec7AYbm5CAMONyFMtWbKkDABeffXVIYWFhd/1yLW0tOCJJ54Is1gseOCBB0rb7tfpdOaKigpFbW2tTQLuiSeeKAGApUuXRl2+fFnZ8fHq6mpZVlaWjy3O1VfslnQjplIgLB+4Fi51JUTkSHfcccfNn/3sZ8V//OMfgxMSEuLnzJlT4ePjY9m3b5/fuXPnNGPHjq19/vnnS9qOT0tLqz59+rT39OnTR06ePLlGrVZbk5KS6hYvXlw1kPPPnz+/5pe//GXB7373u9BRo0aNvv3226siIyObamtrZdeuXVMdPXpUm5KSUjtjxoxztvuqe8ZwcyO+N8UFla+FgX1fTiigHPAf0FsHDZTBoGlx5sWTDQZNS+9H9c2GDRsKkpOT6958883ATz75xNDS0iKEh4c3PvXUUwW/+tWvSry8vL7bJOt3v/tdUWVlpXzv3r26EydO+JrNZixYsKB8oOEGAC+++GJxWlpa7dq1awOPHz/uu3fvXoWvr685KCioefHixaUPPfTQDdt8pX0jWK3dbwr2vCBwxzAXUxwEvL0UMDvtn7PnijsD/PAjqatwPb+yWrv9qJaTk3M5MTGxzJH1kPPIyckxJiYmRnX1GK+5uRm/akDZ5WBgkpLMDIz5p9RVEHkOhpub0dQDUw9LXQV1JFjFXRyIyDEYbm5GgLiJqd6hvdvUm/HHxA8eROQYDDc35HNT3MiUnIRVHOgjt/nqeUTUHYabGxIgrjVJziHgBlcmIXI0hpubCioBhjtsRgn1xL8KUNhswDcR9QXDzU2pmwBtDQBO5pDctAOcdmhPPU1nIvfV28+d4ebGbv87pwVIzb9SvAZK9iEIQkVTU1On5Z7I/TU1NSkFQajo7nGGmxtTN/KNVWoRVwETpxjbjcVi2VVZWdlpsWByf5WVlVqLxbKru8cZbm5M3QhM/kLqKjyXzCxuRUT2Yzab3yopKaksKSkJaGxsVLKL0r1ZrVY0NjYqS0pKAkpKSirNZvNb3R3LRZrcGK/zSCu4GBjBQT12lZKScjk7O3tBUVHRspKSkjutVqtR6prIvgRBqLBYLB+Yzea3UlJSLnd3HMPNzakbAXkL15qUwph/igN7yL5a3+B+2XojAsBuSbc35p/iVjjkWD61XG6LSEoMNyI7GHYRCC2Qugoiz8Vw8wDjj0ldgedJPcprnkRSYri5OQHiwAZO5nacwJLWCfREJBmGmwcIKgHicqWuwjMIFmD0aUDHHbeJJMVw8wAKszhqkuzP56bYJUlE0mK4eYioy+KUALKv2w7xgwSRM2C4eYiYPK5Mb2/KJnEHAA4kIZIew81DCFYuomxvhnIut0XkLBhuHkLVBKQdlLoK98YNYomcB8PNQwgA5Gapq3BvCafYJUnkLLjioAeJvALoKoBKvdSVkMeQAejrbmsRAGJvvev0mXgbF0SeguHmQYzlgHcdw40GKQaAdx+PDQSQ3Mdj5egUhDFjeBGTBobhRuQMdAC8bPyaowBE2/g1ATGwVHZ43S4olRziSwPDcPMgZhlg5UUh+1Gg9y44bwCT0fni3AiIAWdr/HmTh2K4eZDTo4HiYKmrcD8yiI0kzWIAll4OFiC20Bg6RHbFcPMQFgGo8gesHB9rc1MATAcgs3W3IhENGMPNA7TIxVbbgWlSV+J+/ACMBufUEDkbhpsbM8uAvBjg0G1AqQkw86dtcyMBBEldBBF1wrc7Z6dF3+cJtWoAcE0vhtrVCHZF2osawASpiyCiLjHcHEVA50EEKRDDqycJ6NcouiYAHwM413ZOspsEAEapiyCiLjHc7GUobp0LlAggtMMxvhAnrtpQM4BLtn1J6kZXn1eIyDkw3HojQ/etKwHiMDlNF49FwWETXdurB2B1/GmJiJyKe4Vb28focIjhYgveAMaj+4/oTvbx/QAAro9MRJ7OPcJtCMQuvrZWlBe6bk15ALbaiIhcLdw0+FfF3gCmQWw1hUMMNw93HcBVqYsgInICzhluieg6rJIBtF/RXgan6hKUkhXATQDVUhfiYazgryCRM3J8uHkBaL++YRKAsA7H+KPfc7sI+LvUBXiYUwBSAZikLoSIOrFvuAkQuxKNEAdlAIAPgGF2PavHqpW6AA/TAIAbshA5J/uFmw5i9+IUiN2HztkB6jYKIb7ZkmNdgjieiYici+0jxwRgDPq9sgYNzmWI19zIsc5A3J6NiJyLbcMtHMC9EJdKJyIikojtwi0MwH3gkHwiIpKcbdaLl0NcBJjBJhkf2HyZSuqDCnBuIZEzsk24TYU4pJ8kkwhe4pSCHmJvPBE5l8GHmw5APDiT1QnwTVYa/NUncj6DD7cYAIGDL4QGRwAwG5xC6Gi9bcdHRNIYXLjJAdxmm0Jo8LwAjAOnFDqKAOB2qYsgoi4NLtyCwHdSJzMKwAKwBWdvcogtZXZaEDmnwYXbaIjNBXIaAoA4AD+EOMjER9py3JICQDrEFeVsMyKLiGxt4O0ubwCRtiuEbMsLwPcBFAM4CeAouNfbYMkgDtpJgbgADweSEDmvwYVbiO0KIdsTIK57GAhxo4VTAIokrcg1GSB+D5Mgfp5jZwWR8+MVMw8gh7j+4WgAHwHIl7Ycl9C2mftYiNcxjdKWQ0T9xHDzIH4Qr8W9D3EXAepaAsR9caMgtn7Z/UjkehhuHkYLcdOGYgAWiWtxRhMAzAT3yiVydQMf7MULDy4rFcBEqYtwQglgsBG5i4GHW7oNqyCHkkF8I+fORP/iBbErksFG5B4GHm4y8GKECxsCYAmA6eBcLQC4E8BQqYsgIpvhNTcP5gdx9TQzgOMA6qQtRzJyiNci+VmNyH0w3DycDOL6iCkAtsB+oyjbtvxr23OuDMA5O52rv4ZDHBlJRO5jYOE2EpzA7UZkECd5LwZwAcBXsE3ICRA3aE+DuO9ZAP7VBdoAcUL5NgDVNjjXQOkhXj5m1yyRexlYuPkCUNm2EJKeL8T1KGMB5AEoAfA1gGYALX18DSXEX6rJEINjFLreIdwL4jWu+wGUAjgGcXpCX89jCz6t5w9y4DmJyDH6H24KiKMQyG2pIc6FswCYBuBi6w0QW3TXOhwfDXGJKkDct3YIxJDryzWskNZbPMRuyq/huO5KbkVI5L76H27+4HhpDyGD2ECPbb0BQG3rrT1/iEtVDYYCYitvKMQA3QegHkD5IF+3K3qIi0oHg4NIiNxV/8JNC+A+cAK3B/NtvdmLF8S96IYBqIJ4DRAA9sIGozmvhQLZKRgffwaRwy/0fjwRuSzBauVGKERE5F44SIyIiNwOw42IiNwOw42IiNwOw42IiNwOw42IiNwOw42IiNzO/wPBAGpJZYKY6gAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# show results\n", + "new_palette = get_new_pallete(len(labels))\n", + "mask, patches = get_new_mask_pallete(predict, new_palette, out_label_flag=True, labels=labels)\n", + "img = image[0].permute(1,2,0)\n", + "img = img * 0.5 + 0.5\n", + "img = Image.fromarray(np.uint8(255*img)).convert(\"RGBA\")\n", + "seg = mask.convert(\"RGBA\")\n", + "out = Image.blend(img, seg, alpha)\n", + "plt.axis('off')\n", + "plt.imshow(img)\n", + "plt.figure()\n", + "plt.legend(handles=patches, loc='upper right', bbox_to_anchor=(1.5, 1), prop={'size': 20})\n", + "plt.axis('off')\n", + "plt.imshow(seg)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/submodules/lang_seg/modules/lseg_module.py b/submodules/lang_seg/modules/lseg_module.py new file mode 100644 index 0000000000000000000000000000000000000000..1aacbcc90cf037e4cafc98850b8d580a9321dd26 --- /dev/null +++ b/submodules/lang_seg/modules/lseg_module.py @@ -0,0 +1,183 @@ +import re +import torch +import torch.nn as nn +import torchvision.transforms as transforms +from argparse import ArgumentParser +import pytorch_lightning as pl +from .lsegmentation_module import LSegmentationModule +from .models.lseg_net import LSegNet +from encoding.models.sseg.base import up_kwargs + +import os +import clip +import numpy as np + +from scipy import signal +import glob + +from PIL import Image +import matplotlib.pyplot as plt +import pandas as pd + + +class LSegModule(LSegmentationModule): + def __init__(self, data_path, dataset, batch_size, base_lr, max_epochs, **kwargs): + super(LSegModule, self).__init__( + data_path, dataset, batch_size, base_lr, max_epochs, **kwargs + ) + + if dataset == "citys": + self.base_size = 2048 + self.crop_size = 768 + else: + self.base_size = 520 + self.crop_size = 480 + + use_pretrained = True + norm_mean= [0.5, 0.5, 0.5] + norm_std = [0.5, 0.5, 0.5] + + print('** Use norm {}, {} as the mean and std **'.format(norm_mean, norm_std)) + + train_transform = [ + transforms.ToTensor(), + transforms.Normalize(norm_mean, norm_std), + ] + + val_transform = [ + transforms.ToTensor(), + transforms.Normalize(norm_mean, norm_std), + ] + + self.train_transform = transforms.Compose(train_transform) + self.val_transform = transforms.Compose(val_transform) + + self.trainset = self.get_trainset( + dataset, + augment=kwargs["augment"], + base_size=self.base_size, + crop_size=self.crop_size, + ) + + self.valset = self.get_valset( + dataset, + augment=kwargs["augment"], + base_size=self.base_size, + crop_size=self.crop_size, + ) + + use_batchnorm = ( + (not kwargs["no_batchnorm"]) if "no_batchnorm" in kwargs else True + ) + # print(kwargs) + + labels = self.get_labels('ade20k') + + self.net = LSegNet( + labels=labels, + backbone=kwargs["backbone"], + features=kwargs["num_features"], + crop_size=self.crop_size, + arch_option=kwargs["arch_option"], + block_depth=kwargs["block_depth"], + activation=kwargs["activation"], + ) + + self.net.pretrained.model.patch_embed.img_size = ( + self.crop_size, + self.crop_size, + ) + + self._up_kwargs = up_kwargs + self.mean = norm_mean + self.std = norm_std + + self.criterion = self.get_criterion(**kwargs) + + def get_labels(self, dataset): + labels = [] + path = 'label_files/{}_objectInfo150.txt'.format(dataset) + assert os.path.exists(path), '*** Error : {} not exist !!!'.format(path) + f = open(path, 'r') + lines = f.readlines() + for line in lines: + label = line.strip().split(',')[-1].split(';')[0] + labels.append(label) + f.close() + if dataset in ['ade20k']: + labels = labels[1:] + return labels + + + @staticmethod + def add_model_specific_args(parent_parser): + parser = LSegmentationModule.add_model_specific_args(parent_parser) + parser = ArgumentParser(parents=[parser]) + + parser.add_argument( + "--backbone", + type=str, + default="clip_vitl16_384", + help="backbone network", + ) + + parser.add_argument( + "--num_features", + type=int, + default=256, + help="number of featurs that go from encoder to decoder", + ) + + parser.add_argument("--dropout", type=float, default=0.1, help="dropout rate") + + parser.add_argument( + "--finetune_weights", type=str, help="load weights to finetune from" + ) + + parser.add_argument( + "--no-scaleinv", + default=True, + action="store_false", + help="turn off scaleinv layers", + ) + + parser.add_argument( + "--no-batchnorm", + default=False, + action="store_true", + help="turn off batchnorm", + ) + + parser.add_argument( + "--widehead", default=False, action="store_true", help="wider output head" + ) + + parser.add_argument( + "--widehead_hr", + default=False, + action="store_true", + help="wider output head", + ) + + parser.add_argument( + "--arch_option", + type=int, + default=0, + help="which kind of architecture to be used", + ) + + parser.add_argument( + "--block_depth", + type=int, + default=0, + help="how many blocks should be used", + ) + + parser.add_argument( + "--activation", + choices=['lrelu', 'tanh'], + default="lrelu", + help="use which activation to activate the block", + ) + + return parser diff --git a/submodules/lang_seg/modules/lseg_module_zs.py b/submodules/lang_seg/modules/lseg_module_zs.py new file mode 100644 index 0000000000000000000000000000000000000000..fd944d0ecd2204721f87b0b65302ed239ad86837 --- /dev/null +++ b/submodules/lang_seg/modules/lseg_module_zs.py @@ -0,0 +1,148 @@ +import re +import torch +import torch.nn as nn +import torchvision.transforms as transforms +from argparse import ArgumentParser +import pytorch_lightning as pl +from .lsegmentation_module_zs import LSegmentationModuleZS +from .models.lseg_net_zs import LSegNetZS, LSegRNNetZS +from encoding.models.sseg.base import up_kwargs +import os +import clip +import numpy as np +from scipy import signal +import glob +from PIL import Image +import matplotlib.pyplot as plt +import pandas as pd + + +class LSegModuleZS(LSegmentationModuleZS): + def __init__(self, data_path, dataset, batch_size, base_lr, max_epochs, **kwargs): + super(LSegModuleZS, self).__init__( + data_path, dataset, batch_size, base_lr, max_epochs, **kwargs + ) + label_list = self.get_labels(dataset) + self.len_dataloader = len(label_list) + + # print(kwargs) + if kwargs["use_pretrained"] in ['False', False]: + use_pretrained = False + elif kwargs["use_pretrained"] in ['True', True]: + use_pretrained = True + + if kwargs["backbone"] in ["clip_resnet101"]: + self.net = LSegRNNetZS( + label_list=label_list, + backbone=kwargs["backbone"], + features=kwargs["num_features"], + aux=kwargs["aux"], + use_pretrained=use_pretrained, + arch_option=kwargs["arch_option"], + block_depth=kwargs["block_depth"], + activation=kwargs["activation"], + ) + else: + self.net = LSegNetZS( + label_list=label_list, + backbone=kwargs["backbone"], + features=kwargs["num_features"], + aux=kwargs["aux"], + use_pretrained=use_pretrained, + arch_option=kwargs["arch_option"], + block_depth=kwargs["block_depth"], + activation=kwargs["activation"], + ) + + def get_labels(self, dataset): + labels = [] + path = 'label_files/fewshot_{}.txt'.format(dataset) + assert os.path.exists(path), '*** Error : {} not exist !!!'.format(path) + f = open(path, 'r') + lines = f.readlines() + for line in lines: + label = line.strip() + labels.append(label) + f.close() + print(labels) + return labels + + @staticmethod + def add_model_specific_args(parent_parser): + parser = LSegmentationModuleZS.add_model_specific_args(parent_parser) + parser = ArgumentParser(parents=[parser]) + + parser.add_argument( + "--backbone", + type=str, + default="vitb16_384", + help="backbone network", + ) + + parser.add_argument( + "--num_features", + type=int, + default=256, + help="number of featurs that go from encoder to decoder", + ) + + parser.add_argument("--dropout", type=float, default=0.1, help="dropout rate") + + parser.add_argument( + "--finetune_weights", type=str, help="load weights to finetune from" + ) + + parser.add_argument( + "--no-scaleinv", + default=True, + action="store_false", + help="turn off scaleinv layers", + ) + + parser.add_argument( + "--no-batchnorm", + default=False, + action="store_true", + help="turn off batchnorm", + ) + + parser.add_argument( + "--widehead", default=False, action="store_true", help="wider output head" + ) + + parser.add_argument( + "--widehead_hr", + default=False, + action="store_true", + help="wider output head", + ) + + parser.add_argument( + "--use_pretrained", + type=str, + default="True", + help="whether use the default model to intialize the model", + ) + + parser.add_argument( + "--arch_option", + type=int, + default=0, + help="which kind of architecture to be used", + ) + + parser.add_argument( + "--block_depth", + type=int, + default=0, + help="how many blocks should be used", + ) + + parser.add_argument( + "--activation", + choices=['relu', 'lrelu', 'tanh'], + default="relu", + help="use which activation to activate the block", + ) + + return parser diff --git a/submodules/lang_seg/modules/lsegmentation_module.py b/submodules/lang_seg/modules/lsegmentation_module.py new file mode 100644 index 0000000000000000000000000000000000000000..e287b343840fb0286c094beb169510d3577b3c53 --- /dev/null +++ b/submodules/lang_seg/modules/lsegmentation_module.py @@ -0,0 +1,304 @@ +import types +import time +import random +import clip +import torch +import torch.nn as nn +import torchvision.transforms as transforms + +from argparse import ArgumentParser + +import pytorch_lightning as pl + +from data import get_dataset, get_available_datasets + +from encoding.models import get_segmentation_model +from encoding.nn import SegmentationLosses + +from encoding.utils import batch_pix_accuracy, batch_intersection_union + +# add mixed precision +import torch.cuda.amp as amp +import numpy as np + +from encoding.utils import SegmentationMetric + +class LSegmentationModule(pl.LightningModule): + def __init__(self, data_path, dataset, batch_size, base_lr, max_epochs, **kwargs): + super().__init__() + + self.data_path = data_path + self.batch_size = batch_size + self.base_lr = base_lr / 16 * batch_size + self.lr = self.base_lr + + self.epochs = max_epochs + self.other_kwargs = kwargs + self.enabled = False #True mixed precision will make things complicated and leading to NAN error + self.scaler = amp.GradScaler(enabled=self.enabled) + + def forward(self, x): + return self.net(x) + + def evaluate(self, x, target=None): + pred = self.net.forward(x) + if isinstance(pred, (tuple, list)): + pred = pred[0] + if target is None: + return pred + correct, labeled = batch_pix_accuracy(pred.data, target.data) + inter, union = batch_intersection_union(pred.data, target.data, self.nclass) + + return correct, labeled, inter, union + + def evaluate_random(self, x, labelset, target=None): + pred = self.net.forward(x, labelset) + if isinstance(pred, (tuple, list)): + pred = pred[0] + if target is None: + return pred + correct, labeled = batch_pix_accuracy(pred.data, target.data) + inter, union = batch_intersection_union(pred.data, target.data, self.nclass) + + return correct, labeled, inter, union + + + def training_step(self, batch, batch_nb): + img, target = batch + with amp.autocast(enabled=self.enabled): + out = self(img) + multi_loss = isinstance(out, tuple) + if multi_loss: + loss = self.criterion(*out, target) + else: + loss = self.criterion(out, target) + loss = self.scaler.scale(loss) + final_output = out[0] if multi_loss else out + train_pred, train_gt = self._filter_invalid(final_output, target) + if train_gt.nelement() != 0: + self.train_accuracy(train_pred, train_gt) + self.log("train_loss", loss) + return loss + + def training_epoch_end(self, outs): + self.log("train_acc_epoch", self.train_accuracy.compute()) + + def validation_step(self, batch, batch_nb): + img, target = batch + out = self(img) + multi_loss = isinstance(out, tuple) + if multi_loss: + val_loss = self.criterion(*out, target) + else: + val_loss = self.criterion(out, target) + final_output = out[0] if multi_loss else out + valid_pred, valid_gt = self._filter_invalid(final_output, target) + self.val_iou.update(target, final_output) + pixAcc, iou = self.val_iou.get() + self.log("val_loss_step", val_loss) + self.log("pix_acc_step", pixAcc) + self.log( + "val_acc_step", + self.val_accuracy(valid_pred, valid_gt), + ) + self.log("val_iou", iou) + + def validation_epoch_end(self, outs): + pixAcc, iou = self.val_iou.get() + self.log("val_acc_epoch", self.val_accuracy.compute()) + self.log("val_iou_epoch", iou) + self.log("pix_acc_epoch", pixAcc) + + self.val_iou.reset() + + def _filter_invalid(self, pred, target): + valid = target != self.other_kwargs["ignore_index"] + _, mx = torch.max(pred, dim=1) + return mx[valid], target[valid] + + def configure_optimizers(self): + params_list = [ + {"params": self.net.pretrained.parameters(), "lr": self.base_lr}, + ] + if hasattr(self.net, "scratch"): + print("Found output scratch") + params_list.append( + {"params": self.net.scratch.parameters(), "lr": self.base_lr * 10} + ) + if hasattr(self.net, "auxlayer"): + print("Found auxlayer") + params_list.append( + {"params": self.net.auxlayer.parameters(), "lr": self.base_lr * 10} + ) + if hasattr(self.net, "scale_inv_conv"): + print(self.net.scale_inv_conv) + print("Found scaleinv layers") + params_list.append( + { + "params": self.net.scale_inv_conv.parameters(), + "lr": self.base_lr * 10, + } + ) + params_list.append( + {"params": self.net.scale2_conv.parameters(), "lr": self.base_lr * 10} + ) + params_list.append( + {"params": self.net.scale3_conv.parameters(), "lr": self.base_lr * 10} + ) + params_list.append( + {"params": self.net.scale4_conv.parameters(), "lr": self.base_lr * 10} + ) + + if self.other_kwargs["midasproto"]: + print("Using midas optimization protocol") + + opt = torch.optim.Adam( + params_list, + lr=self.base_lr, + betas=(0.9, 0.999), + weight_decay=self.other_kwargs["weight_decay"], + ) + sch = torch.optim.lr_scheduler.LambdaLR( + opt, lambda x: pow(1.0 - x / self.epochs, 0.9) + ) + + else: + opt = torch.optim.SGD( + params_list, + lr=self.base_lr, + momentum=0.9, + weight_decay=self.other_kwargs["weight_decay"], + ) + sch = torch.optim.lr_scheduler.LambdaLR( + opt, lambda x: pow(1.0 - x / self.epochs, 0.9) + ) + return [opt], [sch] + + def train_dataloader(self): + return torch.utils.data.DataLoader( + self.trainset, + batch_size=self.batch_size, + shuffle=True, + num_workers=16, + worker_init_fn=lambda x: random.seed(time.time() + x), + ) + + def val_dataloader(self): + return torch.utils.data.DataLoader( + self.valset, + batch_size=self.batch_size, + shuffle=False, + num_workers=16, + ) + + def get_trainset(self, dset, augment=False, **kwargs): + print(kwargs) + if augment == True: + mode = "train_x" + else: + mode = "train" + + print(mode) + dset = get_dataset( + dset, + root=self.data_path, + split="train", + mode=mode, + transform=self.train_transform, + **kwargs + ) + + self.num_classes = dset.num_class + self.train_accuracy = pl.metrics.Accuracy() + + return dset + + def get_valset(self, dset, augment=False, **kwargs): + self.val_accuracy = pl.metrics.Accuracy() + self.val_iou = SegmentationMetric(self.num_classes) + + if augment == True: + mode = "val_x" + else: + mode = "val" + + print(mode) + return get_dataset( + dset, + root=self.data_path, + split="val", + mode=mode, + transform=self.val_transform, + **kwargs + ) + + + def get_criterion(self, **kwargs): + return SegmentationLosses( + se_loss=kwargs["se_loss"], + aux=kwargs["aux"], + nclass=self.num_classes, + se_weight=kwargs["se_weight"], + aux_weight=kwargs["aux_weight"], + ignore_index=kwargs["ignore_index"], + ) + + @staticmethod + def add_model_specific_args(parent_parser): + parser = ArgumentParser(parents=[parent_parser], add_help=False) + parser.add_argument( + "--data_path", type=str, help="path where dataset is stored" + ) + parser.add_argument( + "--dataset", + choices=get_available_datasets(), + default="ade20k", + help="dataset to train on", + ) + parser.add_argument( + "--batch_size", type=int, default=16, help="size of the batches" + ) + parser.add_argument( + "--base_lr", type=float, default=0.004, help="learning rate" + ) + parser.add_argument("--momentum", type=float, default=0.9, help="SGD momentum") + parser.add_argument( + "--weight_decay", type=float, default=1e-4, help="weight_decay" + ) + parser.add_argument( + "--aux", action="store_true", default=False, help="Auxilary Loss" + ) + parser.add_argument( + "--aux-weight", + type=float, + default=0.2, + help="Auxilary loss weight (default: 0.2)", + ) + parser.add_argument( + "--se-loss", + action="store_true", + default=False, + help="Semantic Encoding Loss SE-loss", + ) + parser.add_argument( + "--se-weight", type=float, default=0.2, help="SE-loss weight (default: 0.2)" + ) + + parser.add_argument( + "--midasproto", action="store_true", default=False, help="midasprotocol" + ) + + parser.add_argument( + "--ignore_index", + type=int, + default=-1, + help="numeric value of ignore label in gt", + ) + parser.add_argument( + "--augment", + action="store_true", + default=False, + help="Use extended augmentations", + ) + + return parser diff --git a/submodules/lang_seg/modules/lsegmentation_module_zs.py b/submodules/lang_seg/modules/lsegmentation_module_zs.py new file mode 100644 index 0000000000000000000000000000000000000000..fcb459e1854d5c184a0ed54dd83444ebd336df38 --- /dev/null +++ b/submodules/lang_seg/modules/lsegmentation_module_zs.py @@ -0,0 +1,445 @@ +import types +import time +import random +import clip +import torch +import torch.nn as nn +import torchvision.transforms as transforms + +from argparse import ArgumentParser + +import pytorch_lightning as pl + +from encoding.models import get_segmentation_model +from encoding.nn import SegmentationLosses + +from encoding.utils import batch_pix_accuracy, batch_intersection_union + +# add mixed precision +import torch.cuda.amp as amp +import numpy as np +from encoding.utils.metrics import SegmentationMetric + +# get fewshot dataloader +from fewshot_data.model.hsnet import HypercorrSqueezeNetwork +from fewshot_data.common.logger import Logger, AverageMeter +from fewshot_data.common.evaluation import Evaluator +from fewshot_data.common import utils +from fewshot_data.data.dataset import FSSDataset + +class Fewshot_args: + datapath = 'fewshot_data/Datasets_HSN' + benchmark = 'pascal' + logpath = '' + nworker = 8 + bsz = 20 + fold = 0 + + +class LSegmentationModuleZS(pl.LightningModule): + def __init__(self, data_path, dataset, batch_size, base_lr, max_epochs, **kwargs): + super().__init__() + + self.batch_size = batch_size + self.base_lr = base_lr / 16 * batch_size + self.lr = self.base_lr + + self.epochs = max_epochs + self.other_kwargs = kwargs + self.enabled = False #True mixed precision will make things complicated and leading to NAN error + self.scaler = amp.GradScaler(enabled=self.enabled) + # for whether fix the encoder or not + self.fixed_encoder = True if kwargs["use_pretrained"] in ['clip_fixed'] else False + + # fewshot hyperparameters + self.cross_entropy_loss = nn.CrossEntropyLoss() + self.args = self.get_fewshot_args() + if data_path: + self.args.datapath = data_path + self.args.logpath = self.other_kwargs["logpath"] + self.args.benchmark = dataset + self.args.bsz = self.batch_size + self.args.fold = self.other_kwargs["fold"] + self.args.nshot = self.other_kwargs["nshot"] + self.args.finetune_mode = self.other_kwargs["finetune_mode"] + Logger.initialize(self.args, training=True) + Evaluator.initialize() + if kwargs["backbone"] in ["clip_resnet101"]: + FSSDataset.initialize(img_size=480, datapath=self.args.datapath, use_original_imgsize=False, imagenet_norm=True) + else: + FSSDataset.initialize(img_size=480, datapath=self.args.datapath, use_original_imgsize=False) + self.best_val_miou = float('-inf') + self.num_classes = 2 + self.labels = ['others', ''] + + self.fewshot_trn_loss = 100 + self.fewshot_trn_miou = 0 + self.fewshot_trn_fb_iou = 0 + + def get_fewshot_args(self): + return Fewshot_args() + + def forward(self, x, class_info): + return self.net(x, class_info) + + + def training_step(self, batch, batch_nb): + if self.args.finetune_mode: + if self.args.nshot == 5: + bshape = batch['support_imgs'].shape + img = batch['support_imgs'].view(-1, bshape[2], bshape[3], bshape[4]) + target = batch['support_masks'].view(-1, bshape[3], bshape[4]) + class_info = batch['class_id'] + for i in range(1, 5): + class_info = torch.cat([class_info, batch['class_id']]) + with amp.autocast(enabled=self.enabled): + out = self(img, class_info) + loss = self.criterion(out, target) + loss = self.scaler.scale(loss) + self.log("train_loss", loss) + # 3. Evaluate prediction + if self.args.benchmark == 'pascal' and batch['support_ignore_idxs'] is not None: + query_ignore_idx = batch['support_ignore_idxs'].view(-1, bshape[3], bshape[4]) + area_inter, area_union = Evaluator.classify_prediction(out.argmax(dim=1), target, query_ignore_idx) + else: + area_inter, area_union = Evaluator.classify_prediction(out.argmax(dim=1), target) + else: + img = batch['support_imgs'].squeeze(1) + target = batch['support_masks'].squeeze(1) + class_info = batch['class_id'] + with amp.autocast(enabled=self.enabled): + out = self(img, class_info) + loss = self.criterion(out, target) + loss = self.scaler.scale(loss) + self.log("train_loss", loss) + # 3. Evaluate prediction + if self.args.benchmark == 'pascal' and batch['support_ignore_idxs'] is not None: + query_ignore_idx = batch['support_ignore_idxs'].squeeze(1) + area_inter, area_union = Evaluator.classify_prediction(out.argmax(dim=1), target, query_ignore_idx) + else: + area_inter, area_union = Evaluator.classify_prediction(out.argmax(dim=1), target) + else: + img = torch.cat([batch['support_imgs'].squeeze(1), batch['query_img']], dim=0) + target = torch.cat([batch['support_masks'].squeeze(1), batch['query_mask']], dim=0) + class_info=torch.cat([batch['class_id'], batch['class_id']], dim=0) + with amp.autocast(enabled=self.enabled): + out = self(img, class_info) + loss = self.criterion(out, target) + loss = self.scaler.scale(loss) + + self.log("train_loss", loss) + # 3. Evaluate prediction + if self.args.benchmark == 'pascal' and batch['query_ignore_idx'] is not None: + query_ignore_idx = torch.cat([batch['support_ignore_idxs'].squeeze(1), batch['query_ignore_idx']], dim=0) + area_inter, area_union = Evaluator.classify_prediction(out.argmax(dim=1), target, query_ignore_idx) + else: + area_inter, area_union = Evaluator.classify_prediction(out.argmax(dim=1), target) + self.train_average_meter.update(area_inter, area_union, class_info, loss.detach().clone()) + if self.global_rank == 0: + return_value = self.train_average_meter.write_process(batch_nb, self.len_train_dataloader, self.current_epoch, write_batch_idx=50) + if return_value is not None: + iou, fb_iou = return_value + self.log("fewshot_train_iou", iou) + self.log("fewshot_trainl_fb_iou", fb_iou) + + return loss + + def training_epoch_end(self, outs): + if self.global_rank == 0: + self.train_average_meter.write_result('Training', self.current_epoch) + self.fewshot_trn_loss = utils.mean(self.train_average_meter.loss_buf) + self.fewshot_trn_miou, self.fewshot_trn_fb_iou = self.train_average_meter.compute_iou() + + self.log("fewshot_trn_loss", self.fewshot_trn_loss) + self.log("fewshot_trn_miou", self.fewshot_trn_miou) + self.log("fewshot_trn_fb_iou", self.fewshot_trn_fb_iou) + + def validation_step(self, batch, batch_nb): + if self.args.finetune_mode and self.args.nshot == 5: + bshape = batch['query_img'].shape + img = batch['query_img'].view(-1, bshape[2], bshape[3], bshape[4]) + target = batch['query_mask'].view(-1, bshape[3], bshape[4]) + class_info = batch['class_id'] + for i in range(1, 5): + class_info = torch.cat([class_info, batch['class_id']]) + out = self(img, class_info) + val_loss = self.criterion(out, target) + # 3. Evaluate prediction + if self.args.benchmark == 'pascal' and batch['query_ignore_idx'] is not None: + query_ignore_idx = batch['query_ignore_idx'].view(-1, bshape[3], bshape[4]) + area_inter, area_union = Evaluator.classify_prediction(out.argmax(dim=1), target, query_ignore_idx) + else: + area_inter, area_union = Evaluator.classify_prediction(out.argmax(dim=1), target) + else: + img = batch['query_img'].squeeze(1) + target = batch['query_mask'].squeeze(1) + class_info = batch['class_id'] + out = self(img, class_info) + val_loss = self.criterion(out, target) + # 3. Evaluate prediction + if self.args.benchmark == 'pascal' and batch['query_ignore_idx'] is not None: + query_ignore_idx = batch['query_ignore_idx'].squeeze(1) + area_inter, area_union = Evaluator.classify_prediction(out.argmax(dim=1), target, query_ignore_idx) + else: + area_inter, area_union = Evaluator.classify_prediction(out.argmax(dim=1), target) + + self.val_average_meter.update(area_inter, area_union, class_info, val_loss.detach().clone()) + if self.global_rank == 0: + return_value = self.val_average_meter.write_process(batch_nb, self.len_val_dataloader, self.current_epoch, write_batch_idx=50) + if return_value is not None: + iou, fb_iou = return_value + self.log("fewshot_val_iou", iou) + self.log("fewshot_val_fb_iou", fb_iou) + + + def validation_epoch_end(self, outs): + if self.global_rank == 0: + self.val_average_meter.write_result('Validation', self.current_epoch) + val_loss = utils.mean(self.val_average_meter.loss_buf) + val_miou, val_fb_iou = self.val_average_meter.compute_iou() + self.log("fewshot_val_loss", val_loss) + self.log("fewshot_val_miou", val_miou) + self.log("fewshot_val_fb_iou", val_fb_iou) + + if self.global_rank == 0: + Logger.tbd_writer.add_scalars('fewshot_data/data/loss', {'trn_loss': self.fewshot_trn_loss, 'val_loss': val_loss}, self.current_epoch) + Logger.tbd_writer.add_scalars('fewshot_data/data/miou', {'trn_miou': self.fewshot_trn_miou, 'val_miou': val_miou}, self.current_epoch) + Logger.tbd_writer.add_scalars('fewshot_data/data/fb_iou', {'trn_fb_iou': self.fewshot_trn_fb_iou, 'val_fb_iou': val_fb_iou}, self.current_epoch) + Logger.tbd_writer.flush() + if self.current_epoch + 1 == self.epochs: + Logger.tbd_writer.close() + Logger.info('==================== Finished Training ====================') + + threshold_epoch = 3 + if self.args.benchmark in ['pascal', 'coco'] and self.current_epoch >= threshold_epoch: + print('End this loop!') + exit() + + def configure_optimizers(self): + # if we want to fix the encoder + if self.fixed_encoder: + params_list = [ + {"params": self.net.pretrained.model.parameters(), "lr": 0}, + ] + params_list.append( + {"params": self.net.pretrained.act_postprocess1.parameters(), "lr": self.base_lr} + ) + params_list.append( + {"params": self.net.pretrained.act_postprocess2.parameters(), "lr": self.base_lr} + ) + params_list.append( + {"params": self.net.pretrained.act_postprocess3.parameters(), "lr": self.base_lr} + ) + params_list.append( + {"params": self.net.pretrained.act_postprocess4.parameters(), "lr": self.base_lr} + ) + else: + params_list = [ + {"params": self.net.pretrained.parameters(), "lr": self.base_lr}, + ] + + if hasattr(self.net, "scratch"): + print("Found output scratch") + params_list.append( + {"params": self.net.scratch.parameters(), "lr": self.base_lr * 10} + ) + if hasattr(self.net, "auxlayer"): + print("Found auxlayer") + params_list.append( + {"params": self.net.auxlayer.parameters(), "lr": self.base_lr * 10} + ) + if hasattr(self.net, "scale_inv_conv"): + print(self.net.scale_inv_conv) + print("Found scaleinv layers") + params_list.append( + { + "params": self.net.scale_inv_conv.parameters(), + "lr": self.base_lr * 10, + } + ) + params_list.append( + {"params": self.net.scale2_conv.parameters(), "lr": self.base_lr * 10} + ) + params_list.append( + {"params": self.net.scale3_conv.parameters(), "lr": self.base_lr * 10} + ) + params_list.append( + {"params": self.net.scale4_conv.parameters(), "lr": self.base_lr * 10} + ) + + if self.other_kwargs["midasproto"]: + print("Using midas optimization protocol") + + opt = torch.optim.Adam( + params_list, + lr=self.base_lr, + betas=(0.9, 0.999), + weight_decay=self.other_kwargs["weight_decay"], + ) + sch = torch.optim.lr_scheduler.LambdaLR( + opt, lambda x: pow(1.0 - x / self.epochs, 0.9) + ) + else: + opt = torch.optim.SGD( + params_list, + lr=self.base_lr, + momentum=0.9, + weight_decay=self.other_kwargs["weight_decay"], + ) + + sch = torch.optim.lr_scheduler.LambdaLR( + opt, lambda x: pow(1.0 - x / self.epochs, 0.9) + ) + return [opt], [sch] + + def train_dataloader(self): + if self.args.finetune_mode: + dataloader = FSSDataset.build_dataloader( + self.args.benchmark, + self.args.bsz, + self.args.nworker, + self.args.fold, + 'test', + self.args.nshot) + else: + dataloader = FSSDataset.build_dataloader( + self.args.benchmark, + self.args.bsz, + self.args.nworker, + self.args.fold, + 'trn') + + self.len_train_dataloader = len(dataloader) // torch.cuda.device_count() + self.train_average_meter = AverageMeter(dataloader.dataset) + return dataloader + + def val_dataloader(self): + self.val_iou = SegmentationMetric(self.num_classes) + if self.args.finetune_mode: + dataloader = FSSDataset.build_dataloader( + self.args.benchmark, + self.args.bsz, + self.args.nworker, + self.args.fold, + 'test', + self.args.nshot) + else: + dataloader = FSSDataset.build_dataloader( + self.args.benchmark, + self.args.bsz, + self.args.nworker, + self.args.fold, + 'val') + self.len_val_dataloader = len(dataloader) // torch.cuda.device_count() + self.val_average_meter = AverageMeter(dataloader.dataset) + return dataloader + + + def criterion(self, logit_mask, gt_mask): + bsz = logit_mask.size(0) + logit_mask = logit_mask.view(bsz, 2, -1) + gt_mask = gt_mask.view(bsz, -1).long() + + return self.cross_entropy_loss(logit_mask, gt_mask) + + + @staticmethod + def add_model_specific_args(parent_parser): + parser = ArgumentParser(parents=[parent_parser], add_help=False) + parser.add_argument( + "--data_path", + type=str, + default='', + help="path where dataset is stored" + ) + parser.add_argument( + "--dataset", + type=str, + default='pascal', + choices=['pascal', 'coco', 'fss'], + ) + parser.add_argument( + "--batch_size", type=int, default=20, help="size of the batches" + ) + parser.add_argument( + "--base_lr", type=float, default=0.004, help="learning rate" + ) + parser.add_argument("--momentum", type=float, default=0.9, help="SGD momentum") + parser.add_argument( + "--weight_decay", type=float, default=1e-4, help="weight_decay" + ) + parser.add_argument( + "--aux", action="store_true", default=False, help="Auxilary Loss" + ) + parser.add_argument( + "--aux-weight", + type=float, + default=0.2, + help="Auxilary loss weight (default: 0.2)", + ) + parser.add_argument( + "--se-loss", + action="store_true", + default=False, + help="Semantic Encoding Loss SE-loss", + ) + parser.add_argument( + "--se-weight", type=float, default=0.2, help="SE-loss weight (default: 0.2)" + ) + + parser.add_argument( + "--midasproto", action="store_true", default=False, help="midasprotocol" + ) + + parser.add_argument( + "--ignore_index", + type=int, + default=-1, + help="numeric value of ignore label in gt", + ) + parser.add_argument( + "--augment", + action="store_true", + default=False, + help="Use extended augmentations", + ) + parser.add_argument( + "--use_relabeled", + action="store_true", + default=False, + help="Use extended augmentations", + ) + + parser.add_argument( + "--nworker", + type=int, + default=8 + ) + + parser.add_argument( + "--fold", + type=int, + default=0, + choices=[0, 1, 2, 3] + ) + + parser.add_argument( + "--logpath", + type=str, + default='' + ) + + parser.add_argument( + "--nshot", + type=int, + default=0 #1 + ) + parser.add_argument( + "--finetune_mode", + action="store_true", + default=False, + help="whether finetune or not" + ) + + + return parser diff --git a/submodules/lang_seg/modules/models/lseg_blocks.py b/submodules/lang_seg/modules/models/lseg_blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..b1565868a42aa52eab226672cba375196a69658a --- /dev/null +++ b/submodules/lang_seg/modules/models/lseg_blocks.py @@ -0,0 +1,359 @@ +import torch +import torch.nn as nn + +from .lseg_vit import ( + _make_pretrained_clip_vitl16_384, + _make_pretrained_clip_vitb32_384, + _make_pretrained_clipRN50x16_vitl16_384, + forward_vit, +) + + +def _make_encoder( + backbone, + features, + use_pretrained=True, + groups=1, + expand=False, + exportable=True, + hooks=None, + use_vit_only=False, + use_readout="ignore", + enable_attention_hooks=False, +): + if backbone == "clip_vitl16_384": + clip_pretrained, pretrained = _make_pretrained_clip_vitl16_384( + use_pretrained, + hooks=hooks, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + scratch = _make_scratch( + [256, 512, 1024, 1024], features, groups=groups, expand=expand + ) + elif backbone == "clipRN50x16_vitl16_384": + clip_pretrained, pretrained = _make_pretrained_clipRN50x16_vitl16_384( + use_pretrained, + hooks=hooks, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + scratch = _make_scratch( + [256, 512, 1024, 1024], features, groups=groups, expand=expand + ) + elif backbone == "clip_vitb32_384": + clip_pretrained, pretrained = _make_pretrained_clip_vitb32_384( + use_pretrained, + hooks=hooks, + use_readout=use_readout, + ) + scratch = _make_scratch( + [96, 192, 384, 768], features, groups=groups, expand=expand + ) + else: + print(f"Backbone '{backbone}' not implemented") + assert False + + return clip_pretrained, pretrained, scratch + + +def _make_scratch(in_shape, out_shape, groups=1, expand=False): + scratch = nn.Module() + + out_shape1 = out_shape + out_shape2 = out_shape + out_shape3 = out_shape + out_shape4 = out_shape + if expand == True: + out_shape1 = out_shape + out_shape2 = out_shape * 2 + out_shape3 = out_shape * 4 + out_shape4 = out_shape * 8 + + scratch.layer1_rn = nn.Conv2d( + in_shape[0], + out_shape1, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer2_rn = nn.Conv2d( + in_shape[1], + out_shape2, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer3_rn = nn.Conv2d( + in_shape[2], + out_shape3, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer4_rn = nn.Conv2d( + in_shape[3], + out_shape4, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + + return scratch + + +class Interpolate(nn.Module): + """Interpolation module.""" + + def __init__(self, scale_factor, mode, align_corners=False): + """Init. + + Args: + scale_factor (float): scaling + mode (str): interpolation mode + """ + super(Interpolate, self).__init__() + + self.interp = nn.functional.interpolate + self.scale_factor = scale_factor + self.mode = mode + self.align_corners = align_corners + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: interpolated data + """ + + x = self.interp( + x, + scale_factor=self.scale_factor, + mode=self.mode, + align_corners=self.align_corners, + ) + + return x + + +class ResidualConvUnit(nn.Module): + """Residual convolution module.""" + + def __init__(self, features): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.conv1 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True + ) + + self.conv2 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True + ) + + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + out = self.relu(x) + out = self.conv1(out) + out = self.relu(out) + out = self.conv2(out) + + return out + x + + +class FeatureFusionBlock(nn.Module): + """Feature fusion block.""" + + def __init__(self, features): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock, self).__init__() + + self.resConfUnit1 = ResidualConvUnit(features) + self.resConfUnit2 = ResidualConvUnit(features) + + def forward(self, *xs): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + output += self.resConfUnit1(xs[1]) + + output = self.resConfUnit2(output) + + output = nn.functional.interpolate( + output, scale_factor=2, mode="bilinear", align_corners=True + ) + + return output + + +class ResidualConvUnit_custom(nn.Module): + """Residual convolution module.""" + + def __init__(self, features, activation, bn): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.bn = bn + + self.groups = 1 + + self.conv1 = nn.Conv2d( + features, + features, + kernel_size=3, + stride=1, + padding=1, + bias=not self.bn, + groups=self.groups, + ) + + self.conv2 = nn.Conv2d( + features, + features, + kernel_size=3, + stride=1, + padding=1, + bias=not self.bn, + groups=self.groups, + ) + + if self.bn == True: + self.bn1 = nn.BatchNorm2d(features) + self.bn2 = nn.BatchNorm2d(features) + + self.activation = activation + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + + out = self.activation(x) + out = self.conv1(out) + if self.bn == True: + out = self.bn1(out) + + out = self.activation(out) + out = self.conv2(out) + if self.bn == True: + out = self.bn2(out) + + if self.groups > 1: + out = self.conv_merge(out) + + return self.skip_add.add(out, x) + + # return out + x + + +class FeatureFusionBlock_custom(nn.Module): + """Feature fusion block.""" + + def __init__( + self, + features, + activation, + deconv=False, + bn=False, + expand=False, + align_corners=True, + ): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock_custom, self).__init__() + + self.deconv = deconv + self.align_corners = align_corners + + self.groups = 1 + + self.expand = expand + out_features = features + if self.expand == True: + out_features = features // 2 + + self.out_conv = nn.Conv2d( + features, + out_features, + kernel_size=1, + stride=1, + padding=0, + bias=True, + groups=1, + ) + + self.resConfUnit1 = ResidualConvUnit_custom(features, activation, bn) + self.resConfUnit2 = ResidualConvUnit_custom(features, activation, bn) + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, *xs): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + res = self.resConfUnit1(xs[1]) + output = self.skip_add.add(output, res) + # output += res + + output = self.resConfUnit2(output) + + output = nn.functional.interpolate( + output, scale_factor=2, mode="bilinear", align_corners=self.align_corners + ) + + output = self.out_conv(output) + + return output + diff --git a/submodules/lang_seg/modules/models/lseg_blocks_zs.py b/submodules/lang_seg/modules/models/lseg_blocks_zs.py new file mode 100644 index 0000000000000000000000000000000000000000..fa6b55ab017116ea775c8d2ebb3ffe0201082555 --- /dev/null +++ b/submodules/lang_seg/modules/models/lseg_blocks_zs.py @@ -0,0 +1,373 @@ +import torch +import torch.nn as nn + +from .lseg_vit_zs import ( + _make_pretrained_clip_vitl16_384, + _make_pretrained_clip_vitb32_384, + _make_pretrained_clip_rn101, + forward_vit, +) + +def _make_encoder( + backbone, + features, + use_pretrained, + groups=1, + expand=False, + exportable=True, + hooks=None, + use_vit_only=False, + use_readout="ignore", + enable_attention_hooks=False, +): + if backbone == "clip_vitl16_384": + clip_pretrained, pretrained = _make_pretrained_clip_vitl16_384( + use_pretrained, + hooks=hooks, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + scratch = _make_scratch( + [256, 512, 1024, 1024], features, groups=groups, expand=expand + ) + elif backbone == "clip_vitb32_384": + clip_pretrained, pretrained = _make_pretrained_clip_vitb32_384( + use_pretrained, + hooks=hooks, + use_readout=use_readout, + ) + scratch = _make_scratch( + [96, 192, 384, 768], features, groups=groups, expand=expand + ) + elif backbone == "clip_resnet101": + clip_pretrained, pretrained = _make_pretrained_clip_rn101( + use_pretrained, + ) + scratch = _make_scratch( + [256, 512, 1024, 2048], features, groups=groups, expand=expand + ) + else: + print(f"Backbone '{backbone}' not implemented") + assert False + + return clip_pretrained, pretrained, scratch + + +def _make_scratch(in_shape, out_shape, groups=1, expand=False): + scratch = nn.Module() + + out_shape1 = out_shape + out_shape2 = out_shape + out_shape3 = out_shape + out_shape4 = out_shape + if expand == True: + out_shape1 = out_shape + out_shape2 = out_shape * 2 + out_shape3 = out_shape * 4 + out_shape4 = out_shape * 8 + + scratch.layer1_rn = nn.Conv2d( + in_shape[0], + out_shape1, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer2_rn = nn.Conv2d( + in_shape[1], + out_shape2, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer3_rn = nn.Conv2d( + in_shape[2], + out_shape3, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer4_rn = nn.Conv2d( + in_shape[3], + out_shape4, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + + return scratch + + +def _make_resnet_backbone(resnet): + pretrained = nn.Module() + pretrained.layer1 = nn.Sequential( + resnet.conv1, resnet.bn1, resnet.relu, resnet.maxpool, resnet.layer1 + ) + + pretrained.layer2 = resnet.layer2 + pretrained.layer3 = resnet.layer3 + pretrained.layer4 = resnet.layer4 + + return pretrained + + +def _make_pretrained_resnext101_wsl(use_pretrained): + resnet = torch.hub.load("facebookresearch/WSL-Images", "resnext101_32x8d_wsl") + return _make_resnet_backbone(resnet) + + +class Interpolate(nn.Module): + """Interpolation module.""" + + def __init__(self, scale_factor, mode, align_corners=False): + """Init. + + Args: + scale_factor (float): scaling + mode (str): interpolation mode + """ + super(Interpolate, self).__init__() + + self.interp = nn.functional.interpolate + self.scale_factor = scale_factor + self.mode = mode + self.align_corners = align_corners + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: interpolated data + """ + + x = self.interp( + x, + scale_factor=self.scale_factor, + mode=self.mode, + align_corners=self.align_corners, + ) + + return x + + +class ResidualConvUnit(nn.Module): + """Residual convolution module.""" + + def __init__(self, features): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.conv1 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True + ) + + self.conv2 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True + ) + + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + out = self.relu(x) + out = self.conv1(out) + out = self.relu(out) + out = self.conv2(out) + + return out + x + + +class FeatureFusionBlock(nn.Module): + """Feature fusion block.""" + + def __init__(self, features): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock, self).__init__() + + self.resConfUnit1 = ResidualConvUnit(features) + self.resConfUnit2 = ResidualConvUnit(features) + + def forward(self, *xs): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + output += self.resConfUnit1(xs[1]) + + output = self.resConfUnit2(output) + + output = nn.functional.interpolate( + output, scale_factor=2, mode="bilinear", align_corners=True + ) + + return output + + +class ResidualConvUnit_custom(nn.Module): + """Residual convolution module.""" + + def __init__(self, features, activation, bn): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.bn = bn + + self.groups = 1 + + self.conv1 = nn.Conv2d( + features, + features, + kernel_size=3, + stride=1, + padding=1, + bias=not self.bn, + groups=self.groups, + ) + + self.conv2 = nn.Conv2d( + features, + features, + kernel_size=3, + stride=1, + padding=1, + bias=not self.bn, + groups=self.groups, + ) + + if self.bn == True: + self.bn1 = nn.BatchNorm2d(features) + self.bn2 = nn.BatchNorm2d(features) + + self.activation = activation + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + + out = self.activation(x) + out = self.conv1(out) + if self.bn == True: + out = self.bn1(out) + + out = self.activation(out) + out = self.conv2(out) + if self.bn == True: + out = self.bn2(out) + + if self.groups > 1: + out = self.conv_merge(out) + + return self.skip_add.add(out, x) + + # return out + x + + +class FeatureFusionBlock_custom(nn.Module): + """Feature fusion block.""" + + def __init__( + self, + features, + activation, + deconv=False, + bn=False, + expand=False, + align_corners=True, + ): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock_custom, self).__init__() + + self.deconv = deconv + self.align_corners = align_corners + + self.groups = 1 + + self.expand = expand + out_features = features + if self.expand == True: + out_features = features // 2 + + self.out_conv = nn.Conv2d( + features, + out_features, + kernel_size=1, + stride=1, + padding=0, + bias=True, + groups=1, + ) + + self.resConfUnit1 = ResidualConvUnit_custom(features, activation, bn) + self.resConfUnit2 = ResidualConvUnit_custom(features, activation, bn) + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, *xs): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + res = self.resConfUnit1(xs[1]) + output = self.skip_add.add(output, res) + # output += res + + output = self.resConfUnit2(output) + + output = nn.functional.interpolate( + output, scale_factor=2, mode="bilinear", align_corners=self.align_corners + ) + + output = self.out_conv(output) + + return output + diff --git a/submodules/lang_seg/modules/models/lseg_net.py b/submodules/lang_seg/modules/models/lseg_net.py new file mode 100644 index 0000000000000000000000000000000000000000..7290b67d3cbca3707d8fdb690cd3290fbdedf6f3 --- /dev/null +++ b/submodules/lang_seg/modules/models/lseg_net.py @@ -0,0 +1,231 @@ +import math +import types + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .lseg_blocks import FeatureFusionBlock, Interpolate, _make_encoder, FeatureFusionBlock_custom, forward_vit +import clip +import numpy as np +import pandas as pd +import os + +class depthwise_clipseg_conv(nn.Module): + def __init__(self): + super(depthwise_clipseg_conv, self).__init__() + self.depthwise = nn.Conv2d(1, 1, kernel_size=3, padding=1) + + def depthwise_clipseg(self, x, channels): + x = torch.cat([self.depthwise(x[:, i].unsqueeze(1)) for i in range(channels)], dim=1) + return x + + def forward(self, x): + channels = x.shape[1] + out = self.depthwise_clipseg(x, channels) + return out + + +class depthwise_conv(nn.Module): + def __init__(self, kernel_size=3, stride=1, padding=1): + super(depthwise_conv, self).__init__() + self.depthwise = nn.Conv2d(1, 1, kernel_size=kernel_size, stride=stride, padding=padding) + + def forward(self, x): + # support for 4D tensor with NCHW + C, H, W = x.shape[1:] + x = x.reshape(-1, 1, H, W) + x = self.depthwise(x) + x = x.view(-1, C, H, W) + return x + + +class depthwise_block(nn.Module): + def __init__(self, kernel_size=3, stride=1, padding=1, activation='relu'): + super(depthwise_block, self).__init__() + self.depthwise = depthwise_conv(kernel_size=3, stride=1, padding=1) + if activation == 'relu': + self.activation = nn.ReLU() + elif activation == 'lrelu': + self.activation = nn.LeakyReLU() + elif activation == 'tanh': + self.activation = nn.Tanh() + + def forward(self, x, act=True): + x = self.depthwise(x) + if act: + x = self.activation(x) + return x + + +class bottleneck_block(nn.Module): + def __init__(self, kernel_size=3, stride=1, padding=1, activation='relu'): + super(bottleneck_block, self).__init__() + self.depthwise = depthwise_conv(kernel_size=3, stride=1, padding=1) + if activation == 'relu': + self.activation = nn.ReLU() + elif activation == 'lrelu': + self.activation = nn.LeakyReLU() + elif activation == 'tanh': + self.activation = nn.Tanh() + + + def forward(self, x, act=True): + sum_layer = x.max(dim=1, keepdim=True)[0] + x = self.depthwise(x) + x = x + sum_layer + if act: + x = self.activation(x) + return x + +class BaseModel(torch.nn.Module): + def load(self, path): + """Load model from file. + Args: + path (str): file path + """ + parameters = torch.load(path, map_location=torch.device("cpu")) + + if "optimizer" in parameters: + parameters = parameters["model"] + + self.load_state_dict(parameters) + +def _make_fusion_block(features, use_bn): + return FeatureFusionBlock_custom( + features, + activation=nn.ReLU(False), + deconv=False, + bn=use_bn, + expand=False, + align_corners=True, + ) + +class LSeg(BaseModel): + def __init__( + self, + head, + features=256, + backbone="clip_vitl16_384", + readout="project", + channels_last=False, + use_bn=False, + **kwargs, + ): + super(LSeg, self).__init__() + + self.channels_last = channels_last + + hooks = { + "clip_vitl16_384": [5, 11, 17, 23], + "clipRN50x16_vitl16_384": [5, 11, 17, 23], + "clip_vitb32_384": [2, 5, 8, 11], + } + + # Instantiate backbone and reassemble blocks + self.clip_pretrained, self.pretrained, self.scratch = _make_encoder( + backbone, + features, + groups=1, + expand=False, + exportable=False, + hooks=hooks[backbone], + use_readout=readout, + ) + + self.scratch.refinenet1 = _make_fusion_block(features, use_bn) + self.scratch.refinenet2 = _make_fusion_block(features, use_bn) + self.scratch.refinenet3 = _make_fusion_block(features, use_bn) + self.scratch.refinenet4 = _make_fusion_block(features, use_bn) + + self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)).exp() + if backbone in ["clipRN50x16_vitl16_384"]: + self.out_c = 768 + else: + self.out_c = 512 + self.scratch.head1 = nn.Conv2d(features, self.out_c, kernel_size=1) + + self.arch_option = kwargs["arch_option"] + if self.arch_option == 1: + self.scratch.head_block = bottleneck_block(activation=kwargs["activation"]) + self.block_depth = kwargs['block_depth'] + elif self.arch_option == 2: + self.scratch.head_block = depthwise_block(activation=kwargs["activation"]) + self.block_depth = kwargs['block_depth'] + + self.scratch.output_conv = head + + self.text = clip.tokenize(self.labels) + + def forward(self, x, labelset=''): + if labelset == '': + text = self.text + else: + text = clip.tokenize(labelset) + + if self.channels_last == True: + x.contiguous(memory_format=torch.channels_last) + + layer_1, layer_2, layer_3, layer_4 = forward_vit(self.pretrained, x) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + text = text.to(x.device) + self.logit_scale = self.logit_scale.to(x.device) + text_features = self.clip_pretrained.encode_text(text) + + image_features = self.scratch.head1(path_1) + + imshape = image_features.shape + image_features = image_features.permute(0,2,3,1).reshape(-1, self.out_c) + + # normalized features + image_features = image_features / image_features.norm(dim=-1, keepdim=True) + text_features = text_features / text_features.norm(dim=-1, keepdim=True) + + logits_per_image = self.logit_scale * image_features.half() @ text_features.t() + + out = logits_per_image.float().view(imshape[0], imshape[2], imshape[3], -1).permute(0,3,1,2) + + if self.arch_option in [1, 2]: + for _ in range(self.block_depth - 1): + out = self.scratch.head_block(out) + out = self.scratch.head_block(out, False) + + out = self.scratch.output_conv(out) + + return out + + +class LSegNet(LSeg): + """Network for semantic segmentation.""" + def __init__(self, labels, path=None, scale_factor=0.5, crop_size=480, **kwargs): + + features = kwargs["features"] if "features" in kwargs else 256 + kwargs["use_bn"] = True + + self.crop_size = crop_size + self.scale_factor = scale_factor + self.labels = labels + + head = nn.Sequential( + Interpolate(scale_factor=2, mode="bilinear", align_corners=True), + ) + + super().__init__(head, **kwargs) + + if path is not None: + self.load(path) + + + + + \ No newline at end of file diff --git a/submodules/lang_seg/modules/models/lseg_net_zs.py b/submodules/lang_seg/modules/models/lseg_net_zs.py new file mode 100644 index 0000000000000000000000000000000000000000..a59e93849e2d992cb44fabbd64f96f7515d23b72 --- /dev/null +++ b/submodules/lang_seg/modules/models/lseg_net_zs.py @@ -0,0 +1,363 @@ +import math +import types + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .lseg_blocks_zs import FeatureFusionBlock, Interpolate, _make_encoder, FeatureFusionBlock_custom, forward_vit +import clip +import numpy as np +import pandas as pd + +import os + +class depthwise_clipseg_conv(nn.Module): + def __init__(self): + super(depthwise_clipseg_conv, self).__init__() + self.depthwise = nn.Conv2d(1, 1, kernel_size=3, padding=1) + + def depthwise_clipseg(self, x, channels): + x = torch.cat([self.depthwise(x[:, i].unsqueeze(1)) for i in range(channels)], dim=1) + return x + + def forward(self, x): + channels = x.shape[1] + out = self.depthwise_clipseg(x, channels) + return out + +class depthwise_conv(nn.Module): + def __init__(self, kernel_size=3, stride=1, padding=1): + super(depthwise_conv, self).__init__() + self.depthwise = nn.Conv2d(1, 1, kernel_size=kernel_size, stride=stride, padding=padding) + + def forward(self, x): + # support for 4D tensor with NCHW + C, H, W = x.shape[1:] + x = x.reshape(-1, 1, H, W) + x = self.depthwise(x) + x = x.view(-1, C, H, W) + return x + +# tanh relu +class depthwise_block(nn.Module): + def __init__(self, kernel_size=3, stride=1, padding=1, activation='relu'): + super(depthwise_block, self).__init__() + self.depthwise = depthwise_conv(kernel_size=3, stride=1, padding=1) + if activation == 'relu': + self.activation = nn.ReLU() + elif activation == 'lrelu': + self.activation = nn.LeakyReLU() + elif activation == 'tanh': + self.activation = nn.Tanh() + + def forward(self, x, act=True): + x = self.depthwise(x) + if act: + x = self.activation(x) + return x + + +class bottleneck_block(nn.Module): + def __init__(self, kernel_size=3, stride=1, padding=1, activation='relu'): + super(bottleneck_block, self).__init__() + self.depthwise = depthwise_conv(kernel_size=3, stride=1, padding=1) + if activation == 'relu': + self.activation = nn.ReLU() + elif activation == 'lrelu': + self.activation = nn.LeakyReLU() + elif activation == 'tanh': + self.activation = nn.Tanh() + + + def forward(self, x, act=True): + sum_layer = x.max(dim=1, keepdim=True)[0] + x = self.depthwise(x) + x = x + sum_layer + if act: + x = self.activation(x) + return x + +class BaseModel(torch.nn.Module): + def load(self, path): + """Load model from file. + Args: + path (str): file path + """ + parameters = torch.load(path, map_location=torch.device("cpu")) + + if "optimizer" in parameters: + parameters = parameters["model"] + + self.load_state_dict(parameters) + + +def _make_fusion_block(features, use_bn): + return FeatureFusionBlock_custom( + features, + activation=nn.ReLU(False), + deconv=False, + bn=use_bn, + expand=False, + align_corners=True, + ) + + +class LSeg(BaseModel): + def __init__( + self, + head, + features=256, + backbone="vitb_rn50_384", + readout="project", + channels_last=False, + use_bn=False, + **kwargs, + ): + super(LSeg, self).__init__() + + self.channels_last = channels_last + + hooks = { + "clip_vitl16_384": [5, 11, 17, 23], + "clipRN50x16_vitl16_384": [5, 11, 17, 23], + "clipRN50x4_vitl16_384": [5, 11, 17, 23], + "clip_vitb32_384": [2, 5, 8, 11], + "clipRN50x16_vitb32_384": [2, 5, 8, 11], + "clipRN50x4_vitb32_384": [2, 5, 8, 11], + "clip_resnet101": [0, 1, 8, 11], + } + + # Instantiate backbone and reassemble blocks + self.clip_pretrained, self.pretrained, self.scratch = _make_encoder( + backbone, + features, + self.use_pretrained, # Set to true of you want to train from scratch, uses ImageNet weights + groups=1, + expand=False, + exportable=False, + hooks=hooks[backbone], + use_readout=readout, + ) + + self.scratch.refinenet1 = _make_fusion_block(features, use_bn) + self.scratch.refinenet2 = _make_fusion_block(features, use_bn) + self.scratch.refinenet3 = _make_fusion_block(features, use_bn) + self.scratch.refinenet4 = _make_fusion_block(features, use_bn) + + # self.scratch.output_conv = head + + self.auxlayer = nn.Sequential( + Interpolate(scale_factor=2, mode="bilinear", align_corners=True), + ) + + # cosine similarity as logits + self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)).exp() + + if backbone in ["clipRN50x16_vitl16_384", "clipRN50x16_vitb32_384"]: + self.out_c = 768 + elif backbone in ["clipRN50x4_vitl16_384", "clipRN50x4_vitb32_384"]: + self.out_c = 640 + else: + self.out_c = 512 + self.scratch.head1 = nn.Conv2d(features, self.out_c, kernel_size=1) + + self.arch_option = kwargs["arch_option"] + + self.scratch.output_conv = head + + self.texts = [] + # original + label = ['others', ''] + for class_i in range(len(self.label_list)): + label[1] = self.label_list[class_i] + text = clip.tokenize(label) + self.texts.append(text) + + def forward(self, x, class_info): + texts = [self.texts[class_i] for class_i in class_info] + + if self.channels_last == True: + x.contiguous(memory_format=torch.channels_last) + + layer_1, layer_2, layer_3, layer_4 = forward_vit(self.pretrained, x) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + self.logit_scale = self.logit_scale.to(x.device) + text_features = [self.clip_pretrained.encode_text(text.to(x.device)) for text in texts] + + image_features = self.scratch.head1(path_1) + + + imshape = image_features.shape + image_features = [image_features[i].unsqueeze(0).permute(0,2,3,1).reshape(-1, self.out_c) for i in range(len(image_features))] + + # normalized features + image_features = [image_feature / image_feature.norm(dim=-1, keepdim=True) for image_feature in image_features] + text_features = [text_feature / text_feature.norm(dim=-1, keepdim=True) for text_feature in text_features] + + logits_per_images = [self.logit_scale * image_feature.half() @ text_feature.t() for image_feature, text_feature in zip(image_features, text_features)] + outs = [logits_per_image.float().view(1, imshape[2], imshape[3], -1).permute(0,3,1,2) for logits_per_image in logits_per_images] + out = torch.cat([out for out in outs], dim=0) + + out = self.scratch.output_conv(out) + + return out + + +class LSegNetZS(LSeg): + """Network for semantic segmentation.""" + def __init__(self, label_list, path=None, scale_factor=0.5, aux=False, use_relabeled=False, use_pretrained=True, **kwargs): + + features = kwargs["features"] if "features" in kwargs else 256 + kwargs["use_bn"] = True + + self.scale_factor = scale_factor + self.aux = aux + self.use_relabeled = use_relabeled + self.label_list = label_list + self.use_pretrained = use_pretrained + + head = nn.Sequential( + Interpolate(scale_factor=2, mode="bilinear", align_corners=True), + ) + + super().__init__(head, **kwargs) + + if path is not None: + self.load(path) + + +class LSegRN(BaseModel): + def __init__( + self, + head, + features=256, + backbone="clip_resnet101", + readout="project", + channels_last=False, + use_bn=False, + **kwargs, + ): + super(LSegRN, self).__init__() + + self.channels_last = channels_last + + # Instantiate backbone and reassemble blocks + self.clip_pretrained, self.pretrained, self.scratch = _make_encoder( + backbone, + features, + self.use_pretrained, # Set to true of you want to train from scratch, uses ImageNet weights + groups=1, + expand=False, + exportable=False, + use_readout=readout, + ) + + self.scratch.refinenet1 = _make_fusion_block(features, use_bn) + self.scratch.refinenet2 = _make_fusion_block(features, use_bn) + self.scratch.refinenet3 = _make_fusion_block(features, use_bn) + self.scratch.refinenet4 = _make_fusion_block(features, use_bn) + + # self.scratch.output_conv = head + + self.auxlayer = nn.Sequential( + Interpolate(scale_factor=2, mode="bilinear", align_corners=True), + ) + + # cosine similarity as logits + self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)).exp() + + if backbone in ["clipRN50x16_vitl16_384", "clipRN50x16_vitb32_384"]: + self.out_c = 768 + elif backbone in ["clipRN50x4_vitl16_384", "clipRN50x4_vitb32_384"]: + self.out_c = 640 + else: + self.out_c = 512 + self.scratch.head1 = nn.Conv2d(features, self.out_c, kernel_size=1) + + self.arch_option = kwargs["arch_option"] + + self.scratch.output_conv = head + + self.texts = [] + # original + label = ['others', ''] + for class_i in range(len(self.label_list)): + label[1] = self.label_list[class_i] + text = clip.tokenize(label) + self.texts.append(text) + + def forward(self, x, class_info): + texts = [self.texts[class_i] for class_i in class_info] + + if self.channels_last == True: + x.contiguous(memory_format=torch.channels_last) + + layer_1 = self.pretrained.layer1(x) + layer_2 = self.pretrained.layer2(layer_1) + layer_3 = self.pretrained.layer3(layer_2) + layer_4 = self.pretrained.layer4(layer_3) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + self.logit_scale = self.logit_scale.to(x.device) + text_features = [self.clip_pretrained.encode_text(text.to(x.device)) for text in texts] + + image_features = self.scratch.head1(path_1) + + imshape = image_features.shape + image_features = [image_features[i].unsqueeze(0).permute(0,2,3,1).reshape(-1, self.out_c) for i in range(len(image_features))] + + # normalized features + image_features = [image_feature / image_feature.norm(dim=-1, keepdim=True) for image_feature in image_features] + text_features = [text_feature / text_feature.norm(dim=-1, keepdim=True) for text_feature in text_features] + + logits_per_images = [self.logit_scale * image_feature.half() @ text_feature.t() for image_feature, text_feature in zip(image_features, text_features)] + outs = [logits_per_image.float().view(1, imshape[2], imshape[3], -1).permute(0,3,1,2) for logits_per_image in logits_per_images] + out = torch.cat([out for out in outs], dim=0) + + out = self.scratch.output_conv(out) + + return out + + +class LSegRNNetZS(LSegRN): + """Network for semantic segmentation.""" + def __init__(self, label_list, path=None, scale_factor=0.5, aux=False, use_relabeled=False, use_pretrained=True, **kwargs): + + features = kwargs["features"] if "features" in kwargs else 256 + kwargs["use_bn"] = True + + self.scale_factor = scale_factor + self.aux = aux + self.use_relabeled = use_relabeled + self.label_list = label_list + self.use_pretrained = use_pretrained + + head = nn.Sequential( + Interpolate(scale_factor=2, mode="bilinear", align_corners=True), + ) + + super().__init__(head, **kwargs) + + if path is not None: + self.load(path) + diff --git a/submodules/lang_seg/modules/models/lseg_vit.py b/submodules/lang_seg/modules/models/lseg_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..dabbfccc89e650136ec486820b0c2186ec940979 --- /dev/null +++ b/submodules/lang_seg/modules/models/lseg_vit.py @@ -0,0 +1,535 @@ +import torch +import torch.nn as nn +import timm +import types +import math +import torch.nn.functional as F +import clip + +activations = {} + + +def get_activation(name): + def hook(model, input, output): + activations[name] = output + + return hook + + +attention = {} + + +def get_attention(name): + def hook(module, input, output): + x = input[0] + B, N, C = x.shape + qkv = ( + module.qkv(x) + .reshape(B, N, 3, module.num_heads, C // module.num_heads) + .permute(2, 0, 3, 1, 4) + ) + q, k, v = ( + qkv[0], + qkv[1], + qkv[2], + ) # make torchscript happy (cannot use tensor as tuple) + + attn = (q @ k.transpose(-2, -1)) * module.scale + + attn = attn.softmax(dim=-1) # [:,:,1,1:] + attention[name] = attn + + return hook + + +def get_mean_attention_map(attn, token, shape): + attn = attn[:, :, token, 1:] + attn = attn.unflatten(2, torch.Size([shape[2] // 16, shape[3] // 16])).float() + attn = torch.nn.functional.interpolate( + attn, size=shape[2:], mode="bicubic", align_corners=False + ).squeeze(0) + + all_attn = torch.mean(attn, 0) + + return all_attn + + +class Slice(nn.Module): + def __init__(self, start_index=1): + super(Slice, self).__init__() + self.start_index = start_index + + def forward(self, x): + return x[:, self.start_index :] + + +class AddReadout(nn.Module): + def __init__(self, start_index=1): + super(AddReadout, self).__init__() + self.start_index = start_index + + def forward(self, x): + if self.start_index == 2: + readout = (x[:, 0] + x[:, 1]) / 2 + else: + readout = x[:, 0] + return x[:, self.start_index :] + readout.unsqueeze(1) + + +class ProjectReadout(nn.Module): + def __init__(self, in_features, start_index=1): + super(ProjectReadout, self).__init__() + self.start_index = start_index + + self.project = nn.Sequential(nn.Linear(2 * in_features, in_features), nn.GELU()) + + def forward(self, x): + readout = x[:, 0].unsqueeze(1).expand_as(x[:, self.start_index :]) + features = torch.cat((x[:, self.start_index :], readout), -1) + + return self.project(features) + + +class Transpose(nn.Module): + def __init__(self, dim0, dim1): + super(Transpose, self).__init__() + self.dim0 = dim0 + self.dim1 = dim1 + + def forward(self, x): + x = x.transpose(self.dim0, self.dim1) + return x + + +def forward_vit(pretrained, x): + b, c, h, w = x.shape + + # encoder + glob = pretrained.model.forward_flex(x) + + layer_1 = pretrained.activations["1"] + layer_2 = pretrained.activations["2"] + layer_3 = pretrained.activations["3"] + layer_4 = pretrained.activations["4"] + + layer_1 = pretrained.act_postprocess1[0:2](layer_1) + layer_2 = pretrained.act_postprocess2[0:2](layer_2) + layer_3 = pretrained.act_postprocess3[0:2](layer_3) + layer_4 = pretrained.act_postprocess4[0:2](layer_4) + + unflatten = nn.Sequential( + nn.Unflatten( + 2, + torch.Size( + [ + h // pretrained.model.patch_size[1], + w // pretrained.model.patch_size[0], + ] + ), + ) + ) + + if layer_1.ndim == 3: + layer_1 = unflatten(layer_1) + if layer_2.ndim == 3: + layer_2 = unflatten(layer_2) + if layer_3.ndim == 3: + layer_3 = unflatten(layer_3) + if layer_4.ndim == 3: + layer_4 = unflatten(layer_4) + + layer_1 = pretrained.act_postprocess1[3 : len(pretrained.act_postprocess1)](layer_1) + layer_2 = pretrained.act_postprocess2[3 : len(pretrained.act_postprocess2)](layer_2) + layer_3 = pretrained.act_postprocess3[3 : len(pretrained.act_postprocess3)](layer_3) + layer_4 = pretrained.act_postprocess4[3 : len(pretrained.act_postprocess4)](layer_4) + + return layer_1, layer_2, layer_3, layer_4 + + +def _resize_pos_embed(self, posemb, gs_h, gs_w): + posemb_tok, posemb_grid = ( + posemb[:, : self.start_index], + posemb[0, self.start_index :], + ) + + gs_old = int(math.sqrt(len(posemb_grid))) + + posemb_grid = posemb_grid.reshape(1, gs_old, gs_old, -1).permute(0, 3, 1, 2) + posemb_grid = F.interpolate(posemb_grid, size=(gs_h, gs_w), mode="bilinear") + posemb_grid = posemb_grid.permute(0, 2, 3, 1).reshape(1, gs_h * gs_w, -1) + + posemb = torch.cat([posemb_tok, posemb_grid], dim=1) + + return posemb + + +def forward_flex(self, x): + b, c, h, w = x.shape + + pos_embed = self._resize_pos_embed( + self.pos_embed, h // self.patch_size[1], w // self.patch_size[0] + ) + + B = x.shape[0] + + if hasattr(self.patch_embed, "backbone"): + x = self.patch_embed.backbone(x) + if isinstance(x, (list, tuple)): + x = x[-1] # last feature if backbone outputs list/tuple of features + x = self.patch_embed.proj(x).flatten(2).transpose(1, 2) + + if getattr(self, "dist_token", None) is not None: + cls_tokens = self.cls_token.expand( + B, -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + dist_token = self.dist_token.expand(B, -1, -1) + x = torch.cat((cls_tokens, dist_token, x), dim=1) + else: + cls_tokens = self.cls_token.expand( + B, -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + x = torch.cat((cls_tokens, x), dim=1) + + x = x + pos_embed + x = self.pos_drop(x) + + for blk in self.blocks: + x = blk(x) + + x = self.norm(x) + + return x + + +def get_readout_oper(vit_features, features, use_readout, start_index=1): + if use_readout == "ignore": + readout_oper = [Slice(start_index)] * len(features) + elif use_readout == "add": + readout_oper = [AddReadout(start_index)] * len(features) + elif use_readout == "project": + readout_oper = [ + ProjectReadout(vit_features, start_index) for out_feat in features + ] + else: + assert ( + False + ), "wrong operation for readout token, use_readout can be 'ignore', 'add', or 'project'" + + return readout_oper + + +def _make_pretrained_clip_vitl16_384( + pretrained, use_readout="ignore", hooks=None, enable_attention_hooks=False +): + clip_pretrained, _ = clip.load("ViT-B/32", device='cuda', jit=False) + model = timm.create_model("vit_large_patch16_384", pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks == None else hooks + + pretrained = _make_vit_b16_backbone( + model, + features=[256, 512, 1024, 1024], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + return clip_pretrained, pretrained + + +def _make_pretrained_clipRN50x16_vitl16_384( + pretrained, use_readout="ignore", hooks=None, enable_attention_hooks=False +): + clip_pretrained, _ = clip.load("RN50x16", device='cuda', jit=False) + model = timm.create_model("vit_large_patch16_384", pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks == None else hooks + + pretrained = _make_vit_b16_backbone( + model, + features=[256, 512, 1024, 1024], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + return clip_pretrained, pretrained + + +def _make_pretrained_clip_vitb32_384(pretrained, use_readout="ignore", hooks=None, enable_attention_hooks=False): + clip_pretrained, _ = clip.load("ViT-B/32", device='cuda', jit=False) + model = timm.create_model("vit_base_patch32_384", pretrained=pretrained) + + hooks = [2, 5, 8, 11] if hooks == None else hooks + + pretrained = _make_vit_b32_backbone( + model, + features=[96, 192, 384, 768], + hooks=hooks, + use_readout=use_readout, + enable_attention_hooks=False, + ) + return clip_pretrained, pretrained + + +def _make_vit_b32_backbone( + model, + features=[96, 192, 384, 768], + size=[384, 384], + hooks=[2, 5, 8, 11], + vit_features=768, + use_readout="ignore", + start_index=1, + enable_attention_hooks=False, +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.blocks[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.blocks[hooks[1]].register_forward_hook(get_activation("2")) + pretrained.model.blocks[hooks[2]].register_forward_hook(get_activation("3")) + pretrained.model.blocks[hooks[3]].register_forward_hook(get_activation("4")) + + pretrained.activations = activations + + pretrained.model.patch_size = [32, 32] + pretrained.model.start_index = start_index + + if enable_attention_hooks: + pretrained.model.blocks[hooks[0]].attn.register_forward_hook( + get_attention("attn_1") + ) + pretrained.model.blocks[hooks[1]].attn.register_forward_hook( + get_attention("attn_2") + ) + pretrained.model.blocks[hooks[2]].attn.register_forward_hook( + get_attention("attn_3") + ) + pretrained.model.blocks[hooks[3]].attn.register_forward_hook( + get_attention("attn_4") + ) + pretrained.attention = attention + + readout_oper = get_readout_oper(vit_features, features, use_readout, start_index) + + pretrained.act_postprocess1 = nn.Sequential( + readout_oper[0], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // pretrained.model.patch_size[1], size[1] // pretrained.model.patch_size[0]])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[0], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[0], + out_channels=features[0], + kernel_size=8, + stride=8, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess2 = nn.Sequential( + readout_oper[1], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // pretrained.model.patch_size[1], size[1] // pretrained.model.patch_size[0]])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[1], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[1], + out_channels=features[1], + kernel_size=4, + stride=4, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess3 = nn.Sequential( + readout_oper[2], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // pretrained.model.patch_size[1], size[1] // pretrained.model.patch_size[0]])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[2], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[2], + out_channels=features[2], + kernel_size=2, + stride=2, + padding=0, + # output_padding=output_padding, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess4 = nn.Sequential( + readout_oper[3], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // pretrained.model.patch_size[1], size[1] // pretrained.model.patch_size[0]])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[3], + kernel_size=1, + stride=1, + padding=0, + ), + ) + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model.forward_flex = types.MethodType(forward_flex, pretrained.model) + pretrained.model._resize_pos_embed = types.MethodType( + _resize_pos_embed, pretrained.model + ) + + return pretrained + + +def _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + size=[384, 384], + hooks=[2, 5, 8, 11], + vit_features=768, + use_readout="ignore", + start_index=1, + enable_attention_hooks=False, +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.blocks[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.blocks[hooks[1]].register_forward_hook(get_activation("2")) + pretrained.model.blocks[hooks[2]].register_forward_hook(get_activation("3")) + pretrained.model.blocks[hooks[3]].register_forward_hook(get_activation("4")) + + pretrained.activations = activations + + if enable_attention_hooks: + pretrained.model.blocks[hooks[0]].attn.register_forward_hook( + get_attention("attn_1") + ) + pretrained.model.blocks[hooks[1]].attn.register_forward_hook( + get_attention("attn_2") + ) + pretrained.model.blocks[hooks[2]].attn.register_forward_hook( + get_attention("attn_3") + ) + pretrained.model.blocks[hooks[3]].attn.register_forward_hook( + get_attention("attn_4") + ) + pretrained.attention = attention + + readout_oper = get_readout_oper(vit_features, features, use_readout, start_index) + + # 32, 48, 136, 384 + pretrained.act_postprocess1 = nn.Sequential( + readout_oper[0], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[0], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[0], + out_channels=features[0], + kernel_size=4, + stride=4, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess2 = nn.Sequential( + readout_oper[1], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[1], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[1], + out_channels=features[1], + kernel_size=2, + stride=2, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess3 = nn.Sequential( + readout_oper[2], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[2], + kernel_size=1, + stride=1, + padding=0, + ), + ) + + pretrained.act_postprocess4 = nn.Sequential( + readout_oper[3], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[3], + kernel_size=1, + stride=1, + padding=0, + ), + nn.Conv2d( + in_channels=features[3], + out_channels=features[3], + kernel_size=3, + stride=2, + padding=1, + ), + ) + + pretrained.model.start_index = start_index + pretrained.model.patch_size = [16, 16] + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model.forward_flex = types.MethodType(forward_flex, pretrained.model) + pretrained.model._resize_pos_embed = types.MethodType( + _resize_pos_embed, pretrained.model + ) + + return pretrained \ No newline at end of file diff --git a/submodules/lang_seg/modules/models/lseg_vit_zs.py b/submodules/lang_seg/modules/models/lseg_vit_zs.py new file mode 100644 index 0000000000000000000000000000000000000000..1795cb5ee28bda2345f0cad2e3690b2364194e18 --- /dev/null +++ b/submodules/lang_seg/modules/models/lseg_vit_zs.py @@ -0,0 +1,979 @@ +import torch +import torch.nn as nn +import timm +import types +import math +import torch.nn.functional as F +import clip +from torchvision import models + +activations = {} + +def get_activation(name): + def hook(model, input, output): + activations[name] = output + + return hook + + +attention = {} + + +def get_attention(name): + def hook(module, input, output): + x = input[0] + B, N, C = x.shape + qkv = ( + module.qkv(x) + .reshape(B, N, 3, module.num_heads, C // module.num_heads) + .permute(2, 0, 3, 1, 4) + ) + q, k, v = ( + qkv[0], + qkv[1], + qkv[2], + ) # make torchscript happy (cannot use tensor as tuple) + + attn = (q @ k.transpose(-2, -1)) * module.scale + + attn = attn.softmax(dim=-1) # [:,:,1,1:] + attention[name] = attn + + return hook + + +def get_mean_attention_map(attn, token, shape): + attn = attn[:, :, token, 1:] + attn = attn.unflatten(2, torch.Size([shape[2] // 16, shape[3] // 16])).float() + attn = torch.nn.functional.interpolate( + attn, size=shape[2:], mode="bicubic", align_corners=False + ).squeeze(0) + + all_attn = torch.mean(attn, 0) + + return all_attn + + +class Slice(nn.Module): + def __init__(self, start_index=1): + super(Slice, self).__init__() + self.start_index = start_index + + def forward(self, x): + return x[:, self.start_index :] + + +class AddReadout(nn.Module): + def __init__(self, start_index=1): + super(AddReadout, self).__init__() + self.start_index = start_index + + def forward(self, x): + if self.start_index == 2: + readout = (x[:, 0] + x[:, 1]) / 2 + else: + readout = x[:, 0] + return x[:, self.start_index :] + readout.unsqueeze(1) + + +class ProjectReadout(nn.Module): + def __init__(self, in_features, start_index=1): + super(ProjectReadout, self).__init__() + self.start_index = start_index + + self.project = nn.Sequential(nn.Linear(2 * in_features, in_features), nn.GELU()) + + def forward(self, x): + readout = x[:, 0].unsqueeze(1).expand_as(x[:, self.start_index :]) + features = torch.cat((x[:, self.start_index :], readout), -1) + + return self.project(features) + + +class Transpose(nn.Module): + def __init__(self, dim0, dim1): + super(Transpose, self).__init__() + self.dim0 = dim0 + self.dim1 = dim1 + + def forward(self, x): + x = x.transpose(self.dim0, self.dim1) + return x + + +def forward_vit(pretrained, x): + b, c, h, w = x.shape + + # encoder + glob = pretrained.model.forward_flex(x) + + # import pdb; pdb.set_trace() + # print(glob.shape) + + # import pdb; pdb.set_trace() + layer_1 = pretrained.activations["1"] + layer_2 = pretrained.activations["2"] + layer_3 = pretrained.activations["3"] + layer_4 = pretrained.activations["4"] + + # import pdb; pdb.set_trace() + layer_1 = pretrained.act_postprocess1[0:2](layer_1) + layer_2 = pretrained.act_postprocess2[0:2](layer_2) + layer_3 = pretrained.act_postprocess3[0:2](layer_3) + layer_4 = pretrained.act_postprocess4[0:2](layer_4) + + unflatten = nn.Sequential( + nn.Unflatten( + 2, + torch.Size( + [ + h // pretrained.model.patch_size[1], + w // pretrained.model.patch_size[0], + ] + ), + ) + ) + + if layer_1.ndim == 3: + layer_1 = unflatten(layer_1) + if layer_2.ndim == 3: + layer_2 = unflatten(layer_2) + if layer_3.ndim == 3: + layer_3 = unflatten(layer_3) + if layer_4.ndim == 3: + layer_4 = unflatten(layer_4) + + layer_1 = pretrained.act_postprocess1[3 : len(pretrained.act_postprocess1)](layer_1) + layer_2 = pretrained.act_postprocess2[3 : len(pretrained.act_postprocess2)](layer_2) + layer_3 = pretrained.act_postprocess3[3 : len(pretrained.act_postprocess3)](layer_3) + layer_4 = pretrained.act_postprocess4[3 : len(pretrained.act_postprocess4)](layer_4) + + return layer_1, layer_2, layer_3, layer_4 + + +def _resize_pos_embed(self, posemb, gs_h, gs_w): + posemb_tok, posemb_grid = ( + posemb[:, : self.start_index], + posemb[0, self.start_index :], + ) + + gs_old = int(math.sqrt(len(posemb_grid))) + + posemb_grid = posemb_grid.reshape(1, gs_old, gs_old, -1).permute(0, 3, 1, 2) + posemb_grid = F.interpolate(posemb_grid, size=(gs_h, gs_w), mode="bilinear") + posemb_grid = posemb_grid.permute(0, 2, 3, 1).reshape(1, gs_h * gs_w, -1) + + posemb = torch.cat([posemb_tok, posemb_grid], dim=1) + + return posemb + + +def forward_flex(self, x): + b, c, h, w = x.shape + + pos_embed = self._resize_pos_embed( + self.pos_embed, h // self.patch_size[1], w // self.patch_size[0] + ) + + B = x.shape[0] + + if hasattr(self.patch_embed, "backbone"): + x = self.patch_embed.backbone(x) + if isinstance(x, (list, tuple)): + x = x[-1] # last feature if backbone outputs list/tuple of features + x = self.patch_embed.proj(x).flatten(2).transpose(1, 2) + + if getattr(self, "dist_token", None) is not None: + cls_tokens = self.cls_token.expand( + B, -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + dist_token = self.dist_token.expand(B, -1, -1) + x = torch.cat((cls_tokens, dist_token, x), dim=1) + else: + cls_tokens = self.cls_token.expand( + B, -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + x = torch.cat((cls_tokens, x), dim=1) + + x = x + pos_embed + x = self.pos_drop(x) + + for blk in self.blocks: + x = blk(x) + + x = self.norm(x) + + return x + + +def get_readout_oper(vit_features, features, use_readout, start_index=1): + if use_readout == "ignore": + readout_oper = [Slice(start_index)] * len(features) + elif use_readout == "add": + readout_oper = [AddReadout(start_index)] * len(features) + elif use_readout == "project": + readout_oper = [ + ProjectReadout(vit_features, start_index) for out_feat in features + ] + else: + assert ( + False + ), "wrong operation for readout token, use_readout can be 'ignore', 'add', or 'project'" + + return readout_oper + + +def _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + size=[384, 384], + hooks=[2, 5, 8, 11], + vit_features=768, + use_readout="ignore", + start_index=1, + enable_attention_hooks=False, +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.blocks[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.blocks[hooks[1]].register_forward_hook(get_activation("2")) + pretrained.model.blocks[hooks[2]].register_forward_hook(get_activation("3")) + pretrained.model.blocks[hooks[3]].register_forward_hook(get_activation("4")) + + pretrained.activations = activations + + if enable_attention_hooks: + pretrained.model.blocks[hooks[0]].attn.register_forward_hook( + get_attention("attn_1") + ) + pretrained.model.blocks[hooks[1]].attn.register_forward_hook( + get_attention("attn_2") + ) + pretrained.model.blocks[hooks[2]].attn.register_forward_hook( + get_attention("attn_3") + ) + pretrained.model.blocks[hooks[3]].attn.register_forward_hook( + get_attention("attn_4") + ) + pretrained.attention = attention + + readout_oper = get_readout_oper(vit_features, features, use_readout, start_index) + + # 32, 48, 136, 384 + pretrained.act_postprocess1 = nn.Sequential( + readout_oper[0], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[0], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[0], + out_channels=features[0], + kernel_size=4, + stride=4, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess2 = nn.Sequential( + readout_oper[1], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[1], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[1], + out_channels=features[1], + kernel_size=2, + stride=2, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess3 = nn.Sequential( + readout_oper[2], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[2], + kernel_size=1, + stride=1, + padding=0, + ), + ) + + pretrained.act_postprocess4 = nn.Sequential( + readout_oper[3], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[3], + kernel_size=1, + stride=1, + padding=0, + ), + nn.Conv2d( + in_channels=features[3], + out_channels=features[3], + kernel_size=3, + stride=2, + padding=1, + ), + ) + + pretrained.model.start_index = start_index + pretrained.model.patch_size = [16, 16] + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model.forward_flex = types.MethodType(forward_flex, pretrained.model) + pretrained.model._resize_pos_embed = types.MethodType( + _resize_pos_embed, pretrained.model + ) + + return pretrained + + +def _make_vit_b_rn50_backbone( + model, + features=[256, 512, 768, 768], + size=[384, 384], + hooks=[0, 1, 8, 11], + vit_features=768, + use_vit_only=False, + use_readout="ignore", + start_index=1, + enable_attention_hooks=False, +): + pretrained = nn.Module() + + pretrained.model = model + + if use_vit_only == True: + pretrained.model.blocks[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.blocks[hooks[1]].register_forward_hook(get_activation("2")) + else: + pretrained.model.patch_embed.backbone.stages[0].register_forward_hook( + get_activation("1") + ) + pretrained.model.patch_embed.backbone.stages[1].register_forward_hook( + get_activation("2") + ) + + pretrained.model.blocks[hooks[2]].register_forward_hook(get_activation("3")) + pretrained.model.blocks[hooks[3]].register_forward_hook(get_activation("4")) + + if enable_attention_hooks: + pretrained.model.blocks[2].attn.register_forward_hook(get_attention("attn_1")) + pretrained.model.blocks[5].attn.register_forward_hook(get_attention("attn_2")) + pretrained.model.blocks[8].attn.register_forward_hook(get_attention("attn_3")) + pretrained.model.blocks[11].attn.register_forward_hook(get_attention("attn_4")) + pretrained.attention = attention + + pretrained.activations = activations + + readout_oper = get_readout_oper(vit_features, features, use_readout, start_index) + + if use_vit_only == True: + pretrained.act_postprocess1 = nn.Sequential( + readout_oper[0], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[0], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[0], + out_channels=features[0], + kernel_size=4, + stride=4, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess2 = nn.Sequential( + readout_oper[1], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[1], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[1], + out_channels=features[1], + kernel_size=2, + stride=2, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + else: + pretrained.act_postprocess1 = nn.Sequential( + nn.Identity(), nn.Identity(), nn.Identity() + ) + pretrained.act_postprocess2 = nn.Sequential( + nn.Identity(), nn.Identity(), nn.Identity() + ) + + pretrained.act_postprocess3 = nn.Sequential( + readout_oper[2], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[2], + kernel_size=1, + stride=1, + padding=0, + ), + ) + + pretrained.act_postprocess4 = nn.Sequential( + readout_oper[3], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[3], + kernel_size=1, + stride=1, + padding=0, + ), + nn.Conv2d( + in_channels=features[3], + out_channels=features[3], + kernel_size=3, + stride=2, + padding=1, + ), + ) + + pretrained.model.start_index = start_index + pretrained.model.patch_size = [16, 16] + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model.forward_flex = types.MethodType(forward_flex, pretrained.model) + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model._resize_pos_embed = types.MethodType( + _resize_pos_embed, pretrained.model + ) + + return pretrained + + +def _make_pretrained_vitb_rn50_384( + pretrained, + use_readout="ignore", + hooks=None, + use_vit_only=False, + enable_attention_hooks=False, +): + model = timm.create_model("vit_base_resnet50_384", pretrained=pretrained) + + hooks = [0, 1, 8, 11] if hooks == None else hooks + return _make_vit_b_rn50_backbone( + model, + features=[256, 512, 768, 768], + size=[384, 384], + hooks=hooks, + use_vit_only=use_vit_only, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + + + +def _make_pretrained_vitl16_384( + pretrained, use_readout="ignore", hooks=None, enable_attention_hooks=False +): + model = timm.create_model("vit_large_patch16_384", pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks == None else hooks + return _make_vit_b16_backbone( + model, + features=[256, 512, 1024, 1024], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + + +def _make_pretrained_vitb16_384( + pretrained, use_readout="ignore", hooks=None, enable_attention_hooks=False +): + model = timm.create_model("vit_base_patch16_384", pretrained=pretrained) + + hooks = [2, 5, 8, 11] if hooks == None else hooks + return _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + hooks=hooks, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + + +def _make_pretrained_deitb16_384( + pretrained, use_readout="ignore", hooks=None, enable_attention_hooks=False +): + model = timm.create_model("vit_deit_base_patch16_384", pretrained=pretrained) + + hooks = [2, 5, 8, 11] if hooks == None else hooks + return _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + hooks=hooks, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + + +def _make_pretrained_deitb16_distil_384( + pretrained, use_readout="ignore", hooks=None, enable_attention_hooks=False +): + model = timm.create_model( + "vit_deit_base_distilled_patch16_384", pretrained=pretrained + ) + + hooks = [2, 5, 8, 11] if hooks == None else hooks + return _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + hooks=hooks, + use_readout=use_readout, + start_index=2, + enable_attention_hooks=enable_attention_hooks, + ) + + +def _make_pretrained_clip_vitl16_384( + pretrained, use_readout="ignore", hooks=None, enable_attention_hooks=False +): + clip_pretrained, _ = clip.load("ViT-B/32", device='cuda', jit=False) + model = timm.create_model("vit_large_patch16_384", pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks == None else hooks + + pretrained = _make_vit_b16_backbone( + model, + features=[256, 512, 1024, 1024], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + return clip_pretrained, pretrained + + +def _make_pretrained_clipb16_vitl16_384( + pretrained, use_readout="ignore", hooks=None, enable_attention_hooks=False +): + clip_pretrained, _ = clip.load("ViT-B/16", device='cuda', jit=False) + model = timm.create_model("vit_large_patch16_384", pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks == None else hooks + + pretrained = _make_vit_b16_backbone( + model, + features=[256, 512, 1024, 1024], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + return clip_pretrained, pretrained + + +def _make_pretrained_clipRN50x16_vitl16_384( + pretrained, use_readout="ignore", hooks=None, enable_attention_hooks=False +): + clip_pretrained, _ = clip.load("RN50x16", device='cuda', jit=False) + model = timm.create_model("vit_large_patch16_384", pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks == None else hooks + + pretrained = _make_vit_b16_backbone( + model, + features=[256, 512, 1024, 1024], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + return clip_pretrained, pretrained + + +def _make_pretrained_clipRN50x4_vitl16_384( + pretrained, use_readout="ignore", hooks=None, enable_attention_hooks=False +): + clip_pretrained, _ = clip.load("RN50x4", device='cuda', jit=False) + model = timm.create_model("vit_large_patch16_384", pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks == None else hooks + + pretrained = _make_vit_b16_backbone( + model, + features=[256, 512, 1024, 1024], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + return clip_pretrained, pretrained + + +def _make_pretrained_clip_vitb16_384( + pretrained, use_readout="ignore", hooks=None, enable_attention_hooks=False +): + clip_pretrained, _ = clip.load("ViT-B/16", device='cuda', jit=False) + + if pretrained is True or pretrained is False: + print('** _make_pretrained_clip_vitb16_384 ==> timm.create_model("vit_base_patch16_384", pretrained={}) **'.format(pretrained)) + model = timm.create_model("vit_base_patch16_384", pretrained=pretrained) + elif pretrained in ['clip', 'clip_fixed']: + print('** _make_pretrained_clip_vitb16_384 ==> use default clip model **') + model = timm.create_model("vit_base_patch16_384", pretrained=False) + + clip_dict = clip_pretrained.visual.state_dict() + model_keys = model.state_dict().keys() + clip_keys = clip_dict.keys() + + model.state_dict()['cls_token'][0,0,:] = clip_dict['class_embedding'] + model.state_dict()['patch_embed.proj.weight'][:] = clip_dict['conv1.weight'] + for i in range(12): + model.state_dict()['blocks.{}.attn.qkv.weight'.format(i)][:] = clip_dict['transformer.resblocks.{}.attn.in_proj_weight'.format(i)] + model.state_dict()['blocks.{}.attn.qkv.bias'.format(i)][:] = clip_dict['transformer.resblocks.{}.attn.in_proj_bias'.format(i)] + model.state_dict()['blocks.{}.attn.proj.weight'.format(i)][:] = clip_dict['transformer.resblocks.{}.attn.out_proj.weight'.format(i)] + model.state_dict()['blocks.{}.attn.proj.bias'.format(i)][:] = clip_dict['transformer.resblocks.{}.attn.out_proj.bias'.format(i)] + + model.state_dict()['blocks.{}.norm1.weight'.format(i)][:] = clip_dict['transformer.resblocks.{}.ln_1.weight'.format(i)] + model.state_dict()['blocks.{}.norm1.bias'.format(i)][:] = clip_dict['transformer.resblocks.{}.ln_1.bias'.format(i)] + model.state_dict()['blocks.{}.norm2.weight'.format(i)][:] = clip_dict['transformer.resblocks.{}.ln_2.weight'.format(i)] + model.state_dict()['blocks.{}.norm2.bias'.format(i)][:] = clip_dict['transformer.resblocks.{}.ln_2.bias'.format(i)] + + model.state_dict()['blocks.{}.mlp.fc1.weight'.format(i)][:] = clip_dict['transformer.resblocks.{}.mlp.c_fc.weight'.format(i)] + model.state_dict()['blocks.{}.mlp.fc1.bias'.format(i)][:] = clip_dict['transformer.resblocks.{}.mlp.c_fc.bias'.format(i)] + model.state_dict()['blocks.{}.mlp.fc2.weight'.format(i)][:] = clip_dict['transformer.resblocks.{}.mlp.c_proj.weight'.format(i)] + model.state_dict()['blocks.{}.mlp.fc2.bias'.format(i)][:] = clip_dict['transformer.resblocks.{}.mlp.c_proj.bias'.format(i)] + + model.state_dict()['norm.weight'][:] = clip_dict['ln_post.weight'] + model.state_dict()['norm.bias'][:] = clip_dict['ln_post.bias'] + + # for key in model_keys: + # print('{} ==> {}'.format(key, model.state_dict()[key].shape)) + # for key in clip_keys: + # print('{} ==> {}'.format(key, clip_dict[key].shape)) + else: + print('Error: pretrained not found!') + exit() + + hooks = [2, 5, 8, 11] if hooks == None else hooks + + pretrained = _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + hooks=hooks, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + return clip_pretrained, pretrained + + +def _make_pretrained_clip_vitb_rn50_384( + pretrained, + use_readout="ignore", + hooks=None, + use_vit_only=False, + enable_attention_hooks=False, +): + clip_pretrained, _ = clip.load("RN50", device='cuda', jit=False) + model = timm.create_model("vit_base_resnet50_384", pretrained=pretrained) + + hooks = [0, 1, 8, 11] if hooks == None else hooks + + pretrained = _make_vit_b_rn50_backbone( + model, + features=[256, 512, 768, 768], + size=[384, 384], + hooks=hooks, + use_vit_only=use_vit_only, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + return clip_pretrained, pretrained + + +def _make_pretrained_clip_rn101( + pretrained, +): + clip_pretrained, _ = clip.load("ViT-B/32", device='cuda', jit=False) + resnet = models.resnet101(pretrained=pretrained) + pretrained = _make_resnet_backbone(resnet) + return clip_pretrained, pretrained + + +def _make_resnet_backbone(resnet): + pretrained = nn.Module() + pretrained.layer1 = nn.Sequential( + resnet.conv1, resnet.bn1, resnet.relu, resnet.maxpool, resnet.layer1 + ) + + pretrained.layer2 = resnet.layer2 + pretrained.layer3 = resnet.layer3 + pretrained.layer4 = resnet.layer4 + + return pretrained + +def _make_pretrained_clip_vitb32_384(pretrained, use_readout="ignore", hooks=None, enable_attention_hooks=False): + clip_pretrained, _ = clip.load("ViT-B/32", device='cuda', jit=False) + if pretrained is True or pretrained is False: + print('** _make_pretrained_clip_vitb32_384 ==> timm.create_model("vit_base_patch32_384", pretrained={}) **'.format(pretrained)) + model = timm.create_model("vit_base_patch32_384", pretrained=pretrained) + elif pretrained in ['clip', 'clip_fixed']: + print('** _make_pretrained_clip_vitb32_384 ==> use default clip model **') + model = timm.create_model("vit_base_patch32_384", pretrained=False) + + clip_dict = clip_pretrained.visual.state_dict() + model_keys = model.state_dict().keys() + clip_keys = clip_dict.keys() + + model.state_dict()['cls_token'][0,0,:] = clip_dict['class_embedding'] + model.state_dict()['patch_embed.proj.weight'][:] = clip_dict['conv1.weight'] + for i in range(12): + model.state_dict()['blocks.{}.attn.qkv.weight'.format(i)][:] = clip_dict['transformer.resblocks.{}.attn.in_proj_weight'.format(i)] + model.state_dict()['blocks.{}.attn.qkv.bias'.format(i)][:] = clip_dict['transformer.resblocks.{}.attn.in_proj_bias'.format(i)] + model.state_dict()['blocks.{}.attn.proj.weight'.format(i)][:] = clip_dict['transformer.resblocks.{}.attn.out_proj.weight'.format(i)] + model.state_dict()['blocks.{}.attn.proj.bias'.format(i)][:] = clip_dict['transformer.resblocks.{}.attn.out_proj.bias'.format(i)] + + model.state_dict()['blocks.{}.norm1.weight'.format(i)][:] = clip_dict['transformer.resblocks.{}.ln_1.weight'.format(i)] + model.state_dict()['blocks.{}.norm1.bias'.format(i)][:] = clip_dict['transformer.resblocks.{}.ln_1.bias'.format(i)] + model.state_dict()['blocks.{}.norm2.weight'.format(i)][:] = clip_dict['transformer.resblocks.{}.ln_2.weight'.format(i)] + model.state_dict()['blocks.{}.norm2.bias'.format(i)][:] = clip_dict['transformer.resblocks.{}.ln_2.bias'.format(i)] + + model.state_dict()['blocks.{}.mlp.fc1.weight'.format(i)][:] = clip_dict['transformer.resblocks.{}.mlp.c_fc.weight'.format(i)] + model.state_dict()['blocks.{}.mlp.fc1.bias'.format(i)][:] = clip_dict['transformer.resblocks.{}.mlp.c_fc.bias'.format(i)] + model.state_dict()['blocks.{}.mlp.fc2.weight'.format(i)][:] = clip_dict['transformer.resblocks.{}.mlp.c_proj.weight'.format(i)] + model.state_dict()['blocks.{}.mlp.fc2.bias'.format(i)][:] = clip_dict['transformer.resblocks.{}.mlp.c_proj.bias'.format(i)] + + model.state_dict()['norm.weight'][:] = clip_dict['ln_post.weight'] + model.state_dict()['norm.bias'][:] = clip_dict['ln_post.bias'] + + # for key in model_keys: + # print('{} ==> {}'.format(key, model.state_dict()[key].shape)) + # for key in clip_keys: + # print('{} ==> {}'.format(key, clip_dict[key].shape)) + else: + print('Error: pretrained not found!') + exit() + + hooks = [2, 5, 8, 11] if hooks == None else hooks + + pretrained = _make_vit_b32_backbone( + model, + features=[96, 192, 384, 768], + hooks=hooks, + use_readout=use_readout, + enable_attention_hooks=False, + ) + return clip_pretrained, pretrained + + +def _make_pretrained_clip_select_vitb32_384( + pretrained, use_readout="ignore", hooks=None, enable_attention_hooks=False, backbone='clip_vitb32_384' +): + if backbone == 'clip_vitb32_384': + clip_pretrained, _ = clip.load("ViT-B/32", device='cuda', jit=False) + elif backbone == 'clipb16_vitb32_384': + clip_pretrained, _ = clip.load("ViT-B/16", device='cuda', jit=False) + elif backbone == 'clipRN50x16_vitb32_384': + clip_pretrained, _ = clip.load("RN50x16", device='cuda', jit=False) + elif backbone == 'clipRN50x4_vitb32_384': + clip_pretrained, _ = clip.load("RN50x4", device='cuda', jit=False) + else: + print('Error in _make_pretrained_clip_select_vitb32_384, no such backbone') + exit() + + model = timm.create_model("vit_base_patch32_384", pretrained=pretrained) + + hooks = [2, 5, 8, 11] if hooks == None else hooks + + pretrained = _make_vit_b32_backbone( + model, + features=[96, 192, 384, 768], + hooks=hooks, + use_readout=use_readout, + enable_attention_hooks=False, + ) + return clip_pretrained, pretrained + + +# current version +def _make_vit_b32_backbone( + model, + features=[96, 192, 384, 768], + size=[384, 384], + hooks=[2, 5, 8, 11], + vit_features=768, + use_readout="ignore", + start_index=1, + enable_attention_hooks=False, +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.blocks[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.blocks[hooks[1]].register_forward_hook(get_activation("2")) + pretrained.model.blocks[hooks[2]].register_forward_hook(get_activation("3")) + pretrained.model.blocks[hooks[3]].register_forward_hook(get_activation("4")) + + pretrained.activations = activations + + pretrained.model.patch_size = [32, 32] + pretrained.model.start_index = start_index + + if enable_attention_hooks: + pretrained.model.blocks[hooks[0]].attn.register_forward_hook( + get_attention("attn_1") + ) + pretrained.model.blocks[hooks[1]].attn.register_forward_hook( + get_attention("attn_2") + ) + pretrained.model.blocks[hooks[2]].attn.register_forward_hook( + get_attention("attn_3") + ) + pretrained.model.blocks[hooks[3]].attn.register_forward_hook( + get_attention("attn_4") + ) + pretrained.attention = attention + + readout_oper = get_readout_oper(vit_features, features, use_readout, start_index) + + # 32, 48, 136, 384 + pretrained.act_postprocess1 = nn.Sequential( + readout_oper[0], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // pretrained.model.patch_size[1], size[1] // pretrained.model.patch_size[0]])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[0], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[0], + out_channels=features[0], + kernel_size=8, + stride=8, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess2 = nn.Sequential( + readout_oper[1], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // pretrained.model.patch_size[1], size[1] // pretrained.model.patch_size[0]])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[1], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[1], + out_channels=features[1], + kernel_size=4, + stride=4, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess3 = nn.Sequential( + readout_oper[2], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // pretrained.model.patch_size[1], size[1] // pretrained.model.patch_size[0]])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[2], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[2], + out_channels=features[2], + kernel_size=2, + stride=2, + padding=0, + # output_padding=output_padding, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess4 = nn.Sequential( + readout_oper[3], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // pretrained.model.patch_size[1], size[1] // pretrained.model.patch_size[0]])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[3], + kernel_size=1, + stride=1, + padding=0, + ), + ) + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model.forward_flex = types.MethodType(forward_flex, pretrained.model) + pretrained.model._resize_pos_embed = types.MethodType( + _resize_pos_embed, pretrained.model + ) + + return pretrained + diff --git a/submodules/lang_seg/prepare_ade20k.py b/submodules/lang_seg/prepare_ade20k.py new file mode 100644 index 0000000000000000000000000000000000000000..6f86a10d020f0e7a93d2ddc0243c224c3e8dbd16 --- /dev/null +++ b/submodules/lang_seg/prepare_ade20k.py @@ -0,0 +1,45 @@ +# + +# revised from https://github.com/zhanghang1989/PyTorch-Encoding/blob/331ecdd5306104614cb414b16fbcd9d1a8d40e1e/scripts/prepare_ade20k.py + +"""Prepare ADE20K dataset""" +import os +import shutil +import argparse +import zipfile +from encoding.utils import download, mkdir +# - + +_TARGET_DIR = os.path.expanduser('../datasets/') + +def parse_args(): + parser = argparse.ArgumentParser( + description='Initialize ADE20K dataset.', + epilog='Example: python prepare_ade20k.py', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--download-dir', default=None, help='dataset directory on disk') + args = parser.parse_args() + return args + +def download_ade(path, overwrite=False): + _AUG_DOWNLOAD_URLS = [ + ('http://data.csail.mit.edu/places/ADEchallenge/ADEChallengeData2016.zip', '219e1696abb36c8ba3a3afe7fb2f4b4606a897c7'), + ('http://data.csail.mit.edu/places/ADEchallenge/release_test.zip', 'e05747892219d10e9243933371a497e905a4860c'),] + download_dir = path + mkdir(download_dir) + for url, checksum in _AUG_DOWNLOAD_URLS: + filename = download(url, path=download_dir, overwrite=overwrite, sha1_hash=checksum) + # extract + with zipfile.ZipFile(filename,"r") as zip_ref: + zip_ref.extractall(path=path) + + +if __name__ == '__main__': + args = parse_args() + mkdir(os.path.expanduser('../datasets/')) + if args.download_dir is not None: + if os.path.isdir(_TARGET_DIR): + os.remove(_TARGET_DIR) + # make symlink + os.symlink(args.download_dir, _TARGET_DIR) + else: + download_ade(_TARGET_DIR, overwrite=False) diff --git a/submodules/lang_seg/requirements.txt b/submodules/lang_seg/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..65bfda46ce11ef203ead3b929734b91fab5894ad --- /dev/null +++ b/submodules/lang_seg/requirements.txt @@ -0,0 +1,118 @@ +absl-py==0.14.1 +aiohttp==3.7.4.post0 +anyio==3.3.4 +argon2-cffi==21.1.0 +async-timeout==3.0.1 +attrs==21.2.0 +Babel==2.9.1 +backcall==0.2.0 +bleach==4.1.0 +cachetools==4.2.4 +certifi==2021.10.8 +cffi==1.15.0 +chardet==4.0.0 +charset-normalizer==2.0.7 +clip @ git+https://github.com/openai/CLIP.git@04f4dc2ca1ed0acc9893bd1a3b526a7e02c4bb10 +cycler==0.10.0 +debugpy==1.5.0 +decorator==5.1.0 +defusedxml==0.7.1 +entrypoints==0.3 +fsspec==2021.10.1 +ftfy==6.0.3 +future==0.18.2 +google-auth==2.3.0 +google-auth-oauthlib==0.4.6 +grpcio==1.41.0 +idna==3.3 +imageio==2.9.0 +ipykernel==6.4.1 +ipython==7.28.0 +ipython-genutils==0.2.0 +ipywidgets==7.6.5 +jedi==0.18.0 +Jinja2==3.0.2 +json5==0.9.6 +jsonschema==4.1.0 +jupyter==1.0.0 +jupyter-client==7.0.6 +jupyter-console==6.4.0 +jupyter-core==4.8.1 +jupyter-server==1.11.1 +jupyterlab==3.2.0 +jupyterlab-pygments==0.1.2 +jupyterlab-server==2.8.2 +jupyterlab-widgets==1.0.2 +kiwisolver==1.3.2 +Markdown==3.3.4 +MarkupSafe==2.0.1 +matplotlib==3.4.3 +matplotlib-inline==0.1.3 +mistune==0.8.4 +multidict==5.2.0 +nbclassic==0.3.2 +nbclient==0.5.4 +nbconvert==6.2.0 +nbformat==5.1.3 +nest-asyncio==1.5.1 +nose==1.3.7 +notebook==6.4.4 +numpy==1.21.2 +oauthlib==3.1.1 +packaging==21.0 +pandas==1.3.4 +pandocfilters==1.5.0 +parso==0.8.2 +pexpect==4.8.0 +pickleshare==0.7.5 +Pillow==8.4.0 +portalocker==2.3.2 +prometheus-client==0.11.0 +prompt-toolkit==3.0.20 +protobuf==3.18.1 +ptyprocess==0.7.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycparser==2.20 +pyDeprecate==0.3.1 +Pygments==2.10.0 +pyparsing==2.4.7 +pyrsistent==0.18.0 +python-dateutil==2.8.2 +pytorch-lightning==1.4.9 +pytz==2021.3 +PyYAML==6.0 +pyzmq==22.3.0 +qtconsole==5.1.1 +QtPy==1.11.2 +regex==2021.10.8 +requests==2.26.0 +requests-oauthlib==1.3.0 +requests-unixsocket==0.2.0 +rsa==4.7.2 +scipy==1.7.1 +Send2Trash==1.8.0 +six==1.16.0 +sniffio==1.2.0 +tensorboard==2.7.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +terminado==0.12.1 +testpath==0.5.0 +timm==0.4.12 +torch==1.9.1+cu111 +torch-encoding @ git+https://github.com/zhanghang1989/PyTorch-Encoding/@331ecdd5306104614cb414b16fbcd9d1a8d40e1e +torchaudio==0.9.1 +torchmetrics==0.5.1 +torchvision==0.10.1+cu111 +tornado==6.1 +tqdm==4.62.3 +traitlets==5.1.0 +typing-extensions==3.10.0.2 +urllib3==1.26.7 +wcwidth==0.2.5 +webencodings==0.5.1 +websocket-client==1.2.1 +Werkzeug==2.0.2 +widgetsnbextension==3.5.1 +yarl==1.7.0 diff --git a/submodules/lang_seg/test.sh b/submodules/lang_seg/test.sh new file mode 100644 index 0000000000000000000000000000000000000000..7f0bacaca844f2e94e946132a269344318df5db6 --- /dev/null +++ b/submodules/lang_seg/test.sh @@ -0,0 +1,6 @@ +export CUDA_VISIBLE_DEVICES=0; python test_lseg.py --backbone clip_vitl16_384 --eval --dataset ade20k --data-path ../datasets/ \ +--weights checkpoints/lseg_ade20k_l16.ckpt --widehead --no-scaleinv + + + + diff --git a/submodules/lang_seg/test_lseg.py b/submodules/lang_seg/test_lseg.py new file mode 100644 index 0000000000000000000000000000000000000000..8b0f7b2614241abb8d97960b9b6e41e803f31903 --- /dev/null +++ b/submodules/lang_seg/test_lseg.py @@ -0,0 +1,436 @@ +import os +import argparse +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +import torch +import torch.nn.functional as F +from torch.utils import data +import torchvision.transforms as transform +from torch.nn.parallel.scatter_gather import gather +import encoding.utils as utils +from encoding.nn import SegmentationLosses, SyncBatchNorm +from encoding.parallel import DataParallelModel, DataParallelCriterion +from encoding.datasets import test_batchify_fn +from encoding.models.sseg import BaseNet +from modules.lseg_module import LSegModule +from utils import Resize +import cv2 +import math +import types +import functools +import torchvision.transforms as torch_transforms +import copy +import itertools +from PIL import Image +import matplotlib.pyplot as plt +import clip +import matplotlib as mpl +import matplotlib.colors as mplc +import matplotlib.figure as mplfigure +import matplotlib.patches as mpatches +from matplotlib.backends.backend_agg import FigureCanvasAgg +from data import get_dataset +from additional_utils.encoding_models import MultiEvalModule as LSeg_MultiEvalModule +import torchvision.transforms as transforms + +class Options: + def __init__(self): + parser = argparse.ArgumentParser(description="PyTorch Segmentation") + # model and dataset + parser.add_argument( + "--model", type=str, default="encnet", help="model name (default: encnet)" + ) + parser.add_argument( + "--backbone", + type=str, + default="clip_vitl16_384", + help="backbone name (default: resnet50)", + ) + parser.add_argument( + "--dataset", + type=str, + default="ade20k", + help="dataset name (default: pascal12)", + ) + parser.add_argument( + "--workers", type=int, default=16, metavar="N", help="dataloader threads" + ) + parser.add_argument( + "--base-size", type=int, default=520, help="base image size" + ) + parser.add_argument( + "--crop-size", type=int, default=480, help="crop image size" + ) + parser.add_argument( + "--train-split", + type=str, + default="train", + help="dataset train split (default: train)", + ) + # training hyper params + parser.add_argument( + "--aux", action="store_true", default=False, help="Auxilary Loss" + ) + parser.add_argument( + "--se-loss", + action="store_true", + default=False, + help="Semantic Encoding Loss SE-loss", + ) + parser.add_argument( + "--se-weight", type=float, default=0.2, help="SE-loss weight (default: 0.2)" + ) + parser.add_argument( + "--batch-size", + type=int, + default=16, + metavar="N", + help="input batch size for \ + training (default: auto)", + ) + parser.add_argument( + "--test-batch-size", + type=int, + default=16, + metavar="N", + help="input batch size for \ + testing (default: same as batch size)", + ) + # cuda, seed and logging + parser.add_argument( + "--no-cuda", + action="store_true", + default=False, + help="disables CUDA training", + ) + parser.add_argument( + "--seed", type=int, default=1, metavar="S", help="random seed (default: 1)" + ) + parser.add_argument( + "--weights", type=str, default=None, help="checkpoint to test" + ) + parser.add_argument( + "--eval", action="store_true", default=False, help="evaluating mIoU" + ) + parser.add_argument( + "--export", + type=str, + default=None, + help="put the path to resuming file if needed", + ) + + parser.add_argument( + "--acc-bn", + action="store_true", + default=False, + help="Re-accumulate BN statistics", + ) + parser.add_argument( + "--test-val", + action="store_true", + default=False, + help="generate masks on val set", + ) + parser.add_argument( + "--no-val", + action="store_true", + default=False, + help="skip validation during training", + ) + parser.add_argument( + "--module", + default='lseg', + help="select model definition", + ) + # test option + parser.add_argument( + "--data-path", type=str, default=None, help="path to test image folder" + ) + parser.add_argument( + "--no-scaleinv", + dest="scale_inv", + default=True, + action="store_false", + help="turn off scaleinv layers", + ) + parser.add_argument( + "--widehead", default=False, action="store_true", help="wider output head" + ) + parser.add_argument( + "--widehead_hr", + default=False, + action="store_true", + help="wider output head", + ) + parser.add_argument( + "--ignore_index", + type=int, + default=-1, + help="numeric value of ignore label in gt", + ) + parser.add_argument( + "--label_src", + type=str, + default="default", + help="how to get the labels", + ) + parser.add_argument( + "--jobname", + type=str, + default="default", + help="select which dataset", + ) + parser.add_argument( + "--no-strict", + dest="strict", + default=True, + action="store_false", + help="no-strict copy the model", + ) + parser.add_argument( + "--arch_option", + type=int, + default=0, + help="which kind of architecture to be used", + ) + parser.add_argument( + "--block_depth", + type=int, + default=0, + help="how many blocks should be used", + ) + parser.add_argument( + "--activation", + choices=['lrelu', 'tanh'], + default="lrelu", + help="use which activation to activate the block", + ) + + self.parser = parser + + def parse(self): + args = self.parser.parse_args() + args.cuda = not args.no_cuda and torch.cuda.is_available() + print(args) + return args + + +def test(args): + + module = LSegModule.load_from_checkpoint( + checkpoint_path=args.weights, + data_path=args.data_path, + dataset=args.dataset, + backbone=args.backbone, + aux=args.aux, + num_features=256, + aux_weight=0, + se_loss=False, + se_weight=0, + base_lr=0, + batch_size=1, + max_epochs=0, + ignore_index=args.ignore_index, + dropout=0.0, + scale_inv=args.scale_inv, + augment=False, + no_batchnorm=False, + widehead=args.widehead, + widehead_hr=args.widehead_hr, + map_locatin="cpu", + arch_option=args.arch_option, + strict=args.strict, + block_depth=args.block_depth, + activation=args.activation, + ) + input_transform = module.val_transform + num_classes = module.num_classes + + # dataset + testset = get_dataset( + args.dataset, + root=args.data_path, + split="val", + mode="testval", + transform=input_transform, + ) + + # dataloader + loader_kwargs = ( + {"num_workers": args.workers, "pin_memory": True} if args.cuda else {} + ) + test_data = data.DataLoader( + testset, + batch_size=args.test_batch_size, + drop_last=False, + shuffle=False, + collate_fn=test_batchify_fn, + **loader_kwargs + ) + + if isinstance(module.net, BaseNet): + model = module.net + else: + model = module + + model = model.eval() + model = model.cpu() + + print(model) + if args.acc_bn: + from encoding.utils.precise_bn import update_bn_stats + + data_kwargs = { + "transform": input_transform, + "base_size": args.base_size, + "crop_size": args.crop_size, + } + trainset = get_dataset( + args.dataset, split=args.train_split, mode="train", **data_kwargs + ) + trainloader = data.DataLoader( + ReturnFirstClosure(trainset), + root=args.data_path, + batch_size=args.batch_size, + drop_last=True, + shuffle=True, + **loader_kwargs + ) + print("Reseting BN statistics") + model.cuda() + update_bn_stats(model, trainloader) + + if args.export: + torch.save(model.state_dict(), args.export + ".pth") + return + + scales = ( + [0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.25] + if args.dataset == "citys" + else [0.5, 0.75, 1.0, 1.25, 1.5, 1.75] + ) + + evaluator = LSeg_MultiEvalModule( + model, num_classes, scales=scales, flip=True + ).cuda() + evaluator.eval() + + metric = utils.SegmentationMetric(testset.num_class) + tbar = tqdm(test_data) + + f = open("logs/log_test_{}_{}.txt".format(args.jobname, args.dataset), "a+") + per_class_iou = np.zeros(testset.num_class) + cnt = 0 + for i, (image, dst) in enumerate(tbar): + if args.eval: + with torch.no_grad(): + if False: + sample = {"image": image[0].cpu().permute(1, 2, 0).numpy()} + out = torch.zeros( + 1, testset.num_class, image[0].shape[1], image[0].shape[2] + ).cuda() + + H, W = image[0].shape[1], image[0].shape[2] + for scale in scales: + long_size = int(math.ceil(520 * scale)) + if H > W: + height = long_size + width = int(1.0 * W * long_size / H + 0.5) + short_size = width + else: + width = long_size + height = int(1.0 * H * long_size / W + 0.5) + short_size = height + + rs = Resize( + width, + height, + resize_target=False, + keep_aspect_ratio=True, + ensure_multiple_of=32, + resize_method="minimal", + image_interpolation_method=cv2.INTER_AREA, + ) + + inf_image = ( + torch.from_numpy(rs(sample)["image"]) + .cuda() + .permute(2, 0, 1) + .unsqueeze(0) + ) + inf_image = torch.cat((inf_image, torch.fliplr(inf_image)), 0) + try: + pred = model(inf_image) + except: + print(H, W, sz, i) + exit() + + pred0 = F.softmax(pred[0], dim=1) + pred1 = F.softmax(pred[1], dim=1) + + pred = pred0 + 0.2 * pred1 + + out += F.interpolate( + pred.sum(0, keepdim=True), + (out.shape[2], out.shape[3]), + mode="bilinear", + align_corners=True, + ) + + predicts = [out] + else: + predicts = evaluator.parallel_forward(image) + + metric.update(dst, predicts) + pixAcc, mIoU = metric.get() + + _, _, total_inter, total_union = metric.get_all() + per_class_iou += 1.0 * total_inter / (np.spacing(1) + total_union) + cnt+=1 + + tbar.set_description("pixAcc: %.4f, mIoU: %.4f" % (pixAcc, mIoU)) + else: + with torch.no_grad(): + outputs = evaluator.parallel_forward(image) + predicts = [ + testset.make_pred(torch.max(output, 1)[1].cpu().numpy()) + for output in outputs + ] + + # output folder + outdir = "outdir_ours" + if not os.path.exists(outdir): + os.makedirs(outdir) + + for predict, impath in zip(predicts, dst): + mask = utils.get_mask_pallete(predict, args.dataset) + outname = os.path.splitext(impath)[0] + ".png" + mask.save(os.path.join(outdir, outname)) + + if args.eval: + each_classes_iou = per_class_iou/cnt + print("pixAcc: %.4f, mIoU: %.4f" % (pixAcc, mIoU)) + print(each_classes_iou) + f.write("dataset {} ==> pixAcc: {:.4f}, mIoU: {:.4f}\n".format(args.dataset, pixAcc, mIoU)) + for per_iou in each_classes_iou: f.write('{:.4f}, '.format(per_iou)) + f.write('\n') + + +class ReturnFirstClosure(object): + def __init__(self, data): + self._data = data + + def __len__(self): + return len(self._data) + + def __getitem__(self, idx): + outputs = self._data[idx] + return outputs[0] + + +if __name__ == "__main__": + args = Options().parse() + torch.manual_seed(args.seed) + args.test_batch_size = torch.cuda.device_count() + test(args) diff --git a/submodules/lang_seg/test_lseg_zs.py b/submodules/lang_seg/test_lseg_zs.py new file mode 100644 index 0000000000000000000000000000000000000000..7899c7f7d394e510f489cdbe1ed50e697e4720ab --- /dev/null +++ b/submodules/lang_seg/test_lseg_zs.py @@ -0,0 +1,323 @@ +import os +import argparse +import numpy as np +from tqdm import tqdm +import torch +import torch.nn.functional as F +import torch.nn as nn +from modules.lseg_module_zs import LSegModuleZS +from additional_utils.models import LSeg_MultiEvalModule +from fewshot_data.common.logger import Logger, AverageMeter +from fewshot_data.common.vis import Visualizer +from fewshot_data.common.evaluation import Evaluator +from fewshot_data.common import utils +from fewshot_data.data.dataset import FSSDataset + + +class Options: + def __init__(self): + parser = argparse.ArgumentParser(description="PyTorch Segmentation") + # model and dataset + parser.add_argument( + "--model", type=str, default="encnet", help="model name (default: encnet)" + ) + parser.add_argument( + "--backbone", + type=str, + default="resnet50", + help="backbone name (default: resnet50)", + ) + parser.add_argument( + "--dataset", + type=str, + default="ade20k", + help="dataset name (default: pascal12)", + ) + parser.add_argument( + "--workers", type=int, default=16, metavar="N", help="dataloader threads" + ) + parser.add_argument( + "--base-size", type=int, default=520, help="base image size" + ) + parser.add_argument( + "--crop-size", type=int, default=480, help="crop image size" + ) + parser.add_argument( + "--train-split", + type=str, + default="train", + help="dataset train split (default: train)", + ) + # training hyper params + parser.add_argument( + "--aux", action="store_true", default=False, help="Auxilary Loss" + ) + parser.add_argument( + "--se-loss", + action="store_true", + default=False, + help="Semantic Encoding Loss SE-loss", + ) + parser.add_argument( + "--se-weight", type=float, default=0.2, help="SE-loss weight (default: 0.2)" + ) + parser.add_argument( + "--batch-size", + type=int, + default=16, + metavar="N", + help="input batch size for \ + training (default: auto)", + ) + parser.add_argument( + "--test-batch-size", + type=int, + default=16, + metavar="N", + help="input batch size for \ + testing (default: same as batch size)", + ) + # cuda, seed and logging + parser.add_argument( + "--no-cuda", + action="store_true", + default=False, + help="disables CUDA training", + ) + parser.add_argument( + "--seed", type=int, default=1, metavar="S", help="random seed (default: 1)" + ) + # checking point + parser.add_argument( + "--weights", type=str, default=None, help="checkpoint to test" + ) + # evaluation option + parser.add_argument( + "--eval", action="store_true", default=False, help="evaluating mIoU" + ) + + parser.add_argument( + "--acc-bn", + action="store_true", + default=False, + help="Re-accumulate BN statistics", + ) + parser.add_argument( + "--test-val", + action="store_true", + default=False, + help="generate masks on val set", + ) + parser.add_argument( + "--no-val", + action="store_true", + default=False, + help="skip validation during training", + ) + + parser.add_argument( + "--module", + default='', + help="select model definition", + ) + + # test option + parser.add_argument( + "--no-scaleinv", + dest="scale_inv", + default=True, + action="store_false", + help="turn off scaleinv layers", + ) + + parser.add_argument( + "--widehead", default=False, action="store_true", help="wider output head" + ) + + parser.add_argument( + "--widehead_hr", + default=False, + action="store_true", + help="wider output head", + ) + + parser.add_argument( + "--ignore_index", + type=int, + default=-1, + help="numeric value of ignore label in gt", + ) + + parser.add_argument( + "--jobname", + type=str, + default="default", + help="select which dataset", + ) + + parser.add_argument( + "--no-strict", + dest="strict", + default=True, + action="store_false", + help="no-strict copy the model", + ) + + parser.add_argument( + "--use_pretrained", + type=str, + default="True", + help="whether use the default model to intialize the model", + ) + + parser.add_argument( + "--arch_option", + type=int, + default=0, + help="which kind of architecture to be used", + ) + + # fewshot options + parser.add_argument( + '--nshot', + type=int, + default=1 + ) + parser.add_argument( + '--fold', + type=int, + default=0, + choices=[0, 1, 2, 3] + ) + parser.add_argument( + '--nworker', + type=int, + default=0 + ) + parser.add_argument( + '--bsz', + type=int, + default=1 + ) + parser.add_argument( + '--benchmark', + type=str, + default='pascal', + choices=['pascal', 'coco', 'fss', 'c2p'] + ) + parser.add_argument( + '--datapath', + type=str, + default='fewshot_data/Datasets_HSN' + ) + + parser.add_argument( + "--activation", + choices=['relu', 'lrelu', 'tanh'], + default="relu", + help="use which activation to activate the block", + ) + + + self.parser = parser + + def parse(self): + args = self.parser.parse_args() + args.cuda = not args.no_cuda and torch.cuda.is_available() + print(args) + return args + + +def test(args): + module_def = LSegModuleZS + + module = module_def.load_from_checkpoint( + checkpoint_path=args.weights, + data_path=args.datapath, + dataset=args.dataset, + backbone=args.backbone, + aux=args.aux, + num_features=256, + aux_weight=0, + se_loss=False, + se_weight=0, + base_lr=0, + batch_size=1, + max_epochs=0, + ignore_index=args.ignore_index, + dropout=0.0, + scale_inv=args.scale_inv, + augment=False, + no_batchnorm=False, + widehead=args.widehead, + widehead_hr=args.widehead_hr, + map_locatin="cpu", + arch_option=args.arch_option, + use_pretrained=args.use_pretrained, + strict=args.strict, + logpath='fewshot/logpath_4T/', + fold=args.fold, + block_depth=0, + nshot=args.nshot, + finetune_mode=False, + activation=args.activation, + ) + + Evaluator.initialize() + if args.backbone in ["clip_resnet101"]: + FSSDataset.initialize(img_size=480, datapath=args.datapath, use_original_imgsize=False, imagenet_norm=True) + else: + FSSDataset.initialize(img_size=480, datapath=args.datapath, use_original_imgsize=False) + # dataloader + args.benchmark = args.dataset + dataloader = FSSDataset.build_dataloader(args.benchmark, args.bsz, args.nworker, args.fold, 'test', args.nshot) + + model = module.net.eval().cuda() + # model = module.net.model.cpu() + + print(model) + + scales = ( + [0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.25] + if args.dataset == "citys" + else [0.5, 0.75, 1.0, 1.25, 1.5, 1.75] + ) + + f = open("logs/fewshot/log_fewshot-test_nshot{}_{}.txt".format(args.nshot, args.dataset), "a+") + + utils.fix_randseed(0) + average_meter = AverageMeter(dataloader.dataset) + for idx, batch in enumerate(dataloader): + batch = utils.to_cuda(batch) + image = batch['query_img'] + target = batch['query_mask'] + class_info = batch['class_id'] + # pred_mask = evaluator.parallel_forward(image, class_info) + pred_mask = model(image, class_info) + # assert pred_mask.argmax(dim=1).size() == batch['query_mask'].size() + # 2. Evaluate prediction + if args.benchmark == 'pascal' and batch['query_ignore_idx'] is not None: + query_ignore_idx = batch['query_ignore_idx'] + area_inter, area_union = Evaluator.classify_prediction(pred_mask.argmax(dim=1), target, query_ignore_idx) + else: + area_inter, area_union = Evaluator.classify_prediction(pred_mask.argmax(dim=1), target) + + average_meter.update(area_inter, area_union, class_info, loss=None) + average_meter.write_process(idx, len(dataloader), epoch=-1, write_batch_idx=1) + + # Write evaluation results + average_meter.write_result('Test', 0) + test_miou, test_fb_iou = average_meter.compute_iou() + + Logger.info('Fold %d, %d-shot ==> mIoU: %5.2f \t FB-IoU: %5.2f' % (args.fold, args.nshot, test_miou.item(), test_fb_iou.item())) + Logger.info('==================== Finished Testing ====================') + f.write('{}\n'.format(args.weights)) + f.write('Fold %d, %d-shot ==> mIoU: %5.2f \t FB-IoU: %5.2f\n' % (args.fold, args.nshot, test_miou.item(), test_fb_iou.item())) + f.close() + + + +if __name__ == "__main__": + args = Options().parse() + torch.manual_seed(args.seed) + test(args) diff --git a/submodules/lang_seg/train.sh b/submodules/lang_seg/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..9682ef8a532788edb287044ca45b9a2c1ab9b70c --- /dev/null +++ b/submodules/lang_seg/train.sh @@ -0,0 +1,6 @@ +#!/bin/bash +#python -u train_lseg.py --dataset ade20k --data_path ../datasets --batch_size 4 --exp_name lseg_ade20k_l16 \ +#--base_lr 0.004 --weight_decay 1e-4 --no-scaleinv --max_epochs 240 --widehead --accumulate_grad_batches 2 --backbone clip_vitl16_384 + +python -u train_lseg.py --dataset ade20k --data_path ../datasets --batch_size 1 --exp_name lseg_ade20k_l16 \ +--base_lr 0.004 --weight_decay 1e-4 --no-scaleinv --max_epochs 240 --widehead --accumulate_grad_batches 2 --backbone clip_vitl16_384 \ No newline at end of file diff --git a/submodules/lang_seg/train_lseg.py b/submodules/lang_seg/train_lseg.py new file mode 100644 index 0000000000000000000000000000000000000000..68716e952b7add3a774cdec5b230a1c3c85fca0c --- /dev/null +++ b/submodules/lang_seg/train_lseg.py @@ -0,0 +1,7 @@ +from modules.lseg_module import LSegModule +from utils import do_training, get_default_argument_parser + +if __name__ == "__main__": + parser = LSegModule.add_model_specific_args(get_default_argument_parser()) + args = parser.parse_args() + do_training(args, LSegModule) diff --git a/submodules/lang_seg/utils.py b/submodules/lang_seg/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..73a9fadbc0da35f5cc84ae219ae54346d385ad83 --- /dev/null +++ b/submodules/lang_seg/utils.py @@ -0,0 +1,368 @@ +import os +import pathlib + +from glob import glob + +from argparse import ArgumentParser +import torch +import pytorch_lightning as pl +import numpy as np +import cv2 +import random +import math +from torchvision import transforms + + +def do_training(hparams, model_constructor): + # instantiate model + model = model_constructor(**vars(hparams)) + # set all sorts of training parameters + hparams.gpus = -1 + hparams.accelerator = "ddp" + hparams.benchmark = True + + if hparams.dry_run: + print("Doing a dry run") + hparams.overfit_batches = hparams.batch_size + + if not hparams.no_resume: + hparams = set_resume_parameters(hparams) + + if not hasattr(hparams, "version") or hparams.version is None: + hparams.version = 0 + + hparams.sync_batchnorm = True + + ttlogger = pl.loggers.TestTubeLogger( + "checkpoints", name=hparams.exp_name, version=hparams.version + ) + + hparams.callbacks = make_checkpoint_callbacks(hparams.exp_name, hparams.version) + + wblogger = get_wandb_logger(hparams) + hparams.logger = [wblogger, ttlogger] + + trainer = pl.Trainer.from_argparse_args(hparams) + trainer.fit(model) + + +def get_default_argument_parser(): + parser = ArgumentParser(add_help=False) + parser.add_argument( + "--num_nodes", + type=int, + default=1, + help="number of nodes for distributed training", + ) + + parser.add_argument( + "--exp_name", type=str, required=True, help="name your experiment" + ) + + parser.add_argument( + "--dry-run", + action="store_true", + default=False, + help="run on batch of train/val/test", + ) + + parser.add_argument( + "--no_resume", + action="store_true", + default=False, + help="resume if we have a checkpoint", + ) + + parser.add_argument( + "--accumulate_grad_batches", + type=int, + default=1, + help="accumulate N batches for gradient computation", + ) + + parser.add_argument( + "--max_epochs", type=int, default=200, help="maximum number of epochs" + ) + + parser.add_argument( + "--project_name", type=str, default="lightseg", help="project name for logging" + ) + + return parser + + +def make_checkpoint_callbacks(exp_name, version, base_path="checkpoints", frequency=1): + version = 0 if version is None else version + + base_callback = pl.callbacks.ModelCheckpoint( + dirpath=f"{base_path}/{exp_name}/version_{version}/checkpoints/", + save_last=True, + verbose=True, + ) + + val_callback = pl.callbacks.ModelCheckpoint( + monitor="val_acc_epoch", + dirpath=f"{base_path}/{exp_name}/version_{version}/checkpoints/", + filename="result-{epoch}-{val_acc_epoch:.2f}", + mode="max", + save_top_k=3, + verbose=True, + ) + + return [base_callback, val_callback] + + +def get_latest_version(folder): + versions = [ + int(pathlib.PurePath(path).name.split("_")[-1]) + for path in glob(f"{folder}/version_*/") + ] + + if len(versions) == 0: + return None + + versions.sort() + return versions[-1] + + +def get_latest_checkpoint(exp_name, version): + while version > -1: + folder = f"./checkpoints/{exp_name}/version_{version}/checkpoints/" + + latest = f"{folder}/last.ckpt" + if os.path.exists(latest): + return latest, version + + chkpts = glob(f"{folder}/epoch=*.ckpt") + + if len(chkpts) > 0: + break + + version -= 1 + + if len(chkpts) == 0: + return None, None + + latest = max(chkpts, key=os.path.getctime) + + return latest, version + + +def set_resume_parameters(hparams): + version = get_latest_version(f"./checkpoints/{hparams.exp_name}") + + if version is not None: + latest, version = get_latest_checkpoint(hparams.exp_name, version) + print(f"Resuming checkpoint {latest}, exp_version={version}") + + hparams.resume_from_checkpoint = latest + hparams.version = version + + wandb_file = "checkpoints/{hparams.exp_name}/version_{version}/wandb_id" + if os.path.exists(wandb_file): + with open(wandb_file, "r") as f: + hparams.wandb_id = f.read() + else: + version = 0 + + return hparams + + +def get_wandb_logger(hparams): + exp_dir = f"checkpoints/{hparams.exp_name}/version_{hparams.version}/" + id_file = f"{exp_dir}/wandb_id" + + if os.path.exists(id_file): + with open(id_file) as f: + hparams.wandb_id = f.read() + else: + hparams.wandb_id = None + + logger = pl.loggers.WandbLogger( + save_dir="checkpoints", + project=hparams.project_name, + name=hparams.exp_name, + id=hparams.wandb_id, + ) + + if hparams.wandb_id is None: + _ = logger.experiment + + if not os.path.exists(exp_dir): + os.makedirs(exp_dir) + + with open(id_file, "w") as f: + f.write(logger.version) + + return logger + + +class Resize(object): + """Resize sample to given size (width, height).""" + + def __init__( + self, + width, + height, + resize_target=True, + keep_aspect_ratio=False, + ensure_multiple_of=1, + resize_method="lower_bound", + image_interpolation_method=cv2.INTER_AREA, + letter_box=False, + ): + """Init. + + Args: + width (int): desired output width + height (int): desired output height + resize_target (bool, optional): + True: Resize the full sample (image, mask, target). + False: Resize image only. + Defaults to True. + keep_aspect_ratio (bool, optional): + True: Keep the aspect ratio of the input sample. + Output sample might not have the given width and height, and + resize behaviour depends on the parameter 'resize_method'. + Defaults to False. + ensure_multiple_of (int, optional): + Output width and height is constrained to be multiple of this parameter. + Defaults to 1. + resize_method (str, optional): + "lower_bound": Output will be at least as large as the given size. + "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.) + "minimal": Scale as least as possible. (Output size might be smaller than given size.) + Defaults to "lower_bound". + """ + self.__width = width + self.__height = height + + self.__resize_target = resize_target + self.__keep_aspect_ratio = keep_aspect_ratio + self.__multiple_of = ensure_multiple_of + self.__resize_method = resize_method + self.__image_interpolation_method = image_interpolation_method + self.__letter_box = letter_box + + def constrain_to_multiple_of(self, x, min_val=0, max_val=None): + y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if max_val is not None and y > max_val: + y = (np.floor(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if y < min_val: + y = (np.ceil(x / self.__multiple_of) * self.__multiple_of).astype(int) + + return y + + def get_size(self, width, height): + # determine new height and width + scale_height = self.__height / height + scale_width = self.__width / width + + if self.__keep_aspect_ratio: + if self.__resize_method == "lower_bound": + # scale such that output size is lower bound + if scale_width > scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "upper_bound": + # scale such that output size is upper bound + if scale_width < scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "minimal": + # scale as least as possbile + if abs(1 - scale_width) < abs(1 - scale_height): + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + else: + raise ValueError( + f"resize_method {self.__resize_method} not implemented" + ) + + if self.__resize_method == "lower_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, min_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, min_val=self.__width + ) + elif self.__resize_method == "upper_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, max_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, max_val=self.__width + ) + elif self.__resize_method == "minimal": + new_height = self.constrain_to_multiple_of(scale_height * height) + new_width = self.constrain_to_multiple_of(scale_width * width) + else: + raise ValueError(f"resize_method {self.__resize_method} not implemented") + + return (new_width, new_height) + + def make_letter_box(self, sample): + top = bottom = (self.__height - sample.shape[0]) // 2 + left = right = (self.__width - sample.shape[1]) // 2 + sample = cv2.copyMakeBorder( + sample, top, bottom, left, right, cv2.BORDER_CONSTANT, None, 0 + ) + return sample + + def __call__(self, sample): + width, height = self.get_size( + sample["image"].shape[1], sample["image"].shape[0] + ) + + # resize sample + sample["image"] = cv2.resize( + sample["image"], + (width, height), + interpolation=self.__image_interpolation_method, + ) + + if self.__letter_box: + sample["image"] = self.make_letter_box(sample["image"]) + + if self.__resize_target: + if "disparity" in sample: + sample["disparity"] = cv2.resize( + sample["disparity"], + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + + if self.__letter_box: + sample["disparity"] = self.make_letter_box(sample["disparity"]) + + if "depth" in sample: + sample["depth"] = cv2.resize( + sample["depth"], (width, height), interpolation=cv2.INTER_NEAREST + ) + + if self.__letter_box: + sample["depth"] = self.make_letter_box(sample["depth"]) + + sample["mask"] = cv2.resize( + sample["mask"].astype(np.float32), + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + + if self.__letter_box: + sample["mask"] = self.make_letter_box(sample["mask"]) + + sample["mask"] = sample["mask"].astype(bool) + + return sample diff --git a/submodules/mast3r/.gitignore b/submodules/mast3r/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..f6afc9205668539e520cec3adb847e325cd016d6 --- /dev/null +++ b/submodules/mast3r/.gitignore @@ -0,0 +1,131 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +checkpoints/ \ No newline at end of file diff --git a/submodules/mast3r/.gitmodules b/submodules/mast3r/.gitmodules new file mode 100644 index 0000000000000000000000000000000000000000..d544dca780e738ef30f618ce3cab4810bad806e3 --- /dev/null +++ b/submodules/mast3r/.gitmodules @@ -0,0 +1,4 @@ +[submodule "dust3r"] + path = dust3r + url = https://github.com/naver/dust3r + branch = cvpr diff --git a/submodules/mast3r/CHECKPOINTS_NOTICE b/submodules/mast3r/CHECKPOINTS_NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..040aed77ec78156cb6c1af8da1652e13ade10bcd --- /dev/null +++ b/submodules/mast3r/CHECKPOINTS_NOTICE @@ -0,0 +1,1376 @@ +MASt3R +Copyright 2024-present NAVER Corp. + +This project's checkpoints were trained on datasets with separate license terms. +Your use of theses checkpoints is subject to the terms and conditions of the following licenses. + +=== +pretrained model: +DUSt3R: DUSt3R_ViTLarge_BaseDecoder_512_dpt +https://github.com/naver/dust3r + +In particular, from the croco training set: + +3D_Street_View +https://github.com/amir32002/3D_Street_View/blob/master/LICENSE +This dataset is made freely available to academic and non-academic entities for non-commercial purposes such as academic research, teaching, scientific publications, or personal experimentation. Permission is granted to use the data given that you agree: + +1. That the dataset comes "AS IS", without express or implied warranty. Although every effort has been made to ensure accuracy, we do not accept any responsibility for errors or omissions. + +2. That you include a reference to the Dataset in any work that makes use of the dataset. For research papers, cite our publication as listed on our website. + +3. That you do not distribute this dataset or modified versions. It is permissible to distribute derivative works in as far as they are abstract representations of this dataset (such as models trained on it or additional annotations that do not directly include any of our data) and do not allow to recover the dataset or something similar in character. + +4. That you may not use the dataset or any derivative work for commercial purposes as, for example, licensing or selling the data, or using the data with a purpose to procure a commercial gain. +That all rights not expressly granted to you are reserved by us. + +In addition, using the dataset is subject to the following standard terms: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +Indoor Visual Localization datasets (IndoorVL) +https://challenge.naverlabs.com/kapture/GangnamStation_LICENSE.txt +https://challenge.naverlabs.com/kapture/HyundaiDepartmentStore_LICENSE.txt + +LICENSE.txt +Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 (modified ver.) +International Public License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution-NonCommercial-NoDerivatives 4.0 International Public +License ("Public License"). To the extent this Public License may be +interpreted as a contract, You are granted the Licensed Rights in +consideration of Your acceptance of these terms and conditions, and the +Licensor grants You such rights in consideration of benefits the +Licensor receives from making the Licensed Material available under +these terms and conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + c. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + d. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + e. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + f. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + g. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + h. NonCommercial means not primarily intended for or directed towards + commercial advantage or monetary compensation. For purposes of + this Public License, the exchange of the Licensed Material for + other material subject to Copyright and Similar Rights by digital + file-sharing or similar means is NonCommercial provided there is + no payment of monetary compensation in connection with the + exchange. + + i. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + j. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + k. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + l. Research purpose means to publish research achievements in a research paper + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part, for NonCommercial purposes only; and + + b. produce and reproduce, but not Share, Adapted Material + for NonCommercial purposes only. + + c. reproduce and share the Adapted Matrerial, in part, + for Research purposes only. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties, including when + the Licensed Material is used other than for NonCommercial + purposes. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material(including in a research paper), + You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + For the avoidance of doubt, You do not have permission under + this Public License to Share Adapted Material. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database for NonCommercial purposes + only and provided You do not Share Adapted Material; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + +=== +CO3Dv2 + +Creative Commons Attribution-NonCommercial 4.0 International Public +License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution-NonCommercial 4.0 International Public License ("Public +License"). To the extent this Public License may be interpreted as a +contract, You are granted the Licensed Rights in consideration of Your +acceptance of these terms and conditions, and the Licensor grants You +such rights in consideration of benefits the Licensor receives from +making the Licensed Material available under these terms and +conditions. + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + i. NonCommercial means not primarily intended for or directed towards + commercial advantage or monetary compensation. For purposes of + this Public License, the exchange of the Licensed Material for + other material subject to Copyright and Similar Rights by digital + file-sharing or similar means is NonCommercial provided there is + no payment of monetary compensation in connection with the + exchange. + + j. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + k. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + l. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part, for NonCommercial purposes only; and + + b. produce, reproduce, and Share Adapted Material for + NonCommercial purposes only. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties, including when + the Licensed Material is used other than for NonCommercial + purposes. + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database for NonCommercial purposes + only; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + +=== +ARKitScenes +Creative Commons Attribution-NonCommercial-ShareAlike 4.0: https://creativecommons.org/licenses/by-nc-sa/4.0/ + +=== +ScanNet++ +https://kaldir.vc.in.tum.de/scannetpp/static/scannetpp-terms-of-use.pdf + +=== +BlendedMVS +Creative Commons Attribution 4.0 International: http://creativecommons.org/licenses/by/4.0/ + +=== +Habitat-Sim +HM3D +https://matterport.com/fr/legal/matterport-end-user-license-agreement-academic-use-model-data + +ScanNet +https://kaldir.vc.in.tum.de/scannet/ScanNet_TOS.pdf + +Replica +Before Facebook Technologies, LLC (“FB”) is able to offer you (“Researcher” or +“You”) access to the Replica Dataset (the “Dataset”), please read the following +agreement (“Agreement”). + +By accessing, and in exchange for receiving permission to access, the Dataset, +Researcher hereby agrees to the following terms and conditions: +1. Researcher may use, modify, improve and/or publish the Dataset only in +connection with a research or educational purpose that is non-commercial or +not-for-profit in nature, and not for any other purpose. +1. Researcher may provide research associates and colleagues with access to the +Dataset provided that they first agree to be bound by these terms and +conditions. +1. Researcher may use the Dataset in the scope of their employment at a +for-profit or commercial entity provided that Researcher complies with Section 1 +of this Agreement. If Researcher is employed by a for-profit or commercial +entity, Researcher's employer shall also be bound by these terms and conditions, +and Researcher hereby represents that they are fully authorized to enter into +this agreement on behalf of such employer. +1. THE DATASET IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL FB OR ANY +CONTRIBUTOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE DATASET OR THE USE OR OTHER DEALINGS IN THE DATASET. +1. The law of the State of California shall apply to all disputes related to +this Dataset. + +ReplicaCAD +Creative Commons Attribution 4.0 International (CC BY 4.0): https://creativecommons.org/licenses/by/4.0/ + +habitat-sim +MIT License + +Copyright (c) Meta Platforms, Inc. and its affiliates. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +=== +MegaDepth +MIT License + +Copyright (c) 2018 Zhengqi Li + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=== +StaticThings3D + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +=== +WildRGB-D +https://github.com/wildrgbd/wildrgbd/ +MIT License + +Copyright (c) 2024 rowdataset + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=== +TartanAir +Creative Commons Attribution 4.0 International License: http://creativecommons.org/licenses/by/4.0/ + +=== +UnrealStereo4K +https://github.com/fabiotosi92/SMD-Nets +MIT License + +Copyright (c) 2021 Fabio Tosi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=== +Virtual KITTI 2 +Creative Commons Attribution-NonCommercial-ShareAlike 3.0: http://creativecommons.org/licenses/by-nc-sa/3.0/legalcode + +=== +DL3DV +DL3DV-10K Term of use and Creative Commons Attribution-NonCommercial 4.0 International License. + +Terms of Use + +Researcher shall use the Dataset only for non-commercial research and educational purposes. +DL3DV-10K organization makes no representations or warranties regarding the dataset, including but not limited to warranties of non-infringement or fitness for a particular purpose. +Researcher accepts full responsibility for his/her/their use of the Dataset and shall defend and indemnify DL3DV-10K organization, including its members, employees, Trustees, officers and agents, against any and all claims arising from Researcher's use of the Dataset, including but not limited to Researcher's use of any copies of copyrighted 3D models that he/she/they may create from the dataset. +Researcher may provide research associates and colleagues with access to the Dataset, after receiving entity has also agreed to and signed these terms and conditions. Sharing the data otherwise is strictly prohibited. +Following General Data Protection Regulation, Researcher must ensure that they can delete all person-specific data upon request. +DL3DV-10K organization reserves the right to terminate Researcher's access to the Dataset at any time. +If Researcher is employed by a for-profit, commercial entity, Researcher's employer shall also be bound by these terms and conditions, and Researcher hereby represents that he/she/they is/are fully authorized to enter into this agreement on behalf of such employer. +The law of the Indiana State shall apply to all disputes under this agreement. + +Creative Commons Attribution-NonCommercial 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-NonCommercial 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. + +Section 1 -- Definitions. + +a. Adapted Material means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. + +b. Adapter's License means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. + +c. Copyright and Similar Rights means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. d. Effective Technological Measures means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. + +e. Exceptions and Limitations means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. + +f. Licensed Material means the artistic or literary work, database, or other material to which the Licensor applied this Public License. + +g. Licensed Rights means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. + +h. Licensor means the individual(s) or entity(ies) granting rights under this Public License. + +i. NonCommercial means not primarily intended for or directed towards commercial advantage or monetary compensation. For purposes of this Public License, the exchange of the Licensed Material for other material subject to Copyright and Similar Rights by digital file-sharing or similar means is NonCommercial provided there is no payment of monetary compensation in connection with the exchange. + +j. Share means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. + +k. Sui Generis Database Rights means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. + +l. You means the individual or entity exercising the Licensed Rights under this Public License. Your has a corresponding meaning. + +Section 2 -- Scope. + +a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part, for NonCommercial purposes only; and + + b. produce, reproduce, and Share Adapted Material for + NonCommercial purposes only. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + +b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties, including when + the Licensed Material is used other than for NonCommercial + purposes. + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the following conditions. + +a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: + +a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database for NonCommercial purposes only; + +b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material; and + +c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + +a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + +b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + +c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. + +Section 6 -- Term and Termination. + +a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. + +b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + +c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. + +d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. + +Section 7 -- Other Terms and Conditions. + +a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. + +b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. + +Section 8 -- Interpretation. + +a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. + +b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. + +c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. + +d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. + +=== +Niantic Map Free Relocalization Dataset License Agreement +This Niantic Map Free Relocalization Dataset License Agreement ("Agreement") is an agreement between you and Niantic, Inc. (“Niantic” or “we”). By downloading or otherwise using Niantic’s Map-Free Relocalization dataset or dataset-derived materials (collectively, the "Dataset") you agree to: + +1. Purpose and Restrictions. You may only use the Dataset only for non-commercial purposes, such as academic research at educational and not-for-profit research institutions, teaching, public demonstrations, and personal experimentation. Non-commercial use expressly excludes any profit-making or commercial activities, including without limitation sale, license, manufacture or development of commercial products, use in commercially-sponsored research, use at a laboratory or other facility owned or controlled (whether in whole or in part) by a commercial entity, provision of consulting service, use for or on behalf of any commercial entity, and use in consulting service, use for or on behalf of any commercial entity, use in research where a commercial party obtains rights to research results or any other benefit. Notwithstanding the foregoing restrictions, you can use this Dataset for publishing comparison results for academic papers, including retraining your models on this Dataset. + +2. License. Subject to this Agreement, Niantic grants you a non-exclusive, non-transferable, non-sublicensable right to download and use the Dataset for the purpose stated in Section 1 of this Agreement. All rights not expressly granted to you in this Agreement are reserved. + +3. Condition of Use. You must not use the Dataset in a way that could diminish, tarnish, or in any way harm Niantic’s reputation or image. + +4. No Warranties. The Dataset comes “as is”, and you will use it at your own risk. Niantic makes no representations or warranties regarding the Dataset, including but not limited to warranties of non-infringement or fitness for a particular purpose. Neither Niantic nor any contributor to the Dataset will be liable for any damages related to the Dataset or this Agreement, including direct, indirect, special, consequential or incidental damages, to the maximum extent the law permits, no matter what legal theory they are based on. We are not obligated to (and will not) provide technical support for the Dataset. + +5. Indemnity. You accept full responsibility for your use of the Dataset and shall defend and indemnify Niantic, including its employees, officers and agents, against any and all claims arising from your use of the Dataset. + +6. Removal. Niantic reserves the right to remove access to the Dataset at any time without cause. If you have downloaded a copy of the Dataset prior to such removal, you may use such a copy subject to this Agreement, but you may not distribute your copy. + +7. Termination. This Agreement will terminate immediately upon your commercial use of the Dataset. + +8. Authorized Representative. If you are employed by a for-profit, commercial entity, your employer shall also be bound by the terms and conditions of this Agreement, and you hereby represent that you are fully authorized to enter into this Agreement on behalf of such employer. + +9. Survivability. Sections 2, 4, 5, 6, 7, 8, 9, and 10 of this Agreement survive the termination of this Agreement. + +10. Misc. This Agreement is governed and construed in all respects in accordance with the laws of the State of California, USA without regard to conflicts of law. If any provision of this Agreement is deemed unenforceable or contrary to law, the rest of this Agreement shall remain in full effect and enforceable. If you do not agree to this Agreement, do not download or use the Dataset. The Dataset is protected by copyright and other intellectual property laws and is licensed, not sold. + +=== +NVIDIA Source Code License for SegFormer + +1. Definitions + +“Licensor” means any person or entity that distributes its Work. + +“Software” means the original work of authorship made available under this License. + +“Work” means the Software and any additions to or derivative works of the Software that are made available under +this License. + +The terms “reproduce,” “reproduction,” “derivative works,” and “distribution” have the meaning as provided under +U.S. copyright law; provided, however, that for the purposes of this License, derivative works shall not include +works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work. + +Works, including the Software, are “made available” under this License by including in or with the Work either +(a) a copyright notice referencing the applicability of this License to the Work, or (b) a copy of this License. + +2. License Grant + +2.1 Copyright Grant. Subject to the terms and conditions of this License, each Licensor grants to you a perpetual, +worldwide, non-exclusive, royalty-free, copyright license to reproduce, prepare derivative works of, publicly +display, publicly perform, sublicense and distribute its Work and any resulting derivative works in any form. + +3. Limitations + +3.1 Redistribution. You may reproduce or distribute the Work only if (a) you do so under this License, (b) you +include a complete copy of this License with your distribution, and (c) you retain without modification any +copyright, patent, trademark, or attribution notices that are present in the Work. + +3.2 Derivative Works. You may specify that additional or different terms apply to the use, reproduction, and +distribution of your derivative works of the Work (“Your Terms”) only if (a) Your Terms provide that the use +limitation in Section 3.3 applies to your derivative works, and (b) you identify the specific derivative works +that are subject to Your Terms. Notwithstanding Your Terms, this License (including the redistribution +requirements in Section 3.1) will continue to apply to the Work itself. + +3.3 Use Limitation. The Work and any derivative works thereof only may be used or intended for use +non-commercially. Notwithstanding the foregoing, NVIDIA and its affiliates may use the Work and any derivative +works commercially. As used herein, “non-commercially” means for research or evaluation purposes only. + +3.4 Patent Claims. If you bring or threaten to bring a patent claim against any Licensor (including any claim, +cross-claim or counterclaim in a lawsuit) to enforce any patents that you allege are infringed by any Work, then +your rights under this License from such Licensor (including the grant in Section 2.1) will terminate immediately. + +3.5 Trademarks. This License does not grant any rights to use any Licensor’s or its affiliates’ names, logos, +or trademarks, except as necessary to reproduce the notices described in this License. + +3.6 Termination. If you violate any term of this License, then your rights under this License (including the +grant in Section 2.1) will terminate immediately. + +4. Disclaimer of Warranty. + +THE WORK IS PROVIDED “AS IS” WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING +WARRANTIES OR CONDITIONS OF M ERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON-INFRINGEMENT. YOU +BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER THIS LICENSE. + +5. Limitation of Liability. + +EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER IN TORT (INCLUDING +NEGLIGENCE), CONTRACT, OR OTHERWISE SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, +INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF OR RELATED TO THIS LICENSE, THE USE OR +INABILITY TO USE THE WORK (INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, LOST PROFITS OR +DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER COMM ERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN +ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +=== +CosXL License Agreement + + +STABILITY AI NON-COMMERCIAL RESEARCH COMMUNITY LICENSE AGREEMENT Dated: April 7th, 2024 +By clicking “I Accept” below or by using or distributing any portion or element of the Models, Software, Software Products or Derivative Works, you agree to the terms of this License. If you do not agree to this License, then you do not have any rights to use the Software Products or Derivative Works through this License, and you must immediately cease using the Software Products or Derivative Works. If you are agreeing to be bound by the terms of this License on behalf of your employer or other entity, you represent and warrant to Stability AI that you have full legal authority to bind your employer or such entity to this License. If you do not have the requisite authority, you may not accept the License or access the Software Products or Derivative Works on behalf of your employer or other entity. +"Agreement" means this Stable Non-Commercial Research Community License Agreement. +“AUP” means the Stability AI Acceptable Use Policy available at https://stability.ai/use-policy, as may be updated from time to time. +"Derivative Work(s)” means (a) any derivative work of the Software Products as recognized by U.S. copyright laws and (b) any modifications to a Model, and any other model created which is based on or derived from the Model or the Model’s output. For clarity, Derivative Works do not include the output of any Model. +“Documentation” means any specifications, manuals, documentation, and other written information provided by Stability AI related to the Software. +"Licensee" or "you" means you, or your employer or any other person or entity (if you are entering into this Agreement on such person or entity's behalf), of the age required under applicable laws, rules or regulations to provide legal consent and that has legal authority to bind your employer or such other person or entity if you are entering in this Agreement on their behalf. +“Model(s)" means, collectively, Stability AI’s proprietary models and algorithms, including machine-learning models, trained model weights and other elements of the foregoing, made available under this Agreement. +“Non-Commercial Uses” means exercising any of the rights granted herein for the purpose of research or non-commercial purposes. Non-Commercial Uses does not include any production use of the Software Products or any Derivative Works. +"Stability AI" or "we" means Stability AI Ltd. and its affiliates. + +"Software" means Stability AI’s proprietary software made available under this Agreement. +“Software Products” means the Models, Software and Documentation, individually or in any combination. + + License Rights and Redistribution. + a. Subject to your compliance with this Agreement, the AUP (which is hereby incorporated herein by reference), and the Documentation, Stability AI grants you a non-exclusive, worldwide, non-transferable, non-sublicensable, revocable, royalty free and limited license under Stability AI’s intellectual property or other rights owned or controlled by Stability AI embodied in the Software Products to use, reproduce, distribute, and create Derivative Works of, the Software Products, in each case for Non-Commercial Uses only. + b. You may not use the Software Products or Derivative Works to enable third parties to use the Software Products or Derivative Works as part of your hosted service or via your APIs, whether you are adding substantial additional functionality thereto or not. Merely distributing the Software Products or Derivative Works for download online without offering any related service (ex. by distributing the Models on HuggingFace) is not a violation of this subsection. If you wish to use the Software Products or any Derivative Works for commercial or production use or you wish to make the Software Products or any Derivative Works available to third parties via your hosted service or your APIs, contact Stability AI at https://stability.ai/contact. + c. If you distribute or make the Software Products, or any Derivative Works thereof, available to a third party, the Software Products, Derivative Works, or any portion thereof, respectively, will remain subject to this Agreement and you must (i) provide a copy of this Agreement to such third party, and (ii) retain the following attribution notice within a "Notice" text file distributed as a part of such copies: "This Stability AI Model is licensed under the Stability AI Non-Commercial Research Community License, Copyright (c) Stability AI Ltd. All Rights Reserved.” If you create a Derivative Work of a Software Product, you may add your own attribution notices to the Notice file included with the Software Product, provided that you clearly indicate which attributions apply to the Software Product and you must state in the NOTICE file that you changed the Software Product and how it was modified. + Disclaimer of Warranty. UNLESS REQUIRED BY APPLICABLE LAW, THE SOFTWARE PRODUCTS AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING THE SOFTWARE PRODUCTS, DERIVATIVE WORKS OR ANY OUTPUT OR RESULTS AND ASSUME ANY RISKS ASSOCIATED WITH YOUR USE OF THE SOFTWARE PRODUCTS, DERIVATIVE WORKS AND ANY OUTPUT AND RESULTS. 3. Limitation of Liability. IN NO EVENT WILL STABILITY AI OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, ARISING OUT OF THIS AGREEMENT, FOR ANY LOST PROFITS OR ANY DIRECT, INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN IF STABILITY AI OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF ANY OF THE FOREGOING. 4. Intellectual Property. + a. No trademark licenses are granted under this Agreement, and in connection with the Software Products or Derivative Works, neither Stability AI nor Licensee may use any name or mark owned by or associated with the other or any of its affiliates, except as required for reasonable and customary use in describing and redistributing the Software Products or Derivative Works. + b. Subject to Stability AI’s ownership of the Software Products and Derivative Works made by or for Stability AI, with respect to any Derivative Works that are made by you, as between you and Stability AI, you are and will be the owner of such Derivative Works + c. If you institute litigation or other proceedings against Stability AI (including a cross-claim or counterclaim in a lawsuit) alleging that the Software Products, Derivative Works or associated outputs or results, or any portion of any of the foregoing, constitutes infringement of intellectual property or other rights owned or licensable by you, then any licenses granted to you under this Agreement shall terminate as of the date such litigation or claim is filed or instituted. You will indemnify and hold harmless Stability AI from and against any claim by any third party arising out of or related to your use or distribution of the Software Products or Derivative Works in violation of this Agreement. + Term and Termination. The term of this Agreement will commence upon your acceptance of this Agreement or access to the Software Products and will continue in full force and effect until terminated in accordance with the terms and conditions herein. Stability AI may terminate this Agreement if you are in breach of any term or condition of this Agreement. Upon termination of this Agreement, you shall delete and cease use of any Software Products or Derivative Works. Sections 2-4 shall survive the termination of this Agreement. + Governing Law. This Agreement will be governed by and construed in accordance with the laws of the United States and the State of California without regard to choice of law + principles. + diff --git a/submodules/mast3r/LICENSE b/submodules/mast3r/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..a97986e3a8ddd49973959f6c748dfa8b881b64d3 --- /dev/null +++ b/submodules/mast3r/LICENSE @@ -0,0 +1,7 @@ +DUSt3R, Copyright (c) 2024-present Naver Corporation, is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 license. + +A summary of the CC BY-NC-SA 4.0 license is located here: + https://creativecommons.org/licenses/by-nc-sa/4.0/ + +The CC BY-NC-SA 4.0 license is located here: + https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode diff --git a/submodules/mast3r/NOTICE b/submodules/mast3r/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..86583416b75cc1749cac38d437b376842975ca06 --- /dev/null +++ b/submodules/mast3r/NOTICE @@ -0,0 +1,103 @@ +MASt3R +Copyright 2024-present NAVER Corp. + +This project contains subcomponents with separate copyright notices and license terms. +Your use of the source code for these subcomponents is subject to the terms and conditions of the following licenses. + +==== + +naver/dust3r +https://github.com/naver/dust3r/ + +Creative Commons Attribution-NonCommercial-ShareAlike 4.0 + +==== + +naver/croco +https://github.com/naver/croco/ + +Creative Commons Attribution-NonCommercial-ShareAlike 4.0 + +==== + +pytorch/pytorch +https://github.com/pytorch/pytorch + +From PyTorch: + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +From Caffe2: + +Copyright (c) 2016-present, Facebook Inc. All rights reserved. + +All contributions by Facebook: +Copyright (c) 2016 Facebook Inc. + +All contributions by Google: +Copyright (c) 2015 Google Inc. +All rights reserved. + +All contributions by Yangqing Jia: +Copyright (c) 2015 Yangqing Jia +All rights reserved. + +All contributions by Kakao Brain: +Copyright 2019-2020 Kakao Brain + +All contributions by Cruise LLC: +Copyright (c) 2022 Cruise LLC. +All rights reserved. + +All contributions from Caffe: +Copyright(c) 2013, 2014, 2015, the respective contributors +All rights reserved. + +All other contributions: +Copyright(c) 2015, 2016 the respective contributors +All rights reserved. + +Caffe2 uses a copyright model similar to Caffe: each contributor holds +copyright over their contributions to Caffe2. The project versioning records +all such contribution and copyright details. If a contributor wants to further +mark their specific copyright on a particular contribution, they should +indicate their copyright solely in the commit message of the change when it is +committed. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America + and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + diff --git a/submodules/mast3r/README.md b/submodules/mast3r/README.md new file mode 100644 index 0000000000000000000000000000000000000000..32ddce4b5f3071633f3581ab6c29696b758c1524 --- /dev/null +++ b/submodules/mast3r/README.md @@ -0,0 +1,343 @@ +![banner](assets/mast3r.jpg) + +Official implementation of `Grounding Image Matching in 3D with MASt3R` +[[Project page](https://europe.naverlabs.com/blog/mast3r-matching-and-stereo-3d-reconstruction/)], [[MASt3R arxiv](https://arxiv.org/abs/2406.09756)], [[DUSt3R arxiv](https://arxiv.org/abs/2312.14132)] + +![Example of matching results obtained from MASt3R](assets/examples.jpg) + +![High level overview of MASt3R's architecture](assets/mast3r_archi.jpg) + +```bibtex +@misc{mast3r_arxiv24, + title={Grounding Image Matching in 3D with MASt3R}, + author={Vincent Leroy and Yohann Cabon and Jerome Revaud}, + year={2024}, + eprint={2406.09756}, + archivePrefix={arXiv}, + primaryClass={cs.CV} +} + +@inproceedings{dust3r_cvpr24, + title={DUSt3R: Geometric 3D Vision Made Easy}, + author={Shuzhe Wang and Vincent Leroy and Yohann Cabon and Boris Chidlovskii and Jerome Revaud}, + booktitle = {CVPR}, + year = {2024} +} +``` + +## Table of Contents + +- [Table of Contents](#table-of-contents) +- [License](#license) +- [Get Started](#get-started) + - [Installation](#installation) + - [Checkpoints](#checkpoints) + - [Interactive demo](#interactive-demo) + - [Interactive demo with docker](#interactive-demo-with-docker) +- [Usage](#usage) +- [Training](#training) + - [Datasets](#datasets) + - [Demo](#demo) + - [Our Hyperparameters](#our-hyperparameters) +- [Visual Localization](#visual-localization) + - [Dataset Preparation](#dataset-preparation) + - [Example Commands](#example-commands) + +## License + +The code is distributed under the CC BY-NC-SA 4.0 License. +See [LICENSE](LICENSE) for more information. + +```python +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +``` + +## Get Started + +### Installation + +1. Clone MASt3R. +```bash +git clone --recursive https://github.com/naver/mast3r +cd mast3r +# if you have already cloned mast3r: +# git submodule update --init --recursive +``` + +2. Create the environment, here we show an example using conda. +```bash +conda create -n mast3r python=3.11 cmake=3.14.0 +conda activate mast3r +conda install pytorch torchvision pytorch-cuda=12.1 -c pytorch -c nvidia # use the correct version of cuda for your system +pip install -r requirements.txt +pip install -r dust3r/requirements.txt +# Optional: you can also install additional packages to: +# - add support for HEIC images +# - add required packages for visloc.py +pip install -r dust3r/requirements_optional.txt +``` + +3. Optional, compile the cuda kernels for RoPE (as in CroCo v2). +```bash +# DUST3R relies on RoPE positional embeddings for which you can compile some cuda kernels for faster runtime. +cd dust3r/croco/models/curope/ +python setup.py build_ext --inplace +cd ../../../../ +``` + + +### Checkpoints + +You can obtain the checkpoints by two ways: + +1) You can use our huggingface_hub integration: the models will be downloaded automatically. + +2) Otherwise, We provide several pre-trained models: + +| Modelname | Training resolutions | Head | Encoder | Decoder | +|-------------|----------------------|------|---------|---------| +| [`MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric`](https://download.europe.naverlabs.com/ComputerVision/MASt3R/MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric.pth) | 512x384, 512x336, 512x288, 512x256, 512x160 | CatMLP+DPT | ViT-L | ViT-B | + +You can check the hyperparameters we used to train these models in the [section: Our Hyperparameters](#our-hyperparameters) +Make sure to check license of the datasets we used. + +To download a specific model, for example `MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric.pth`: +```bash +mkdir -p checkpoints/ +wget https://download.europe.naverlabs.com/ComputerVision/MASt3R/MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric.pth -P checkpoints/ +``` + +For these checkpoints, make sure to agree to the license of all the training datasets we used, in addition to CC-BY-NC-SA 4.0. +The mapfree dataset license in particular is very restrictive. For more information, check [CHECKPOINTS_NOTICE](CHECKPOINTS_NOTICE). + + +### Interactive demo + +We made one huggingface space running the new sparse global alignment in a simplified demo for small scenes: [naver/MASt3R](https://huggingface.co/spaces/naver/MASt3R) +There are two demos available to run locally: + +``` +demo.py is the updated demo for MASt3R. It uses our new sparse global alignment method that allows you to reconstruct larger scenes + +python3 demo.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric + +# Use --weights to load a checkpoint from a local file, eg --weights checkpoints/MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric.pth +# Use --local_network to make it accessible on the local network, or --server_name to specify the url manually +# Use --server_port to change the port, by default it will search for an available port starting at 7860 +# Use --device to use a different device, by default it's "cuda" + +demo_dust3r_ga.py is the same demo as in dust3r (+ compatibility for MASt3R models) +see https://github.com/naver/dust3r?tab=readme-ov-file#interactive-demo for details +``` + +### Interactive demo with docker + +To run MASt3R using Docker, including with NVIDIA CUDA support, follow these instructions: + +1. **Install Docker**: If not already installed, download and install `docker` and `docker compose` from the [Docker website](https://www.docker.com/get-started). + +2. **Install NVIDIA Docker Toolkit**: For GPU support, install the NVIDIA Docker toolkit from the [Nvidia website](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). + +3. **Build the Docker image and run it**: `cd` into the `./docker` directory and run the following commands: + +```bash +cd docker +bash run.sh --with-cuda --model_name="MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric" +``` + +Or if you want to run the demo without CUDA support, run the following command: + +```bash +cd docker +bash run.sh --model_name="MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric" +``` + +By default, `demo.py` is launched with the option `--local_network`. +Visit `http://localhost:7860/` to access the web UI (or replace `localhost` with the machine's name to access it from the network). + +`run.sh` will launch docker-compose using either the [docker-compose-cuda.yml](docker/docker-compose-cuda.yml) or [docker-compose-cpu.ym](docker/docker-compose-cpu.yml) config file, then it starts the demo using [entrypoint.sh](docker/files/entrypoint.sh). + +___ + +![demo](assets/demo.jpg) + +## Usage + +```python +from mast3r.model import AsymmetricMASt3R +from mast3r.fast_nn import fast_reciprocal_NNs + +import mast3r.utils.path_to_dust3r +from dust3r.inference import inference +from dust3r.utils.image import load_images + +if __name__ == '__main__': + device = 'cuda' + schedule = 'cosine' + lr = 0.01 + niter = 300 + + model_name = "naver/MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric" + # you can put the path to a local checkpoint in model_name if needed + model = AsymmetricMASt3R.from_pretrained(model_name).to(device) + images = load_images(['dust3r/croco/assets/Chateau1.png', 'dust3r/croco/assets/Chateau2.png'], size=512) + output = inference([tuple(images)], model, device, batch_size=1, verbose=False) + + # at this stage, you have the raw dust3r predictions + view1, pred1 = output['view1'], output['pred1'] + view2, pred2 = output['view2'], output['pred2'] + + desc1, desc2 = pred1['desc'].squeeze(0).detach(), pred2['desc'].squeeze(0).detach() + + # find 2D-2D matches between the two images + matches_im0, matches_im1 = fast_reciprocal_NNs(desc1, desc2, subsample_or_initxy1=8, + device=device, dist='dot', block_size=2**13) + + # ignore small border around the edge + H0, W0 = view1['true_shape'][0] + valid_matches_im0 = (matches_im0[:, 0] >= 3) & (matches_im0[:, 0] < int(W0) - 3) & ( + matches_im0[:, 1] >= 3) & (matches_im0[:, 1] < int(H0) - 3) + + H1, W1 = view2['true_shape'][0] + valid_matches_im1 = (matches_im1[:, 0] >= 3) & (matches_im1[:, 0] < int(W1) - 3) & ( + matches_im1[:, 1] >= 3) & (matches_im1[:, 1] < int(H1) - 3) + + valid_matches = valid_matches_im0 & valid_matches_im1 + matches_im0, matches_im1 = matches_im0[valid_matches], matches_im1[valid_matches] + + # visualize a few matches + import numpy as np + import torch + import torchvision.transforms.functional + from matplotlib import pyplot as pl + + n_viz = 20 + num_matches = matches_im0.shape[0] + match_idx_to_viz = np.round(np.linspace(0, num_matches - 1, n_viz)).astype(int) + viz_matches_im0, viz_matches_im1 = matches_im0[match_idx_to_viz], matches_im1[match_idx_to_viz] + + image_mean = torch.as_tensor([0.5, 0.5, 0.5], device='cpu').reshape(1, 3, 1, 1) + image_std = torch.as_tensor([0.5, 0.5, 0.5], device='cpu').reshape(1, 3, 1, 1) + + viz_imgs = [] + for i, view in enumerate([view1, view2]): + rgb_tensor = view['img'] * image_std + image_mean + viz_imgs.append(rgb_tensor.squeeze(0).permute(1, 2, 0).cpu().numpy()) + + H0, W0, H1, W1 = *viz_imgs[0].shape[:2], *viz_imgs[1].shape[:2] + img0 = np.pad(viz_imgs[0], ((0, max(H1 - H0, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img1 = np.pad(viz_imgs[1], ((0, max(H0 - H1, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img = np.concatenate((img0, img1), axis=1) + pl.figure() + pl.imshow(img) + cmap = pl.get_cmap('jet') + for i in range(n_viz): + (x0, y0), (x1, y1) = viz_matches_im0[i].T, viz_matches_im1[i].T + pl.plot([x0, x1 + W0], [y0, y1], '-+', color=cmap(i / (n_viz - 1)), scalex=False, scaley=False) + pl.show(block=True) +``` +![matching example on croco pair](assets/matching.jpg) + +## Training + +In this section, we present a short demonstration to get started with training MASt3R. + +### Datasets + +See [Datasets section in DUSt3R](https://github.com/naver/dust3r?tab=readme-ov-file#datasets) + +### Demo + +Like for the DUSt3R training demo, we're going to download and prepare the same subset of [CO3Dv2](https://github.com/facebookresearch/co3d) - [Creative Commons Attribution-NonCommercial 4.0 International](https://github.com/facebookresearch/co3d/blob/main/LICENSE) and launch the training code on it. +It is the exact same process as DUSt3R. +The demo model will be trained for a few epochs on a very small dataset. +It will not be very good. + +```bash +# download and prepare the co3d subset +mkdir -p data/co3d_subset +cd data/co3d_subset +git clone https://github.com/facebookresearch/co3d +cd co3d +python3 ./co3d/download_dataset.py --download_folder ../ --single_sequence_subset +rm ../*.zip +cd ../../.. + +python3 datasets_preprocess/preprocess_co3d.py --co3d_dir data/co3d_subset --output_dir data/co3d_subset_processed --single_sequence_subset + +# download the pretrained dust3r checkpoint +mkdir -p checkpoints/ +wget https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth -P checkpoints/ + +# for this example we'll do fewer epochs, for the actual hyperparameters we used in the paper, see the next section: "Our Hyperparameters" +torchrun --nproc_per_node=4 train.py \ + --train_dataset "1000 @ Co3d(split='train', ROOT='data/co3d_subset_processed', aug_crop='auto', aug_monocular=0.005, aug_rot90='diff', mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], n_corres=8192, nneg=0.5, transform=ColorJitter)" \ + --test_dataset "100 @ Co3d(split='test', ROOT='data/co3d_subset_processed', resolution=(512,384), n_corres=1024, seed=777)" \ + --model "AsymmetricMASt3R(pos_embed='RoPE100', patch_embed_cls='ManyAR_PatchEmbed', img_size=(512, 512), head_type='catmlp+dpt', output_mode='pts3d+desc24', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12, two_confs=True)" \ + --train_criterion "ConfLoss(Regr3D(L21, norm_mode='?avg_dis'), alpha=0.2) + 0.075*ConfMatchingLoss(MatchingLoss(InfoNCE(mode='proper', temperature=0.05), negatives_padding=0, blocksize=8192), alpha=10.0, confmode='mean')" \ + --test_criterion "Regr3D_ScaleShiftInv(L21, norm_mode='?avg_dis', gt_scale=True, sky_loss_value=0) + -1.*MatchingLoss(APLoss(nq='torch', fp=torch.float16), negatives_padding=12288)" \ + --pretrained "checkpoints/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth" \ + --lr 0.0001 --min_lr 1e-06 --warmup_epochs 1 --epochs 10 --batch_size 4 --accum_iter 4 \ + --save_freq 1 --keep_freq 5 --eval_freq 1 --disable_cudnn_benchmark \ + --output_dir "checkpoints/mast3r_demo" + +``` + +### Our Hyperparameters +We didn't release all the training datasets, but here are the commands we used for training our models: + +```bash +# MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric - train mast3r with metric regression and matching loss +# we used cosxl to generate variations of DL3DV: "foggy", "night", "rainy", "snow", "sunny" but we were not convinced by it. + +torchrun --nproc_per_node=8 train.py \ + --train_dataset "57_000 @ Habitat512(1_000_000, split='train', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 68_400 @ BlendedMVS(split='train', mask_sky=True, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 68_400 @ MegaDepth(split='train', mask_sky=True, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 45_600 @ ARKitScenes(split='train', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 22_800 @ Co3d(split='train', mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 22_800 @ StaticThings3D(mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 45_600 @ ScanNetpp(split='train', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 45_600 @ TartanAir(pairs_subset='', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 4_560 @ UnrealStereo4K(resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 1_140 @ VirtualKitti(optical_center_is_centered=True, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 22_800 @ WildRgbd(split='train', mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 145_920 @ NianticMapFree(split='train', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 57_000 @ DL3DV(split='nlight', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 57_000 @ DL3DV(split='not-nlight', cosxl_augmentations=None, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 34_200 @ InternalUnreleasedDataset(resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5)" \ + --test_dataset "Habitat512(1_000, split='val', resolution=(512,384), seed=777, n_corres=1024) + 1_000 @ BlendedMVS(split='val', resolution=(512,384), mask_sky=True, seed=777, n_corres=1024) + 1_000 @ ARKitScenes(split='test', resolution=(512,384), seed=777, n_corres=1024) + 1_000 @ MegaDepth(split='val', mask_sky=True, resolution=(512,336), seed=777, n_corres=1024) + 1_000 @ Co3d(split='test', resolution=(512,384), mask_bg='rand', seed=777, n_corres=1024)" \ + --model "AsymmetricMASt3R(pos_embed='RoPE100', patch_embed_cls='ManyAR_PatchEmbed', img_size=(512, 512), head_type='catmlp+dpt', output_mode='pts3d+desc24', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12, two_confs=True, desc_conf_mode=('exp', 0, inf))" \ + --train_criterion "ConfLoss(Regr3D(L21, norm_mode='?avg_dis'), alpha=0.2, loss_in_log=False) + 0.075*ConfMatchingLoss(MatchingLoss(InfoNCE(mode='proper', temperature=0.05), negatives_padding=0, blocksize=8192), alpha=10.0, confmode='mean')" \ + --test_criterion "Regr3D(L21, norm_mode='?avg_dis', gt_scale=True, sky_loss_value=0) + -1.*MatchingLoss(APLoss(nq='torch', fp=torch.float16), negatives_padding=12288)" \ + --pretrained "checkpoints/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth" \ + --lr 0.0001 --min_lr 1e-06 --warmup_epochs 8 --epochs 50 --batch_size 4 --accum_iter 2 \ + --save_freq 1 --keep_freq 5 --eval_freq 1 --print_freq=10 --disable_cudnn_benchmark \ + --output_dir "checkpoints/MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric" + +``` + +## Visual Localization + +### Dataset preparation + +See [Visloc section in DUSt3R](https://github.com/naver/dust3r/blob/main/dust3r_visloc/README.md#dataset-preparation) + +### Example Commands + +With `visloc.py` you can run our visual localization experiments on Aachen-Day-Night, InLoc, Cambridge Landmarks and 7 Scenes. + + +```bash +# Aachen-Day-Night-v1.1: +# scene in 'day' 'night' +# scene can also be 'all' +python3 visloc.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric --dataset "VislocAachenDayNight('/path/to/prepared/Aachen-Day-Night-v1.1/', subscene='${scene}', pairsfile='fire_top50', topk=20)" --pixel_tol 5 --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/Aachen-Day-Night-v1.1/${scene}/loc + +# or with coarse to fine: + +python3 visloc.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric --dataset "VislocAachenDayNight('/path/to/prepared/Aachen-Day-Night-v1.1/', subscene='${scene}', pairsfile='fire_top50', topk=20)" --pixel_tol 5 --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/Aachen-Day-Night-v1.1/${scene}/loc --coarse_to_fine --max_batch_size 48 --c2f_crop_with_homography + +# InLoc +python3 visloc.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric --dataset "VislocInLoc('/path/to/prepared/InLoc/', pairsfile='pairs-query-netvlad40-temporal', topk=20)" --pixel_tol 5 --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/InLoc/loc + +# or with coarse to fine: + +python3 visloc.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric --dataset "VislocInLoc('/path/to/prepared/InLoc/', pairsfile='pairs-query-netvlad40-temporal', topk=20)" --pixel_tol 5 --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/InLoc/loc --coarse_to_fine --max_image_size 1200 --max_batch_size 48 --c2f_crop_with_homography + +# 7-scenes: +# scene in 'chess' 'fire' 'heads' 'office' 'pumpkin' 'redkitchen' 'stairs' +python3 visloc.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric --dataset "VislocSevenScenes('/path/to/prepared/7-scenes/', subscene='${scene}', pairsfile='APGeM-LM18_top20', topk=1)" --pixel_tol 5 --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/7-scenes/${scene}/loc + +# Cambridge Landmarks: +# scene in 'ShopFacade' 'GreatCourt' 'KingsCollege' 'OldHospital' 'StMarysChurch' +python3 visloc.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric --dataset "VislocCambridgeLandmarks('/path/to/prepared/Cambridge_Landmarks/', subscene='${scene}', pairsfile='APGeM-LM18_top50', topk=20)" --pixel_tol 5 --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/Cambridge_Landmarks/${scene}/loc + +``` diff --git a/submodules/mast3r/assets/NLE_tower/01D90321-69C8-439F-B0B0-E87E7634741C-83120-000041DAE419D7AE.jpg b/submodules/mast3r/assets/NLE_tower/01D90321-69C8-439F-B0B0-E87E7634741C-83120-000041DAE419D7AE.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fd6b2d4edcfb5b277350a7ab0948a144f01f809c Binary files /dev/null and b/submodules/mast3r/assets/NLE_tower/01D90321-69C8-439F-B0B0-E87E7634741C-83120-000041DAE419D7AE.jpg differ diff --git a/submodules/mast3r/assets/NLE_tower/1AD85EF5-B651-4291-A5C0-7BDB7D966384-83120-000041DADF639E09.jpg b/submodules/mast3r/assets/NLE_tower/1AD85EF5-B651-4291-A5C0-7BDB7D966384-83120-000041DADF639E09.jpg new file mode 100644 index 0000000000000000000000000000000000000000..999cc01f92686bdab008e91004a457769613845f Binary files /dev/null and b/submodules/mast3r/assets/NLE_tower/1AD85EF5-B651-4291-A5C0-7BDB7D966384-83120-000041DADF639E09.jpg differ diff --git a/submodules/mast3r/assets/NLE_tower/2679C386-1DC0-4443-81B5-93D7EDE4AB37-83120-000041DADB2EA917.jpg b/submodules/mast3r/assets/NLE_tower/2679C386-1DC0-4443-81B5-93D7EDE4AB37-83120-000041DADB2EA917.jpg new file mode 100644 index 0000000000000000000000000000000000000000..79e8bf4c76d6b281db4ebb1ad8e08ef4ec03b66a Binary files /dev/null and b/submodules/mast3r/assets/NLE_tower/2679C386-1DC0-4443-81B5-93D7EDE4AB37-83120-000041DADB2EA917.jpg differ diff --git a/submodules/mast3r/assets/NLE_tower/28EDBB63-B9F9-42FB-AC86-4852A33ED71B-83120-000041DAF22407A1.jpg b/submodules/mast3r/assets/NLE_tower/28EDBB63-B9F9-42FB-AC86-4852A33ED71B-83120-000041DAF22407A1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ba840a9d53c7c48087d080a98033ebe7354ebd84 Binary files /dev/null and b/submodules/mast3r/assets/NLE_tower/28EDBB63-B9F9-42FB-AC86-4852A33ED71B-83120-000041DAF22407A1.jpg differ diff --git a/submodules/mast3r/assets/NLE_tower/91E9B685-7A7D-42D7-B933-23A800EE4129-83120-000041DAE12C8176.jpg b/submodules/mast3r/assets/NLE_tower/91E9B685-7A7D-42D7-B933-23A800EE4129-83120-000041DAE12C8176.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6c261b1cd68d26158b33e8874cd2530dbcf1e75f Binary files /dev/null and b/submodules/mast3r/assets/NLE_tower/91E9B685-7A7D-42D7-B933-23A800EE4129-83120-000041DAE12C8176.jpg differ diff --git a/submodules/mast3r/assets/NLE_tower/CDBBD885-54C3-4EB4-9181-226059A60EE0-83120-000041DAE0C3D612.jpg b/submodules/mast3r/assets/NLE_tower/CDBBD885-54C3-4EB4-9181-226059A60EE0-83120-000041DAE0C3D612.jpg new file mode 100644 index 0000000000000000000000000000000000000000..224d0f7992d19e87464d430e1ee9c0469e9a4028 Binary files /dev/null and b/submodules/mast3r/assets/NLE_tower/CDBBD885-54C3-4EB4-9181-226059A60EE0-83120-000041DAE0C3D612.jpg differ diff --git a/submodules/mast3r/assets/NLE_tower/FF5599FD-768B-431A-AB83-BDA5FB44CB9D-83120-000041DADDE35483.jpg b/submodules/mast3r/assets/NLE_tower/FF5599FD-768B-431A-AB83-BDA5FB44CB9D-83120-000041DADDE35483.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9974efd5907393daa5ae47a047daec9698866a38 Binary files /dev/null and b/submodules/mast3r/assets/NLE_tower/FF5599FD-768B-431A-AB83-BDA5FB44CB9D-83120-000041DADDE35483.jpg differ diff --git a/submodules/mast3r/assets/demo.jpg b/submodules/mast3r/assets/demo.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0c3c6560eca4b1cb41e8c29b52c7f25e9afd5173 Binary files /dev/null and b/submodules/mast3r/assets/demo.jpg differ diff --git a/submodules/mast3r/assets/examples.jpg b/submodules/mast3r/assets/examples.jpg new file mode 100644 index 0000000000000000000000000000000000000000..aa40b208a5107269aed3058ccd5315457ec14676 Binary files /dev/null and b/submodules/mast3r/assets/examples.jpg differ diff --git a/submodules/mast3r/assets/mast3r.jpg b/submodules/mast3r/assets/mast3r.jpg new file mode 100644 index 0000000000000000000000000000000000000000..de4a0ff4a8727f20030d673c0fcad653fc7e56d4 Binary files /dev/null and b/submodules/mast3r/assets/mast3r.jpg differ diff --git a/submodules/mast3r/assets/mast3r_archi.jpg b/submodules/mast3r/assets/mast3r_archi.jpg new file mode 100644 index 0000000000000000000000000000000000000000..126a371c984498587437c5f05ca842c5fda63cc2 Binary files /dev/null and b/submodules/mast3r/assets/mast3r_archi.jpg differ diff --git a/submodules/mast3r/assets/matching.jpg b/submodules/mast3r/assets/matching.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c35487745cd0e01ef1c3f5a72127cbf429840e59 Binary files /dev/null and b/submodules/mast3r/assets/matching.jpg differ diff --git a/submodules/mast3r/demo.py b/submodules/mast3r/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee5ee1030af1214f6204af9826de5e22a53ecfa --- /dev/null +++ b/submodules/mast3r/demo.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# gradio demo executable +# -------------------------------------------------------- +import os +import torch +import tempfile +from contextlib import nullcontext + +from mast3r.demo import get_args_parser, main_demo + +from mast3r.model import AsymmetricMASt3R +from mast3r.utils.misc import hash_md5 + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.demo import set_print_with_timestamp + +import matplotlib.pyplot as pl +pl.ion() + +torch.backends.cuda.matmul.allow_tf32 = True # for gpu >= Ampere and pytorch >= 1.12 + +if __name__ == '__main__': + parser = get_args_parser() + args = parser.parse_args() + set_print_with_timestamp() + + if args.server_name is not None: + server_name = args.server_name + else: + server_name = '0.0.0.0' if args.local_network else '127.0.0.1' + + if args.weights is not None: + weights_path = args.weights + else: + weights_path = "naver/" + args.model_name + + model = AsymmetricMASt3R.from_pretrained(weights_path).to(args.device) + chkpt_tag = hash_md5(weights_path) + + def get_context(tmp_dir): + return tempfile.TemporaryDirectory(suffix='_mast3r_gradio_demo') if tmp_dir is None \ + else nullcontext(tmp_dir) + with get_context(args.tmp_dir) as tmpdirname: + cache_path = os.path.join(tmpdirname, chkpt_tag) + os.makedirs(cache_path, exist_ok=True) + main_demo(cache_path, model, args.device, args.image_size, server_name, args.server_port, silent=args.silent, + share=args.share, gradio_delete_cache=args.gradio_delete_cache) diff --git a/submodules/mast3r/demo_dust3r_ga.py b/submodules/mast3r/demo_dust3r_ga.py new file mode 100644 index 0000000000000000000000000000000000000000..361c10e392e42525d57765b3f95fec43a89035a3 --- /dev/null +++ b/submodules/mast3r/demo_dust3r_ga.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# mast3r gradio demo executable +# -------------------------------------------------------- +import os +import torch +import tempfile + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.model import AsymmetricCroCo3DStereo +from mast3r.model import AsymmetricMASt3R +from dust3r.demo import get_args_parser as dust3r_get_args_parser +from dust3r.demo import main_demo, set_print_with_timestamp + +import matplotlib.pyplot as pl +pl.ion() + +torch.backends.cuda.matmul.allow_tf32 = True # for gpu >= Ampere and pytorch >= 1.12 + + +def get_args_parser(): + parser = dust3r_get_args_parser() + + actions = parser._actions + for action in actions: + if action.dest == 'model_name': + action.choices.append('MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric') + # change defaults + parser.prog = 'mast3r demo' + return parser + + +if __name__ == '__main__': + parser = get_args_parser() + args = parser.parse_args() + set_print_with_timestamp() + + if args.tmp_dir is not None: + tmp_path = args.tmp_dir + os.makedirs(tmp_path, exist_ok=True) + tempfile.tempdir = tmp_path + + if args.server_name is not None: + server_name = args.server_name + else: + server_name = '0.0.0.0' if args.local_network else '127.0.0.1' + + if args.weights is not None: + weights_path = args.weights + else: + weights_path = "naver/" + args.model_name + + try: + model = AsymmetricMASt3R.from_pretrained(weights_path).to(args.device) + except Exception as e: + model = AsymmetricCroCo3DStereo.from_pretrained(weights_path).to(args.device) + + # dust3r will write the 3D model inside tmpdirname + with tempfile.TemporaryDirectory(suffix='dust3r_gradio_demo') as tmpdirname: + if not args.silent: + print('Outputing stuff in', tmpdirname) + main_demo(tmpdirname, model, args.device, args.image_size, server_name, args.server_port, silent=args.silent) diff --git a/submodules/mast3r/docker/docker-compose-cpu.yml b/submodules/mast3r/docker/docker-compose-cpu.yml new file mode 100644 index 0000000000000000000000000000000000000000..746fe20a790cf609f467a8eba0ae1461669fa5f6 --- /dev/null +++ b/submodules/mast3r/docker/docker-compose-cpu.yml @@ -0,0 +1,16 @@ +version: '3.8' +services: + mast3r-demo: + build: + context: ./files + dockerfile: cpu.Dockerfile + ports: + - "7860:7860" + volumes: + - ./files/checkpoints:/mast3r/checkpoints + environment: + - DEVICE=cpu + - MODEL=${MODEL:-MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric.pth} + cap_add: + - IPC_LOCK + - SYS_RESOURCE diff --git a/submodules/mast3r/docker/docker-compose-cuda.yml b/submodules/mast3r/docker/docker-compose-cuda.yml new file mode 100644 index 0000000000000000000000000000000000000000..30670bd837c09ecd3f8546e640eca87119784769 --- /dev/null +++ b/submodules/mast3r/docker/docker-compose-cuda.yml @@ -0,0 +1,23 @@ +version: '3.8' +services: + mast3r-demo: + build: + context: ./files + dockerfile: cuda.Dockerfile + ports: + - "7860:7860" + environment: + - DEVICE=cuda + - MODEL=${MODEL:-MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric.pth} + volumes: + - ./files/checkpoints:/mast3r/checkpoints + cap_add: + - IPC_LOCK + - SYS_RESOURCE + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] diff --git a/submodules/mast3r/docker/files/cpu.Dockerfile b/submodules/mast3r/docker/files/cpu.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..fcfff9fcec9df0aa1b99b6ecd7b7cfb3c7ffa082 --- /dev/null +++ b/submodules/mast3r/docker/files/cpu.Dockerfile @@ -0,0 +1,39 @@ +FROM python:3.11-slim + +LABEL description="Docker container for MASt3R with dependencies installed. CPU VERSION" + +ENV DEVICE="cpu" +ENV MODEL="MASt3R_ViTLarge_BaseDecoder_512_dpt.pth" +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + git \ + libgl1-mesa-glx \ + libegl1-mesa \ + libxrandr2 \ + libxrandr2 \ + libxss1 \ + libxcursor1 \ + libxcomposite1 \ + libasound2 \ + libxi6 \ + libxtst6 \ + libglib2.0-0 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN git clone --recursive https://github.com/naver/mast3r /mast3r +WORKDIR /mast3r/dust3r + +RUN pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu +RUN pip install -r requirements.txt +RUN pip install -r requirements_optional.txt +RUN pip install opencv-python==4.8.0.74 + +WORKDIR /mast3r +RUN pip install -r requirements.txt + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/submodules/mast3r/docker/files/cuda.Dockerfile b/submodules/mast3r/docker/files/cuda.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..2351878136f03992a24f8f297d7a0b1b57029457 --- /dev/null +++ b/submodules/mast3r/docker/files/cuda.Dockerfile @@ -0,0 +1,29 @@ +FROM nvcr.io/nvidia/pytorch:24.01-py3 + +LABEL description="Docker container for MASt3R with dependencies installed. CUDA VERSION" +ENV DEVICE="cuda" +ENV MODEL="MASt3R_ViTLarge_BaseDecoder_512_dpt.pth" +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + git=1:2.34.1-1ubuntu1.10 \ + libglib2.0-0=2.72.4-0ubuntu2.2 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN git clone --recursive https://github.com/naver/mast3r /mast3r +WORKDIR /mast3r/dust3r +RUN pip install -r requirements.txt +RUN pip install -r requirements_optional.txt +RUN pip install opencv-python==4.8.0.74 + +WORKDIR /mast3r/dust3r/croco/models/curope/ +RUN python setup.py build_ext --inplace + +WORKDIR /mast3r +RUN pip install -r requirements.txt + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/submodules/mast3r/docker/files/entrypoint.sh b/submodules/mast3r/docker/files/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..1d39d8a294f171e42bd39ec59324926f7617c055 --- /dev/null +++ b/submodules/mast3r/docker/files/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eux + +DEVICE=${DEVICE:-cuda} +MODEL=${MODEL:-MASt3R_ViTLarge_BaseDecoder_512_dpt.pth} + +exec python3 demo.py --weights "checkpoints/$MODEL" --device "$DEVICE" --local_network "$@" diff --git a/submodules/mast3r/docker/run.sh b/submodules/mast3r/docker/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..4cd0380ef2be2ef462361e77cf83828973a0a5f9 --- /dev/null +++ b/submodules/mast3r/docker/run.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +set -eux + +# Default model name +model_name="MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric.pth" + +check_docker() { + if ! command -v docker &>/dev/null; then + echo "Docker could not be found. Please install Docker and try again." + exit 1 + fi +} + +download_model_checkpoint() { + if [ -f "./files/checkpoints/${model_name}" ]; then + echo "Model checkpoint ${model_name} already exists. Skipping download." + return + fi + echo "Downloading model checkpoint ${model_name}..." + wget "https://download.europe.naverlabs.com/ComputerVision/MASt3R/${model_name}" -P ./files/checkpoints +} + +set_dcomp() { + if command -v docker-compose &>/dev/null; then + dcomp="docker-compose" + elif command -v docker &>/dev/null && docker compose version &>/dev/null; then + dcomp="docker compose" + else + echo "Docker Compose could not be found. Please install Docker Compose and try again." + exit 1 + fi +} + +run_docker() { + export MODEL=${model_name} + if [ "$with_cuda" -eq 1 ]; then + $dcomp -f docker-compose-cuda.yml up --build + else + $dcomp -f docker-compose-cpu.yml up --build + fi +} + +with_cuda=0 +for arg in "$@"; do + case $arg in + --with-cuda) + with_cuda=1 + ;; + --model_name=*) + model_name="${arg#*=}.pth" + ;; + *) + echo "Unknown parameter passed: $arg" + exit 1 + ;; + esac +done + + +main() { + check_docker + download_model_checkpoint + set_dcomp + run_docker +} + +main diff --git a/submodules/mast3r/dust3r/.gitignore b/submodules/mast3r/dust3r/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..194e236cbd708160926c3513b4232285eb47b029 --- /dev/null +++ b/submodules/mast3r/dust3r/.gitignore @@ -0,0 +1,132 @@ +data/ +checkpoints/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/submodules/mast3r/dust3r/.gitmodules b/submodules/mast3r/dust3r/.gitmodules new file mode 100644 index 0000000000000000000000000000000000000000..c950ef981a8d2e47599dd7acbbe1bf8de9a42aca --- /dev/null +++ b/submodules/mast3r/dust3r/.gitmodules @@ -0,0 +1,3 @@ +[submodule "croco"] + path = croco + url = https://github.com/naver/croco diff --git a/submodules/mast3r/dust3r/LICENSE b/submodules/mast3r/dust3r/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..a97986e3a8ddd49973959f6c748dfa8b881b64d3 --- /dev/null +++ b/submodules/mast3r/dust3r/LICENSE @@ -0,0 +1,7 @@ +DUSt3R, Copyright (c) 2024-present Naver Corporation, is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 license. + +A summary of the CC BY-NC-SA 4.0 license is located here: + https://creativecommons.org/licenses/by-nc-sa/4.0/ + +The CC BY-NC-SA 4.0 license is located here: + https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode diff --git a/submodules/mast3r/dust3r/NOTICE b/submodules/mast3r/dust3r/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..81da544dd534c5465361f35cf6a5a0cfff7c1d3f --- /dev/null +++ b/submodules/mast3r/dust3r/NOTICE @@ -0,0 +1,12 @@ +DUSt3R +Copyright 2024-present NAVER Corp. + +This project contains subcomponents with separate copyright notices and license terms. +Your use of the source code for these subcomponents is subject to the terms and conditions of the following licenses. + +==== + +naver/croco +https://github.com/naver/croco/ + +Creative Commons Attribution-NonCommercial-ShareAlike 4.0 diff --git a/submodules/mast3r/dust3r/README.md b/submodules/mast3r/dust3r/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e7c7a4f9328a62e55a93f757fc41dcbca18ef546 --- /dev/null +++ b/submodules/mast3r/dust3r/README.md @@ -0,0 +1,390 @@ +![demo](assets/dust3r.jpg) + +Official implementation of `DUSt3R: Geometric 3D Vision Made Easy` +[[Project page](https://dust3r.europe.naverlabs.com/)], [[DUSt3R arxiv](https://arxiv.org/abs/2312.14132)] + +> **Make sure to also check [MASt3R](https://github.com/naver/mast3r): Our new model with a local feature head, metric pointmaps, and a more scalable global alignment!** + +![Example of reconstruction from two images](assets/pipeline1.jpg) + +![High level overview of DUSt3R capabilities](assets/dust3r_archi.jpg) + +```bibtex +@inproceedings{dust3r_cvpr24, + title={DUSt3R: Geometric 3D Vision Made Easy}, + author={Shuzhe Wang and Vincent Leroy and Yohann Cabon and Boris Chidlovskii and Jerome Revaud}, + booktitle = {CVPR}, + year = {2024} +} + +@misc{dust3r_arxiv23, + title={DUSt3R: Geometric 3D Vision Made Easy}, + author={Shuzhe Wang and Vincent Leroy and Yohann Cabon and Boris Chidlovskii and Jerome Revaud}, + year={2023}, + eprint={2312.14132}, + archivePrefix={arXiv}, + primaryClass={cs.CV} +} +``` + +## Table of Contents + +- [Table of Contents](#table-of-contents) +- [License](#license) +- [Get Started](#get-started) + - [Installation](#installation) + - [Checkpoints](#checkpoints) + - [Interactive demo](#interactive-demo) + - [Interactive demo with docker](#interactive-demo-with-docker) +- [Usage](#usage) +- [Training](#training) + - [Datasets](#datasets) + - [Demo](#demo) + - [Our Hyperparameters](#our-hyperparameters) + +## License + +The code is distributed under the CC BY-NC-SA 4.0 License. +See [LICENSE](LICENSE) for more information. + +```python +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +``` + +## Get Started + +### Installation + +1. Clone DUSt3R. +```bash +git clone --recursive https://github.com/naver/dust3r +cd dust3r +# if you have already cloned dust3r: +# git submodule update --init --recursive +``` + +2. Create the environment, here we show an example using conda. +```bash +conda create -n dust3r python=3.11 cmake=3.14.0 +conda activate dust3r +conda install pytorch torchvision pytorch-cuda=12.1 -c pytorch -c nvidia # use the correct version of cuda for your system +pip install -r requirements.txt +# Optional: you can also install additional packages to: +# - add support for HEIC images +# - add pyrender, used to render depthmap in some datasets preprocessing +# - add required packages for visloc.py +pip install -r requirements_optional.txt +``` + +3. Optional, compile the cuda kernels for RoPE (as in CroCo v2). +```bash +# DUST3R relies on RoPE positional embeddings for which you can compile some cuda kernels for faster runtime. +cd croco/models/curope/ +python setup.py build_ext --inplace +cd ../../../ +``` + +### Checkpoints + +You can obtain the checkpoints by two ways: + +1) You can use our huggingface_hub integration: the models will be downloaded automatically. + +2) Otherwise, We provide several pre-trained models: + +| Modelname | Training resolutions | Head | Encoder | Decoder | +|-------------|----------------------|------|---------|---------| +| [`DUSt3R_ViTLarge_BaseDecoder_224_linear.pth`](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_224_linear.pth) | 224x224 | Linear | ViT-L | ViT-B | +| [`DUSt3R_ViTLarge_BaseDecoder_512_linear.pth`](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_512_linear.pth) | 512x384, 512x336, 512x288, 512x256, 512x160 | Linear | ViT-L | ViT-B | +| [`DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth`](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth) | 512x384, 512x336, 512x288, 512x256, 512x160 | DPT | ViT-L | ViT-B | + +You can check the hyperparameters we used to train these models in the [section: Our Hyperparameters](#our-hyperparameters) + +To download a specific model, for example `DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth`: +```bash +mkdir -p checkpoints/ +wget https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth -P checkpoints/ +``` + +For the checkpoints, make sure to agree to the license of all the public training datasets and base checkpoints we used, in addition to CC-BY-NC-SA 4.0. Again, see [section: Our Hyperparameters](#our-hyperparameters) for details. + +### Interactive demo + +In this demo, you should be able run DUSt3R on your machine to reconstruct a scene. +First select images that depicts the same scene. + +You can adjust the global alignment schedule and its number of iterations. + +> [!NOTE] +> If you selected one or two images, the global alignment procedure will be skipped (mode=GlobalAlignerMode.PairViewer) + +Hit "Run" and wait. +When the global alignment ends, the reconstruction appears. +Use the slider "min_conf_thr" to show or remove low confidence areas. + +```bash +python3 demo.py --model_name DUSt3R_ViTLarge_BaseDecoder_512_dpt + +# Use --weights to load a checkpoint from a local file, eg --weights checkpoints/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth +# Use --image_size to select the correct resolution for the selected checkpoint. 512 (default) or 224 +# Use --local_network to make it accessible on the local network, or --server_name to specify the url manually +# Use --server_port to change the port, by default it will search for an available port starting at 7860 +# Use --device to use a different device, by default it's "cuda" +``` + +### Interactive demo with docker + +To run DUSt3R using Docker, including with NVIDIA CUDA support, follow these instructions: + +1. **Install Docker**: If not already installed, download and install `docker` and `docker compose` from the [Docker website](https://www.docker.com/get-started). + +2. **Install NVIDIA Docker Toolkit**: For GPU support, install the NVIDIA Docker toolkit from the [Nvidia website](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). + +3. **Build the Docker image and run it**: `cd` into the `./docker` directory and run the following commands: + +```bash +cd docker +bash run.sh --with-cuda --model_name="DUSt3R_ViTLarge_BaseDecoder_512_dpt" +``` + +Or if you want to run the demo without CUDA support, run the following command: + +```bash +cd docker +bash run.sh --model_name="DUSt3R_ViTLarge_BaseDecoder_512_dpt" +``` + +By default, `demo.py` is lanched with the option `--local_network`. +Visit `http://localhost:7860/` to access the web UI (or replace `localhost` with the machine's name to access it from the network). + +`run.sh` will launch docker-compose using either the [docker-compose-cuda.yml](docker/docker-compose-cuda.yml) or [docker-compose-cpu.ym](docker/docker-compose-cpu.yml) config file, then it starts the demo using [entrypoint.sh](docker/files/entrypoint.sh). + + +![demo](assets/demo.jpg) + +## Usage + +```python +from dust3r.inference import inference +from dust3r.model import AsymmetricCroCo3DStereo +from dust3r.utils.image import load_images +from dust3r.image_pairs import make_pairs +from dust3r.cloud_opt import global_aligner, GlobalAlignerMode + +if __name__ == '__main__': + device = 'cuda' + batch_size = 1 + schedule = 'cosine' + lr = 0.01 + niter = 300 + + model_name = "naver/DUSt3R_ViTLarge_BaseDecoder_512_dpt" + # you can put the path to a local checkpoint in model_name if needed + model = AsymmetricCroCo3DStereo.from_pretrained(model_name).to(device) + # load_images can take a list of images or a directory + images = load_images(['croco/assets/Chateau1.png', 'croco/assets/Chateau2.png'], size=512) + pairs = make_pairs(images, scene_graph='complete', prefilter=None, symmetrize=True) + output = inference(pairs, model, device, batch_size=batch_size) + + # at this stage, you have the raw dust3r predictions + view1, pred1 = output['view1'], output['pred1'] + view2, pred2 = output['view2'], output['pred2'] + # here, view1, pred1, view2, pred2 are dicts of lists of len(2) + # -> because we symmetrize we have (im1, im2) and (im2, im1) pairs + # in each view you have: + # an integer image identifier: view1['idx'] and view2['idx'] + # the img: view1['img'] and view2['img'] + # the image shape: view1['true_shape'] and view2['true_shape'] + # an instance string output by the dataloader: view1['instance'] and view2['instance'] + # pred1 and pred2 contains the confidence values: pred1['conf'] and pred2['conf'] + # pred1 contains 3D points for view1['img'] in view1['img'] space: pred1['pts3d'] + # pred2 contains 3D points for view2['img'] in view1['img'] space: pred2['pts3d_in_other_view'] + + # next we'll use the global_aligner to align the predictions + # depending on your task, you may be fine with the raw output and not need it + # with only two input images, you could use GlobalAlignerMode.PairViewer: it would just convert the output + # if using GlobalAlignerMode.PairViewer, no need to run compute_global_alignment + scene = global_aligner(output, device=device, mode=GlobalAlignerMode.PointCloudOptimizer) + loss = scene.compute_global_alignment(init="mst", niter=niter, schedule=schedule, lr=lr) + + # retrieve useful values from scene: + imgs = scene.imgs + focals = scene.get_focals() + poses = scene.get_im_poses() + pts3d = scene.get_pts3d() + confidence_masks = scene.get_masks() + + # visualize reconstruction + scene.show() + + # find 2D-2D matches between the two images + from dust3r.utils.geometry import find_reciprocal_matches, xy_grid + pts2d_list, pts3d_list = [], [] + for i in range(2): + conf_i = confidence_masks[i].cpu().numpy() + pts2d_list.append(xy_grid(*imgs[i].shape[:2][::-1])[conf_i]) # imgs[i].shape[:2] = (H, W) + pts3d_list.append(pts3d[i].detach().cpu().numpy()[conf_i]) + reciprocal_in_P2, nn2_in_P1, num_matches = find_reciprocal_matches(*pts3d_list) + print(f'found {num_matches} matches') + matches_im1 = pts2d_list[1][reciprocal_in_P2] + matches_im0 = pts2d_list[0][nn2_in_P1][reciprocal_in_P2] + + # visualize a few matches + import numpy as np + from matplotlib import pyplot as pl + n_viz = 10 + match_idx_to_viz = np.round(np.linspace(0, num_matches-1, n_viz)).astype(int) + viz_matches_im0, viz_matches_im1 = matches_im0[match_idx_to_viz], matches_im1[match_idx_to_viz] + + H0, W0, H1, W1 = *imgs[0].shape[:2], *imgs[1].shape[:2] + img0 = np.pad(imgs[0], ((0, max(H1 - H0, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img1 = np.pad(imgs[1], ((0, max(H0 - H1, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img = np.concatenate((img0, img1), axis=1) + pl.figure() + pl.imshow(img) + cmap = pl.get_cmap('jet') + for i in range(n_viz): + (x0, y0), (x1, y1) = viz_matches_im0[i].T, viz_matches_im1[i].T + pl.plot([x0, x1 + W0], [y0, y1], '-+', color=cmap(i / (n_viz - 1)), scalex=False, scaley=False) + pl.show(block=True) + +``` +![matching example on croco pair](assets/matching.jpg) + +## Training + +In this section, we present a short demonstration to get started with training DUSt3R. + +### Datasets +At this moment, we have added the following training datasets: + - [CO3Dv2](https://github.com/facebookresearch/co3d) - [Creative Commons Attribution-NonCommercial 4.0 International](https://github.com/facebookresearch/co3d/blob/main/LICENSE) + - [ARKitScenes](https://github.com/apple/ARKitScenes) - [Creative Commons Attribution-NonCommercial-ShareAlike 4.0](https://github.com/apple/ARKitScenes/tree/main?tab=readme-ov-file#license) + - [ScanNet++](https://kaldir.vc.in.tum.de/scannetpp/) - [non-commercial research and educational purposes](https://kaldir.vc.in.tum.de/scannetpp/static/scannetpp-terms-of-use.pdf) + - [BlendedMVS](https://github.com/YoYo000/BlendedMVS) - [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/) + - [WayMo Open dataset](https://github.com/waymo-research/waymo-open-dataset) - [Non-Commercial Use](https://waymo.com/open/terms/) + - [Habitat-Sim](https://github.com/facebookresearch/habitat-sim/blob/main/DATASETS.md) + - [MegaDepth](https://www.cs.cornell.edu/projects/megadepth/) + - [StaticThings3D](https://github.com/lmb-freiburg/robustmvd/blob/master/rmvd/data/README.md#staticthings3d) + - [WildRGB-D](https://github.com/wildrgbd/wildrgbd/) + +For each dataset, we provide a preprocessing script in the `datasets_preprocess` directory and an archive containing the list of pairs when needed. +You have to download the datasets yourself from their official sources, agree to their license, download our list of pairs, and run the preprocessing script. + +Links: + +[ARKitScenes pairs](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/arkitscenes_pairs.zip) +[ScanNet++ pairs](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/scannetpp_pairs.zip) +[BlendedMVS pairs](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/blendedmvs_pairs.npy) +[WayMo Open dataset pairs](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/waymo_pairs.npz) +[Habitat metadata](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/habitat_5views_v1_512x512_metadata.tar.gz) +[MegaDepth pairs](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/megadepth_pairs.npz) +[StaticThings3D pairs](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/staticthings_pairs.npy) + +> [!NOTE] +> They are not strictly equivalent to what was used to train DUSt3R, but they should be close enough. + +### Demo +For this training demo, we're going to download and prepare a subset of [CO3Dv2](https://github.com/facebookresearch/co3d) - [Creative Commons Attribution-NonCommercial 4.0 International](https://github.com/facebookresearch/co3d/blob/main/LICENSE) and launch the training code on it. +The demo model will be trained for a few epochs on a very small dataset. +It will not be very good. + +```bash +# download and prepare the co3d subset +mkdir -p data/co3d_subset +cd data/co3d_subset +git clone https://github.com/facebookresearch/co3d +cd co3d +python3 ./co3d/download_dataset.py --download_folder ../ --single_sequence_subset +rm ../*.zip +cd ../../.. + +python3 datasets_preprocess/preprocess_co3d.py --co3d_dir data/co3d_subset --output_dir data/co3d_subset_processed --single_sequence_subset + +# download the pretrained croco v2 checkpoint +mkdir -p checkpoints/ +wget https://download.europe.naverlabs.com/ComputerVision/CroCo/CroCo_V2_ViTLarge_BaseDecoder.pth -P checkpoints/ + +# the training of dust3r is done in 3 steps. +# for this example we'll do fewer epochs, for the actual hyperparameters we used in the paper, see the next section: "Our Hyperparameters" +# step 1 - train dust3r for 224 resolution +torchrun --nproc_per_node=4 train.py \ + --train_dataset "1000 @ Co3d(split='train', ROOT='data/co3d_subset_processed', aug_crop=16, mask_bg='rand', resolution=224, transform=ColorJitter)" \ + --test_dataset "100 @ Co3d(split='test', ROOT='data/co3d_subset_processed', resolution=224, seed=777)" \ + --model "AsymmetricCroCo3DStereo(pos_embed='RoPE100', img_size=(224, 224), head_type='linear', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ + --train_criterion "ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ + --test_criterion "Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ + --pretrained "checkpoints/CroCo_V2_ViTLarge_BaseDecoder.pth" \ + --lr 0.0001 --min_lr 1e-06 --warmup_epochs 1 --epochs 10 --batch_size 16 --accum_iter 1 \ + --save_freq 1 --keep_freq 5 --eval_freq 1 \ + --output_dir "checkpoints/dust3r_demo_224" + +# step 2 - train dust3r for 512 resolution +torchrun --nproc_per_node=4 train.py \ + --train_dataset "1000 @ Co3d(split='train', ROOT='data/co3d_subset_processed', aug_crop=16, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter)" \ + --test_dataset "100 @ Co3d(split='test', ROOT='data/co3d_subset_processed', resolution=(512,384), seed=777)" \ + --model "AsymmetricCroCo3DStereo(pos_embed='RoPE100', patch_embed_cls='ManyAR_PatchEmbed', img_size=(512, 512), head_type='linear', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ + --train_criterion "ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ + --test_criterion "Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ + --pretrained "checkpoints/dust3r_demo_224/checkpoint-best.pth" \ + --lr 0.0001 --min_lr 1e-06 --warmup_epochs 1 --epochs 10 --batch_size 4 --accum_iter 4 \ + --save_freq 1 --keep_freq 5 --eval_freq 1 \ + --output_dir "checkpoints/dust3r_demo_512" + +# step 3 - train dust3r for 512 resolution with dpt +torchrun --nproc_per_node=4 train.py \ + --train_dataset "1000 @ Co3d(split='train', ROOT='data/co3d_subset_processed', aug_crop=16, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter)" \ + --test_dataset "100 @ Co3d(split='test', ROOT='data/co3d_subset_processed', resolution=(512,384), seed=777)" \ + --model "AsymmetricCroCo3DStereo(pos_embed='RoPE100', patch_embed_cls='ManyAR_PatchEmbed', img_size=(512, 512), head_type='dpt', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ + --train_criterion "ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ + --test_criterion "Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ + --pretrained "checkpoints/dust3r_demo_512/checkpoint-best.pth" \ + --lr 0.0001 --min_lr 1e-06 --warmup_epochs 1 --epochs 10 --batch_size 2 --accum_iter 8 \ + --save_freq 1 --keep_freq 5 --eval_freq 1 --disable_cudnn_benchmark \ + --output_dir "checkpoints/dust3r_demo_512dpt" + +``` + +### Our Hyperparameters + +Here are the commands we used for training the models: + +```bash +# NOTE: ROOT path omitted for datasets +# 224 linear +torchrun --nproc_per_node 8 train.py \ + --train_dataset=" + 100_000 @ Habitat(1_000_000, split='train', aug_crop=16, resolution=224, transform=ColorJitter) + 100_000 @ BlendedMVS(split='train', aug_crop=16, resolution=224, transform=ColorJitter) + 100_000 @ MegaDepth(split='train', aug_crop=16, resolution=224, transform=ColorJitter) + 100_000 @ ARKitScenes(aug_crop=256, resolution=224, transform=ColorJitter) + 100_000 @ Co3d(split='train', aug_crop=16, mask_bg='rand', resolution=224, transform=ColorJitter) + 100_000 @ StaticThings3D(aug_crop=256, mask_bg='rand', resolution=224, transform=ColorJitter) + 100_000 @ ScanNetpp(split='train', aug_crop=256, resolution=224, transform=ColorJitter) + 100_000 @ InternalUnreleasedDataset(aug_crop=128, resolution=224, transform=ColorJitter) " \ + --test_dataset=" Habitat(1_000, split='val', resolution=224, seed=777) + 1_000 @ BlendedMVS(split='val', resolution=224, seed=777) + 1_000 @ MegaDepth(split='val', resolution=224, seed=777) + 1_000 @ Co3d(split='test', mask_bg='rand', resolution=224, seed=777) " \ + --train_criterion="ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ + --test_criterion="Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ + --model="AsymmetricCroCo3DStereo(pos_embed='RoPE100', img_size=(224, 224), head_type='linear', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ + --pretrained="checkpoints/CroCo_V2_ViTLarge_BaseDecoder.pth" \ + --lr=0.0001 --min_lr=1e-06 --warmup_epochs=10 --epochs=100 --batch_size=16 --accum_iter=1 \ + --save_freq=5 --keep_freq=10 --eval_freq=1 \ + --output_dir="checkpoints/dust3r_224" + +# 512 linear +torchrun --nproc_per_node 8 train.py \ + --train_dataset=" + 10_000 @ Habitat(1_000_000, split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ BlendedMVS(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ MegaDepth(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ARKitScenes(aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ Co3d(split='train', aug_crop=16, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ StaticThings3D(aug_crop=256, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ScanNetpp(split='train', aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ InternalUnreleasedDataset(aug_crop=128, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) " \ + --test_dataset=" Habitat(1_000, split='val', resolution=(512,384), seed=777) + 1_000 @ BlendedMVS(split='val', resolution=(512,384), seed=777) + 1_000 @ MegaDepth(split='val', resolution=(512,336), seed=777) + 1_000 @ Co3d(split='test', resolution=(512,384), seed=777) " \ + --train_criterion="ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ + --test_criterion="Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ + --model="AsymmetricCroCo3DStereo(pos_embed='RoPE100', patch_embed_cls='ManyAR_PatchEmbed', img_size=(512, 512), head_type='linear', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ + --pretrained="checkpoints/dust3r_224/checkpoint-best.pth" \ + --lr=0.0001 --min_lr=1e-06 --warmup_epochs=20 --epochs=100 --batch_size=4 --accum_iter=2 \ + --save_freq=10 --keep_freq=10 --eval_freq=1 --print_freq=10 \ + --output_dir="checkpoints/dust3r_512" + +# 512 dpt +torchrun --nproc_per_node 8 train.py \ + --train_dataset=" + 10_000 @ Habitat(1_000_000, split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ BlendedMVS(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ MegaDepth(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ARKitScenes(aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ Co3d(split='train', aug_crop=16, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ StaticThings3D(aug_crop=256, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ScanNetpp(split='train', aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ InternalUnreleasedDataset(aug_crop=128, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) " \ + --test_dataset=" Habitat(1_000, split='val', resolution=(512,384), seed=777) + 1_000 @ BlendedMVS(split='val', resolution=(512,384), seed=777) + 1_000 @ MegaDepth(split='val', resolution=(512,336), seed=777) + 1_000 @ Co3d(split='test', resolution=(512,384), seed=777) " \ + --train_criterion="ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ + --test_criterion="Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ + --model="AsymmetricCroCo3DStereo(pos_embed='RoPE100', patch_embed_cls='ManyAR_PatchEmbed', img_size=(512, 512), head_type='dpt', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ + --pretrained="checkpoints/dust3r_512/checkpoint-best.pth" \ + --lr=0.0001 --min_lr=1e-06 --warmup_epochs=15 --epochs=90 --batch_size=4 --accum_iter=2 \ + --save_freq=5 --keep_freq=10 --eval_freq=1 --print_freq=10 --disable_cudnn_benchmark \ + --output_dir="checkpoints/dust3r_512dpt" + +``` diff --git a/submodules/mast3r/dust3r/assets/demo.jpg b/submodules/mast3r/dust3r/assets/demo.jpg new file mode 100644 index 0000000000000000000000000000000000000000..684b4b08729e1ec3e3a221db8531ff06f0d6dd54 Binary files /dev/null and b/submodules/mast3r/dust3r/assets/demo.jpg differ diff --git a/submodules/mast3r/dust3r/assets/dust3r.jpg b/submodules/mast3r/dust3r/assets/dust3r.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2f65b18fdb613950a683186b2b0fbcbbbcad82e4 Binary files /dev/null and b/submodules/mast3r/dust3r/assets/dust3r.jpg differ diff --git a/submodules/mast3r/dust3r/assets/dust3r_archi.jpg b/submodules/mast3r/dust3r/assets/dust3r_archi.jpg new file mode 100644 index 0000000000000000000000000000000000000000..332de7f7dfd78ef70b9cf3defcebafec1e1a8d6e Binary files /dev/null and b/submodules/mast3r/dust3r/assets/dust3r_archi.jpg differ diff --git a/submodules/mast3r/dust3r/assets/matching.jpg b/submodules/mast3r/dust3r/assets/matching.jpg new file mode 100644 index 0000000000000000000000000000000000000000..68e0e739d7dc25c8ccc6b5dd164022e9a84b2162 Binary files /dev/null and b/submodules/mast3r/dust3r/assets/matching.jpg differ diff --git a/submodules/mast3r/dust3r/assets/pipeline1.jpg b/submodules/mast3r/dust3r/assets/pipeline1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5a0fc1e800b92fae577d6293ad50c8ee1815c3e8 Binary files /dev/null and b/submodules/mast3r/dust3r/assets/pipeline1.jpg differ diff --git a/submodules/mast3r/dust3r/croco/LICENSE b/submodules/mast3r/dust3r/croco/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d9b84b1a65f9db6d8920a9048d162f52ba3ea56d --- /dev/null +++ b/submodules/mast3r/dust3r/croco/LICENSE @@ -0,0 +1,52 @@ +CroCo, Copyright (c) 2022-present Naver Corporation, is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 license. + +A summary of the CC BY-NC-SA 4.0 license is located here: + https://creativecommons.org/licenses/by-nc-sa/4.0/ + +The CC BY-NC-SA 4.0 license is located here: + https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode + + +SEE NOTICE BELOW WITH RESPECT TO THE FILE: models/pos_embed.py, models/blocks.py + +*************************** + +NOTICE WITH RESPECT TO THE FILE: models/pos_embed.py + +This software is being redistributed in a modifiled form. The original form is available here: + +https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py + +This software in this file incorporates parts of the following software available here: + +Transformer: https://github.com/tensorflow/models/blob/master/official/legacy/transformer/model_utils.py +available under the following license: https://github.com/tensorflow/models/blob/master/LICENSE + +MoCo v3: https://github.com/facebookresearch/moco-v3 +available under the following license: https://github.com/facebookresearch/moco-v3/blob/main/LICENSE + +DeiT: https://github.com/facebookresearch/deit +available under the following license: https://github.com/facebookresearch/deit/blob/main/LICENSE + + +ORIGINAL COPYRIGHT NOTICE AND PERMISSION NOTICE AVAILABLE HERE IS REPRODUCE BELOW: + +https://github.com/facebookresearch/mae/blob/main/LICENSE + +Attribution-NonCommercial 4.0 International + +*************************** + +NOTICE WITH RESPECT TO THE FILE: models/blocks.py + +This software is being redistributed in a modifiled form. The original form is available here: + +https://github.com/rwightman/pytorch-image-models + +ORIGINAL COPYRIGHT NOTICE AND PERMISSION NOTICE AVAILABLE HERE IS REPRODUCE BELOW: + +https://github.com/rwightman/pytorch-image-models/blob/master/LICENSE + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/NOTICE b/submodules/mast3r/dust3r/croco/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..d51bb365036c12d428d6e3a4fd00885756d5261c --- /dev/null +++ b/submodules/mast3r/dust3r/croco/NOTICE @@ -0,0 +1,21 @@ +CroCo +Copyright 2022-present NAVER Corp. + +This project contains subcomponents with separate copyright notices and license terms. +Your use of the source code for these subcomponents is subject to the terms and conditions of the following licenses. + +==== + +facebookresearch/mae +https://github.com/facebookresearch/mae + +Attribution-NonCommercial 4.0 International + +==== + +rwightman/pytorch-image-models +https://github.com/rwightman/pytorch-image-models + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/README.MD b/submodules/mast3r/dust3r/croco/README.MD new file mode 100644 index 0000000000000000000000000000000000000000..38e33b001a60bd16749317fb297acd60f28a6f1b --- /dev/null +++ b/submodules/mast3r/dust3r/croco/README.MD @@ -0,0 +1,124 @@ +# CroCo + CroCo v2 / CroCo-Stereo / CroCo-Flow + +[[`CroCo arXiv`](https://arxiv.org/abs/2210.10716)] [[`CroCo v2 arXiv`](https://arxiv.org/abs/2211.10408)] [[`project page and demo`](https://croco.europe.naverlabs.com/)] + +This repository contains the code for our CroCo model presented in our NeurIPS'22 paper [CroCo: Self-Supervised Pre-training for 3D Vision Tasks by Cross-View Completion](https://openreview.net/pdf?id=wZEfHUM5ri) and its follow-up extension published at ICCV'23 [Improved Cross-view Completion Pre-training for Stereo Matching and Optical Flow](https://openaccess.thecvf.com/content/ICCV2023/html/Weinzaepfel_CroCo_v2_Improved_Cross-view_Completion_Pre-training_for_Stereo_Matching_and_ICCV_2023_paper.html), refered to as CroCo v2: + +![image](assets/arch.jpg) + +```bibtex +@inproceedings{croco, + title={{CroCo: Self-Supervised Pre-training for 3D Vision Tasks by Cross-View Completion}}, + author={{Weinzaepfel, Philippe and Leroy, Vincent and Lucas, Thomas and Br\'egier, Romain and Cabon, Yohann and Arora, Vaibhav and Antsfeld, Leonid and Chidlovskii, Boris and Csurka, Gabriela and Revaud J\'er\^ome}}, + booktitle={{NeurIPS}}, + year={2022} +} + +@inproceedings{croco_v2, + title={{CroCo v2: Improved Cross-view Completion Pre-training for Stereo Matching and Optical Flow}}, + author={Weinzaepfel, Philippe and Lucas, Thomas and Leroy, Vincent and Cabon, Yohann and Arora, Vaibhav and Br{\'e}gier, Romain and Csurka, Gabriela and Antsfeld, Leonid and Chidlovskii, Boris and Revaud, J{\'e}r{\^o}me}, + booktitle={ICCV}, + year={2023} +} +``` + +## License + +The code is distributed under the CC BY-NC-SA 4.0 License. See [LICENSE](LICENSE) for more information. +Some components are based on code from [MAE](https://github.com/facebookresearch/mae) released under the CC BY-NC-SA 4.0 License and [timm](https://github.com/rwightman/pytorch-image-models) released under the Apache 2.0 License. +Some components for stereo matching and optical flow are based on code from [unimatch](https://github.com/autonomousvision/unimatch) released under the MIT license. + +## Preparation + +1. Install dependencies on a machine with a NVidia GPU using e.g. conda. Note that `habitat-sim` is required only for the interactive demo and the synthetic pre-training data generation. If you don't plan to use it, you can ignore the line installing it and use a more recent python version. + +```bash +conda create -n croco python=3.7 cmake=3.14.0 +conda activate croco +conda install habitat-sim headless -c conda-forge -c aihabitat +conda install pytorch torchvision -c pytorch +conda install notebook ipykernel matplotlib +conda install ipywidgets widgetsnbextension +conda install scikit-learn tqdm quaternion opencv # only for pretraining / habitat data generation + +``` + +2. Compile cuda kernels for RoPE + +CroCo v2 relies on RoPE positional embeddings for which you need to compile some cuda kernels. +```bash +cd models/curope/ +python setup.py build_ext --inplace +cd ../../ +``` + +This can be a bit long as we compile for all cuda architectures, feel free to update L9 of `models/curope/setup.py` to compile for specific architectures only. +You might also need to set the environment `CUDA_HOME` in case you use a custom cuda installation. + +In case you cannot provide, we also provide a slow pytorch version, which will be automatically loaded. + +3. Download pre-trained model + +We provide several pre-trained models: + +| modelname | pre-training data | pos. embed. | Encoder | Decoder | +|------------------------------------------------------------------------------------------------------------------------------------|-------------------|-------------|---------|---------| +| [`CroCo.pth`](https://download.europe.naverlabs.com/ComputerVision/CroCo/CroCo.pth) | Habitat | cosine | ViT-B | Small | +| [`CroCo_V2_ViTBase_SmallDecoder.pth`](https://download.europe.naverlabs.com/ComputerVision/CroCo/CroCo_V2_ViTBase_SmallDecoder.pth) | Habitat + real | RoPE | ViT-B | Small | +| [`CroCo_V2_ViTBase_BaseDecoder.pth`](https://download.europe.naverlabs.com/ComputerVision/CroCo/CroCo_V2_ViTBase_BaseDecoder.pth) | Habitat + real | RoPE | ViT-B | Base | +| [`CroCo_V2_ViTLarge_BaseDecoder.pth`](https://download.europe.naverlabs.com/ComputerVision/CroCo/CroCo_V2_ViTLarge_BaseDecoder.pth) | Habitat + real | RoPE | ViT-L | Base | + +To download a specific model, i.e., the first one (`CroCo.pth`) +```bash +mkdir -p pretrained_models/ +wget https://download.europe.naverlabs.com/ComputerVision/CroCo/CroCo.pth -P pretrained_models/ +``` + +## Reconstruction example + +Simply run after downloading the `CroCo_V2_ViTLarge_BaseDecoder` pretrained model (or update the corresponding line in `demo.py`) +```bash +python demo.py +``` + +## Interactive demonstration of cross-view completion reconstruction on the Habitat simulator + +First download the test scene from Habitat: +```bash +python -m habitat_sim.utils.datasets_download --uids habitat_test_scenes --data-path habitat-sim-data/ +``` + +Then, run the Notebook demo `interactive_demo.ipynb`. + +In this demo, you should be able to sample a random reference viewpoint from an [Habitat](https://github.com/facebookresearch/habitat-sim) test scene. Use the sliders to change viewpoint and select a masked target view to reconstruct using CroCo. +![croco_interactive_demo](https://user-images.githubusercontent.com/1822210/200516576-7937bc6a-55f8-49ed-8618-3ddf89433ea4.jpg) + +## Pre-training + +### CroCo + +To pre-train CroCo, please first generate the pre-training data from the Habitat simulator, following the instructions in [datasets/habitat_sim/README.MD](datasets/habitat_sim/README.MD) and then run the following command: +``` +torchrun --nproc_per_node=4 pretrain.py --output_dir ./output/pretraining/ +``` + +Our CroCo pre-training was launched on a single server with 4 GPUs. +It should take around 10 days with A100 or 15 days with V100 to do the 400 pre-training epochs, but decent performances are obtained earlier in training. +Note that, while the code contains the same scaling rule of the learning rate as MAE when changing the effective batch size, we did not experimented if it is valid in our case. +The first run can take a few minutes to start, to parse all available pre-training pairs. + +### CroCo v2 + +For CroCo v2 pre-training, in addition to the generation of the pre-training data from the Habitat simulator above, please pre-extract the crops from the real datasets following the instructions in [datasets/crops/README.MD](datasets/crops/README.MD). +Then, run the following command for the largest model (ViT-L encoder, Base decoder): +``` +torchrun --nproc_per_node=8 pretrain.py --model "CroCoNet(enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_num_heads=12, dec_depth=12, pos_embed='RoPE100')" --dataset "habitat_release+ARKitScenes+MegaDepth+3DStreetView+IndoorVL" --warmup_epochs 12 --max_epoch 125 --epochs 250 --amp 0 --keep_freq 5 --output_dir ./output/pretraining_crocov2/ +``` + +Our CroCo v2 pre-training was launched on a single server with 8 GPUs for the largest model, and on a single server with 4 GPUs for the smaller ones, keeping a batch size of 64 per gpu in all cases. +The largest model should take around 12 days on A100. +Note that, while the code contains the same scaling rule of the learning rate as MAE when changing the effective batch size, we did not experimented if it is valid in our case. + +## Stereo matching and Optical flow downstream tasks + +For CroCo-Stereo and CroCo-Flow, please refer to [stereoflow/README.MD](stereoflow/README.MD). diff --git a/submodules/mast3r/dust3r/croco/assets/Chateau1.png b/submodules/mast3r/dust3r/croco/assets/Chateau1.png new file mode 100644 index 0000000000000000000000000000000000000000..d282fc6a51c00b8dd8267d5d507220ae253c2d65 Binary files /dev/null and b/submodules/mast3r/dust3r/croco/assets/Chateau1.png differ diff --git a/submodules/mast3r/dust3r/croco/assets/Chateau2.png b/submodules/mast3r/dust3r/croco/assets/Chateau2.png new file mode 100644 index 0000000000000000000000000000000000000000..722b2fc553ec089346722efb9445526ddfa8e7bd Binary files /dev/null and b/submodules/mast3r/dust3r/croco/assets/Chateau2.png differ diff --git a/submodules/mast3r/dust3r/croco/assets/arch.jpg b/submodules/mast3r/dust3r/croco/assets/arch.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3f5b032729ddc58c06d890a0ebda1749276070c4 Binary files /dev/null and b/submodules/mast3r/dust3r/croco/assets/arch.jpg differ diff --git a/submodules/mast3r/dust3r/croco/croco-stereo-flow-demo.ipynb b/submodules/mast3r/dust3r/croco/croco-stereo-flow-demo.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..2b00a7607ab5f82d1857041969bfec977e56b3e0 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/croco-stereo-flow-demo.ipynb @@ -0,0 +1,191 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9bca0f41", + "metadata": {}, + "source": [ + "# Simple inference example with CroCo-Stereo or CroCo-Flow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80653ef7", + "metadata": {}, + "outputs": [], + "source": [ + "# Copyright (C) 2022-present Naver Corporation. All rights reserved.\n", + "# Licensed under CC BY-NC-SA 4.0 (non-commercial use only)." + ] + }, + { + "cell_type": "markdown", + "id": "4f033862", + "metadata": {}, + "source": [ + "First download the model(s) of your choice by running\n", + "```\n", + "bash stereoflow/download_model.sh crocostereo.pth\n", + "bash stereoflow/download_model.sh crocoflow.pth\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1fb2e392", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "use_gpu = torch.cuda.is_available() and torch.cuda.device_count()>0\n", + "device = torch.device('cuda:0' if use_gpu else 'cpu')\n", + "import matplotlib.pylab as plt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e0e25d77", + "metadata": {}, + "outputs": [], + "source": [ + "from stereoflow.test import _load_model_and_criterion\n", + "from stereoflow.engine import tiled_pred\n", + "from stereoflow.datasets_stereo import img_to_tensor, vis_disparity\n", + "from stereoflow.datasets_flow import flowToColor\n", + "tile_overlap=0.7 # recommended value, higher value can be slightly better but slower" + ] + }, + { + "cell_type": "markdown", + "id": "86a921f5", + "metadata": {}, + "source": [ + "### CroCo-Stereo example" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64e483cb", + "metadata": {}, + "outputs": [], + "source": [ + "image1 = np.asarray(Image.open(''))\n", + "image2 = np.asarray(Image.open(''))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0d04303", + "metadata": {}, + "outputs": [], + "source": [ + "model, _, cropsize, with_conf, task, tile_conf_mode = _load_model_and_criterion('stereoflow_models/crocostereo.pth', None, device)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47dc14b5", + "metadata": {}, + "outputs": [], + "source": [ + "im1 = img_to_tensor(image1).to(device).unsqueeze(0)\n", + "im2 = img_to_tensor(image2).to(device).unsqueeze(0)\n", + "with torch.inference_mode():\n", + " pred, _, _ = tiled_pred(model, None, im1, im2, None, conf_mode=tile_conf_mode, overlap=tile_overlap, crop=cropsize, with_conf=with_conf, return_time=False)\n", + "pred = pred.squeeze(0).squeeze(0).cpu().numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "583b9f16", + "metadata": {}, + "outputs": [], + "source": [ + "plt.imshow(vis_disparity(pred))\n", + "plt.axis('off')" + ] + }, + { + "cell_type": "markdown", + "id": "d2df5d70", + "metadata": {}, + "source": [ + "### CroCo-Flow example" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ee257a7", + "metadata": {}, + "outputs": [], + "source": [ + "image1 = np.asarray(Image.open(''))\n", + "image2 = np.asarray(Image.open(''))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5edccf0", + "metadata": {}, + "outputs": [], + "source": [ + "model, _, cropsize, with_conf, task, tile_conf_mode = _load_model_and_criterion('stereoflow_models/crocoflow.pth', None, device)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b19692c3", + "metadata": {}, + "outputs": [], + "source": [ + "im1 = img_to_tensor(image1).to(device).unsqueeze(0)\n", + "im2 = img_to_tensor(image2).to(device).unsqueeze(0)\n", + "with torch.inference_mode():\n", + " pred, _, _ = tiled_pred(model, None, im1, im2, None, conf_mode=tile_conf_mode, overlap=tile_overlap, crop=cropsize, with_conf=with_conf, return_time=False)\n", + "pred = pred.squeeze(0).permute(1,2,0).cpu().numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26f79db3", + "metadata": {}, + "outputs": [], + "source": [ + "plt.imshow(flowToColor(pred))\n", + "plt.axis('off')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/submodules/mast3r/dust3r/croco/datasets/__init__.py b/submodules/mast3r/dust3r/croco/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/submodules/mast3r/dust3r/croco/datasets/crops/README.MD b/submodules/mast3r/dust3r/croco/datasets/crops/README.MD new file mode 100644 index 0000000000000000000000000000000000000000..47ddabebb177644694ee247ae878173a3a16644f --- /dev/null +++ b/submodules/mast3r/dust3r/croco/datasets/crops/README.MD @@ -0,0 +1,104 @@ +## Generation of crops from the real datasets + +The instructions below allow to generate the crops used for pre-training CroCo v2 from the following real-world datasets: ARKitScenes, MegaDepth, 3DStreetView and IndoorVL. + +### Download the metadata of the crops to generate + +First, download the metadata and put them in `./data/`: +``` +mkdir -p data +cd data/ +wget https://download.europe.naverlabs.com/ComputerVision/CroCo/data/crop_metadata.zip +unzip crop_metadata.zip +rm crop_metadata.zip +cd .. +``` + +### Prepare the original datasets + +Second, download the original datasets in `./data/original_datasets/`. +``` +mkdir -p data/original_datasets +``` + +##### ARKitScenes + +Download the `raw` dataset from https://github.com/apple/ARKitScenes/blob/main/DATA.md and put it in `./data/original_datasets/ARKitScenes/`. +The resulting file structure should be like: +``` +./data/original_datasets/ARKitScenes/ +└───Training + └───40753679 + │ │ ultrawide + │ │ ... + └───40753686 + │ + ... +``` + +##### MegaDepth + +Download `MegaDepth v1 Dataset` from https://www.cs.cornell.edu/projects/megadepth/ and put it in `./data/original_datasets/MegaDepth/`. +The resulting file structure should be like: + +``` +./data/original_datasets/MegaDepth/ +└───0000 +│ └───images +│ │ │ 1000557903_87fa96b8a4_o.jpg +│ │ └ ... +│ └─── ... +└───0001 +│ │ +│ └ ... +└─── ... +``` + +##### 3DStreetView + +Download `3D_Street_View` dataset from https://github.com/amir32002/3D_Street_View and put it in `./data/original_datasets/3DStreetView/`. +The resulting file structure should be like: + +``` +./data/original_datasets/3DStreetView/ +└───dataset_aligned +│ └───0002 +│ │ │ 0000002_0000001_0000002_0000001.jpg +│ │ └ ... +│ └─── ... +└───dataset_unaligned +│ └───0003 +│ │ │ 0000003_0000001_0000002_0000001.jpg +│ │ └ ... +│ └─── ... +``` + +##### IndoorVL + +Download the `IndoorVL` datasets using [Kapture](https://github.com/naver/kapture). + +``` +pip install kapture +mkdir -p ./data/original_datasets/IndoorVL +cd ./data/original_datasets/IndoorVL +kapture_download_dataset.py update +kapture_download_dataset.py install "HyundaiDepartmentStore_*" +kapture_download_dataset.py install "GangnamStation_*" +cd - +``` + +### Extract the crops + +Now, extract the crops for each of the dataset: +``` +for dataset in ARKitScenes MegaDepth 3DStreetView IndoorVL; +do + python3 datasets/crops/extract_crops_from_images.py --crops ./data/crop_metadata/${dataset}/crops_release.txt --root-dir ./data/original_datasets/${dataset}/ --output-dir ./data/${dataset}_crops/ --imsize 256 --nthread 8 --max-subdir-levels 5 --ideal-number-pairs-in-dir 500; +done +``` + +##### Note for IndoorVL + +Due to some legal issues, we can only release 144,228 pairs out of the 1,593,689 pairs used in the paper. +To account for it in terms of number of pre-training iterations, the pre-training command in this repository uses 125 training epochs including 12 warm-up epochs and learning rate cosine schedule of 250, instead of 100, 10 and 200 respectively. +The impact on the performance is negligible. diff --git a/submodules/mast3r/dust3r/croco/datasets/crops/extract_crops_from_images.py b/submodules/mast3r/dust3r/croco/datasets/crops/extract_crops_from_images.py new file mode 100644 index 0000000000000000000000000000000000000000..eb66a0474ce44b54c44c08887cbafdb045b11ff3 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/datasets/crops/extract_crops_from_images.py @@ -0,0 +1,159 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Extracting crops for pre-training +# -------------------------------------------------------- + +import os +import argparse +from tqdm import tqdm +from PIL import Image +import functools +from multiprocessing import Pool +import math + + +def arg_parser(): + parser = argparse.ArgumentParser('Generate cropped image pairs from image crop list') + + parser.add_argument('--crops', type=str, required=True, help='crop file') + parser.add_argument('--root-dir', type=str, required=True, help='root directory') + parser.add_argument('--output-dir', type=str, required=True, help='output directory') + parser.add_argument('--imsize', type=int, default=256, help='size of the crops') + parser.add_argument('--nthread', type=int, required=True, help='number of simultaneous threads') + parser.add_argument('--max-subdir-levels', type=int, default=5, help='maximum number of subdirectories') + parser.add_argument('--ideal-number-pairs-in-dir', type=int, default=500, help='number of pairs stored in a dir') + return parser + + +def main(args): + listing_path = os.path.join(args.output_dir, 'listing.txt') + + print(f'Loading list of crops ... ({args.nthread} threads)') + crops, num_crops_to_generate = load_crop_file(args.crops) + + print(f'Preparing jobs ({len(crops)} candidate image pairs)...') + num_levels = min(math.ceil(math.log(num_crops_to_generate, args.ideal_number_pairs_in_dir)), args.max_subdir_levels) + num_pairs_in_dir = math.ceil(num_crops_to_generate ** (1/num_levels)) + + jobs = prepare_jobs(crops, num_levels, num_pairs_in_dir) + del crops + + os.makedirs(args.output_dir, exist_ok=True) + mmap = Pool(args.nthread).imap_unordered if args.nthread > 1 else map + call = functools.partial(save_image_crops, args) + + print(f"Generating cropped images to {args.output_dir} ...") + with open(listing_path, 'w') as listing: + listing.write('# pair_path\n') + for results in tqdm(mmap(call, jobs), total=len(jobs)): + for path in results: + listing.write(f'{path}\n') + print('Finished writing listing to', listing_path) + + +def load_crop_file(path): + data = open(path).read().splitlines() + pairs = [] + num_crops_to_generate = 0 + for line in tqdm(data): + if line.startswith('#'): + continue + line = line.split(', ') + if len(line) < 8: + img1, img2, rotation = line + pairs.append((img1, img2, int(rotation), [])) + else: + l1, r1, t1, b1, l2, r2, t2, b2 = map(int, line) + rect1, rect2 = (l1, t1, r1, b1), (l2, t2, r2, b2) + pairs[-1][-1].append((rect1, rect2)) + num_crops_to_generate += 1 + return pairs, num_crops_to_generate + + +def prepare_jobs(pairs, num_levels, num_pairs_in_dir): + jobs = [] + powers = [num_pairs_in_dir**level for level in reversed(range(num_levels))] + + def get_path(idx): + idx_array = [] + d = idx + for level in range(num_levels - 1): + idx_array.append(idx // powers[level]) + idx = idx % powers[level] + idx_array.append(d) + return '/'.join(map(lambda x: hex(x)[2:], idx_array)) + + idx = 0 + for pair_data in tqdm(pairs): + img1, img2, rotation, crops = pair_data + if -60 <= rotation and rotation <= 60: + rotation = 0 # most likely not a true rotation + paths = [get_path(idx + k) for k in range(len(crops))] + idx += len(crops) + jobs.append(((img1, img2), rotation, crops, paths)) + return jobs + + +def load_image(path): + try: + return Image.open(path).convert('RGB') + except Exception as e: + print('skipping', path, e) + raise OSError() + + +def save_image_crops(args, data): + # load images + img_pair, rot, crops, paths = data + try: + img1, img2 = [load_image(os.path.join(args.root_dir, impath)) for impath in img_pair] + except OSError as e: + return [] + + def area(sz): + return sz[0] * sz[1] + + tgt_size = (args.imsize, args.imsize) + + def prepare_crop(img, rect, rot=0): + # actual crop + img = img.crop(rect) + + # resize to desired size + interp = Image.Resampling.LANCZOS if area(img.size) > 4*area(tgt_size) else Image.Resampling.BICUBIC + img = img.resize(tgt_size, resample=interp) + + # rotate the image + rot90 = (round(rot/90) % 4) * 90 + if rot90 == 90: + img = img.transpose(Image.Transpose.ROTATE_90) + elif rot90 == 180: + img = img.transpose(Image.Transpose.ROTATE_180) + elif rot90 == 270: + img = img.transpose(Image.Transpose.ROTATE_270) + return img + + results = [] + for (rect1, rect2), path in zip(crops, paths): + crop1 = prepare_crop(img1, rect1) + crop2 = prepare_crop(img2, rect2, rot) + + fullpath1 = os.path.join(args.output_dir, path+'_1.jpg') + fullpath2 = os.path.join(args.output_dir, path+'_2.jpg') + os.makedirs(os.path.dirname(fullpath1), exist_ok=True) + + assert not os.path.isfile(fullpath1), fullpath1 + assert not os.path.isfile(fullpath2), fullpath2 + crop1.save(fullpath1) + crop2.save(fullpath2) + results.append(path) + + return results + + +if __name__ == '__main__': + args = arg_parser().parse_args() + main(args) + diff --git a/submodules/mast3r/dust3r/croco/datasets/habitat_sim/README.MD b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/README.MD new file mode 100644 index 0000000000000000000000000000000000000000..a505781ff9eb91bce7f1d189e848f8ba1c560940 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/README.MD @@ -0,0 +1,76 @@ +## Generation of synthetic image pairs using Habitat-Sim + +These instructions allow to generate pre-training pairs from the Habitat simulator. +As we did not save metadata of the pairs used in the original paper, they are not strictly the same, but these data use the same setting and are equivalent. + +### Download Habitat-Sim scenes +Download Habitat-Sim scenes: +- Download links can be found here: https://github.com/facebookresearch/habitat-sim/blob/main/DATASETS.md +- We used scenes from the HM3D, habitat-test-scenes, Replica, ReplicaCad and ScanNet datasets. +- Please put the scenes under `./data/habitat-sim-data/scene_datasets/` following the structure below, or update manually paths in `paths.py`. +``` +./data/ +└──habitat-sim-data/ + └──scene_datasets/ + ├──hm3d/ + ├──gibson/ + ├──habitat-test-scenes/ + ├──replica_cad_baked_lighting/ + ├──replica_cad/ + ├──ReplicaDataset/ + └──scannet/ +``` + +### Image pairs generation +We provide metadata to generate reproducible images pairs for pretraining and validation. +Experiments described in the paper used similar data, but whose generation was not reproducible at the time. + +Specifications: +- 256x256 resolution images, with 60 degrees field of view . +- Up to 1000 image pairs per scene. +- Number of scenes considered/number of images pairs per dataset: + - Scannet: 1097 scenes / 985 209 pairs + - HM3D: + - hm3d/train: 800 / 800k pairs + - hm3d/val: 100 scenes / 100k pairs + - hm3d/minival: 10 scenes / 10k pairs + - habitat-test-scenes: 3 scenes / 3k pairs + - replica_cad_baked_lighting: 13 scenes / 13k pairs + +- Scenes from hm3d/val and hm3d/minival pairs were not used for the pre-training but kept for validation purposes. + +Download metadata and extract it: +```bash +mkdir -p data/habitat_release_metadata/ +cd data/habitat_release_metadata/ +wget https://download.europe.naverlabs.com/ComputerVision/CroCo/data/habitat_release_metadata/multiview_habitat_metadata.tar.gz +tar -xvf multiview_habitat_metadata.tar.gz +cd ../.. +# Location of the metadata +METADATA_DIR="./data/habitat_release_metadata/multiview_habitat_metadata" +``` + +Generate image pairs from metadata: +- The following command will print a list of commandlines to generate image pairs for each scene: +```bash +# Target output directory +PAIRS_DATASET_DIR="./data/habitat_release/" +python datasets/habitat_sim/generate_from_metadata_files.py --input_dir=$METADATA_DIR --output_dir=$PAIRS_DATASET_DIR +``` +- One can launch multiple of such commands in parallel e.g. using GNU Parallel: +```bash +python datasets/habitat_sim/generate_from_metadata_files.py --input_dir=$METADATA_DIR --output_dir=$PAIRS_DATASET_DIR | parallel -j 16 +``` + +## Metadata generation + +Image pairs were randomly sampled using the following commands, whose outputs contain randomness and are thus not exactly reproducible: +```bash +# Print commandlines to generate image pairs from the different scenes available. +PAIRS_DATASET_DIR=MY_CUSTOM_PATH +python datasets/habitat_sim/generate_multiview_images.py --list_commands --output_dir=$PAIRS_DATASET_DIR + +# Once a dataset is generated, pack metadata files for reproducibility. +METADATA_DIR=MY_CUSTON_PATH +python datasets/habitat_sim/pack_metadata_files.py $PAIRS_DATASET_DIR $METADATA_DIR +``` diff --git a/submodules/mast3r/dust3r/croco/datasets/habitat_sim/__init__.py b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/submodules/mast3r/dust3r/croco/datasets/habitat_sim/generate_from_metadata.py b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/generate_from_metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..fbe0d399084359495250dc8184671ff498adfbf2 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/generate_from_metadata.py @@ -0,0 +1,92 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +""" +Script to generate image pairs for a given scene reproducing poses provided in a metadata file. +""" +import os +from datasets.habitat_sim.multiview_habitat_sim_generator import MultiviewHabitatSimGenerator +from datasets.habitat_sim.paths import SCENES_DATASET +import argparse +import quaternion +import PIL.Image +import cv2 +import json +from tqdm import tqdm + +def generate_multiview_images_from_metadata(metadata_filename, + output_dir, + overload_params = dict(), + scene_datasets_paths=None, + exist_ok=False): + """ + Generate images from a metadata file for reproducibility purposes. + """ + # Reorder paths by decreasing label length, to avoid collisions when testing if a string by such label + if scene_datasets_paths is not None: + scene_datasets_paths = dict(sorted(scene_datasets_paths.items(), key= lambda x: len(x[0]), reverse=True)) + + with open(metadata_filename, 'r') as f: + input_metadata = json.load(f) + metadata = dict() + for key, value in input_metadata.items(): + # Optionally replace some paths + if key in ("scene_dataset_config_file", "scene", "navmesh") and value != "": + if scene_datasets_paths is not None: + for dataset_label, dataset_path in scene_datasets_paths.items(): + if value.startswith(dataset_label): + value = os.path.normpath(os.path.join(dataset_path, os.path.relpath(value, dataset_label))) + break + metadata[key] = value + + # Overload some parameters + for key, value in overload_params.items(): + metadata[key] = value + + generation_entries = dict([(key, value) for key, value in metadata.items() if not (key in ('multiviews', 'output_dir', 'generate_depth'))]) + generate_depth = metadata["generate_depth"] + + os.makedirs(output_dir, exist_ok=exist_ok) + + generator = MultiviewHabitatSimGenerator(**generation_entries) + + # Generate views + for idx_label, data in tqdm(metadata['multiviews'].items()): + positions = data["positions"] + orientations = data["orientations"] + n = len(positions) + for oidx in range(n): + observation = generator.render_viewpoint(positions[oidx], quaternion.from_float_array(orientations[oidx])) + observation_label = f"{oidx + 1}" # Leonid is indexing starting from 1 + # Color image saved using PIL + img = PIL.Image.fromarray(observation['color'][:,:,:3]) + filename = os.path.join(output_dir, f"{idx_label}_{observation_label}.jpeg") + img.save(filename) + if generate_depth: + # Depth image as EXR file + filename = os.path.join(output_dir, f"{idx_label}_{observation_label}_depth.exr") + cv2.imwrite(filename, observation['depth'], [cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_HALF]) + # Camera parameters + camera_params = dict([(key, observation[key].tolist()) for key in ("camera_intrinsics", "R_cam2world", "t_cam2world")]) + filename = os.path.join(output_dir, f"{idx_label}_{observation_label}_camera_params.json") + with open(filename, "w") as f: + json.dump(camera_params, f) + # Save metadata + with open(os.path.join(output_dir, "metadata.json"), "w") as f: + json.dump(metadata, f) + + generator.close() + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--metadata_filename", required=True) + parser.add_argument("--output_dir", required=True) + args = parser.parse_args() + + generate_multiview_images_from_metadata(metadata_filename=args.metadata_filename, + output_dir=args.output_dir, + scene_datasets_paths=SCENES_DATASET, + overload_params=dict(), + exist_ok=True) + + \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/datasets/habitat_sim/generate_from_metadata_files.py b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/generate_from_metadata_files.py new file mode 100644 index 0000000000000000000000000000000000000000..962ef849d8c31397b8622df4f2d9140175d78873 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/generate_from_metadata_files.py @@ -0,0 +1,27 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +""" +Script generating commandlines to generate image pairs from metadata files. +""" +import os +import glob +from tqdm import tqdm +import argparse + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--input_dir", required=True) + parser.add_argument("--output_dir", required=True) + parser.add_argument("--prefix", default="", help="Commanline prefix, useful e.g. to setup environment.") + args = parser.parse_args() + + input_metadata_filenames = glob.iglob(f"{args.input_dir}/**/metadata.json", recursive=True) + + for metadata_filename in tqdm(input_metadata_filenames): + output_dir = os.path.join(args.output_dir, os.path.relpath(os.path.dirname(metadata_filename), args.input_dir)) + # Do not process the scene if the metadata file already exists + if os.path.exists(os.path.join(output_dir, "metadata.json")): + continue + commandline = f"{args.prefix}python datasets/habitat_sim/generate_from_metadata.py --metadata_filename={metadata_filename} --output_dir={output_dir}" + print(commandline) diff --git a/submodules/mast3r/dust3r/croco/datasets/habitat_sim/generate_multiview_images.py b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/generate_multiview_images.py new file mode 100644 index 0000000000000000000000000000000000000000..421d49a1696474415940493296b3f2d982398850 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/generate_multiview_images.py @@ -0,0 +1,177 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +import os +from tqdm import tqdm +import argparse +import PIL.Image +import numpy as np +import json +from datasets.habitat_sim.multiview_habitat_sim_generator import MultiviewHabitatSimGenerator, NoNaviguableSpaceError +from datasets.habitat_sim.paths import list_scenes_available +import cv2 +import quaternion +import shutil + +def generate_multiview_images_for_scene(scene_dataset_config_file, + scene, + navmesh, + output_dir, + views_count, + size, + exist_ok=False, + generate_depth=False, + **kwargs): + """ + Generate tuples of overlapping views for a given scene. + generate_depth: generate depth images and camera parameters. + """ + if os.path.exists(output_dir) and not exist_ok: + print(f"Scene {scene}: data already generated. Ignoring generation.") + return + try: + print(f"Scene {scene}: {size} multiview acquisitions to generate...") + os.makedirs(output_dir, exist_ok=exist_ok) + + metadata_filename = os.path.join(output_dir, "metadata.json") + + metadata_template = dict(scene_dataset_config_file=scene_dataset_config_file, + scene=scene, + navmesh=navmesh, + views_count=views_count, + size=size, + generate_depth=generate_depth, + **kwargs) + metadata_template["multiviews"] = dict() + + if os.path.exists(metadata_filename): + print("Metadata file already exists:", metadata_filename) + print("Loading already generated metadata file...") + with open(metadata_filename, "r") as f: + metadata = json.load(f) + + for key in metadata_template.keys(): + if key != "multiviews": + assert metadata_template[key] == metadata[key], f"existing file is inconsistent with the input parameters:\nKey: {key}\nmetadata: {metadata[key]}\ntemplate: {metadata_template[key]}." + else: + print("No temporary file found. Starting generation from scratch...") + metadata = metadata_template + + starting_id = len(metadata["multiviews"]) + print(f"Starting generation from index {starting_id}/{size}...") + if starting_id >= size: + print("Generation already done.") + return + + generator = MultiviewHabitatSimGenerator(scene_dataset_config_file=scene_dataset_config_file, + scene=scene, + navmesh=navmesh, + views_count = views_count, + size = size, + **kwargs) + + for idx in tqdm(range(starting_id, size)): + # Generate / re-generate the observations + try: + data = generator[idx] + observations = data["observations"] + positions = data["positions"] + orientations = data["orientations"] + + idx_label = f"{idx:08}" + for oidx, observation in enumerate(observations): + observation_label = f"{oidx + 1}" # Leonid is indexing starting from 1 + # Color image saved using PIL + img = PIL.Image.fromarray(observation['color'][:,:,:3]) + filename = os.path.join(output_dir, f"{idx_label}_{observation_label}.jpeg") + img.save(filename) + if generate_depth: + # Depth image as EXR file + filename = os.path.join(output_dir, f"{idx_label}_{observation_label}_depth.exr") + cv2.imwrite(filename, observation['depth'], [cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_HALF]) + # Camera parameters + camera_params = dict([(key, observation[key].tolist()) for key in ("camera_intrinsics", "R_cam2world", "t_cam2world")]) + filename = os.path.join(output_dir, f"{idx_label}_{observation_label}_camera_params.json") + with open(filename, "w") as f: + json.dump(camera_params, f) + metadata["multiviews"][idx_label] = {"positions": positions.tolist(), + "orientations": orientations.tolist(), + "covisibility_ratios": data["covisibility_ratios"].tolist(), + "valid_fractions": data["valid_fractions"].tolist(), + "pairwise_visibility_ratios": data["pairwise_visibility_ratios"].tolist()} + except RecursionError: + print("Recursion error: unable to sample observations for this scene. We will stop there.") + break + + # Regularly save a temporary metadata file, in case we need to restart the generation + if idx % 10 == 0: + with open(metadata_filename, "w") as f: + json.dump(metadata, f) + + # Save metadata + with open(metadata_filename, "w") as f: + json.dump(metadata, f) + + generator.close() + except NoNaviguableSpaceError: + pass + +def create_commandline(scene_data, generate_depth, exist_ok=False): + """ + Create a commandline string to generate a scene. + """ + def my_formatting(val): + if val is None or val == "": + return '""' + else: + return val + commandline = f"""python {__file__} --scene {my_formatting(scene_data.scene)} + --scene_dataset_config_file {my_formatting(scene_data.scene_dataset_config_file)} + --navmesh {my_formatting(scene_data.navmesh)} + --output_dir {my_formatting(scene_data.output_dir)} + --generate_depth {int(generate_depth)} + --exist_ok {int(exist_ok)} + """ + commandline = " ".join(commandline.split()) + return commandline + +if __name__ == "__main__": + os.umask(2) + + parser = argparse.ArgumentParser(description="""Example of use -- listing commands to generate data for scenes available: + > python datasets/habitat_sim/generate_multiview_habitat_images.py --list_commands + """) + + parser.add_argument("--output_dir", type=str, required=True) + parser.add_argument("--list_commands", action='store_true', help="list commandlines to run if true") + parser.add_argument("--scene", type=str, default="") + parser.add_argument("--scene_dataset_config_file", type=str, default="") + parser.add_argument("--navmesh", type=str, default="") + + parser.add_argument("--generate_depth", type=int, default=1) + parser.add_argument("--exist_ok", type=int, default=0) + + kwargs = dict(resolution=(256,256), hfov=60, views_count = 2, size=1000) + + args = parser.parse_args() + generate_depth=bool(args.generate_depth) + exist_ok = bool(args.exist_ok) + + if args.list_commands: + # Listing scenes available... + scenes_data = list_scenes_available(base_output_dir=args.output_dir) + + for scene_data in scenes_data: + print(create_commandline(scene_data, generate_depth=generate_depth, exist_ok=exist_ok)) + else: + if args.scene == "" or args.output_dir == "": + print("Missing scene or output dir argument!") + print(parser.format_help()) + else: + generate_multiview_images_for_scene(scene=args.scene, + scene_dataset_config_file = args.scene_dataset_config_file, + navmesh = args.navmesh, + output_dir = args.output_dir, + exist_ok=exist_ok, + generate_depth=generate_depth, + **kwargs) \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/datasets/habitat_sim/multiview_habitat_sim_generator.py b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/multiview_habitat_sim_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..91e5f923b836a645caf5d8e4aacc425047e3c144 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/multiview_habitat_sim_generator.py @@ -0,0 +1,390 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +import os +import numpy as np +import quaternion +import habitat_sim +import json +from sklearn.neighbors import NearestNeighbors +import cv2 + +# OpenCV to habitat camera convention transformation +R_OPENCV2HABITAT = np.stack((habitat_sim.geo.RIGHT, -habitat_sim.geo.UP, habitat_sim.geo.FRONT), axis=0) +R_HABITAT2OPENCV = R_OPENCV2HABITAT.T +DEG2RAD = np.pi / 180 + +def compute_camera_intrinsics(height, width, hfov): + f = width/2 / np.tan(hfov/2 * np.pi/180) + cu, cv = width/2, height/2 + return f, cu, cv + +def compute_camera_pose_opencv_convention(camera_position, camera_orientation): + R_cam2world = quaternion.as_rotation_matrix(camera_orientation) @ R_OPENCV2HABITAT + t_cam2world = np.asarray(camera_position) + return R_cam2world, t_cam2world + +def compute_pointmap(depthmap, hfov): + """ Compute a HxWx3 pointmap in camera frame from a HxW depth map.""" + height, width = depthmap.shape + f, cu, cv = compute_camera_intrinsics(height, width, hfov) + # Cast depth map to point + z_cam = depthmap + u, v = np.meshgrid(range(width), range(height)) + x_cam = (u - cu) / f * z_cam + y_cam = (v - cv) / f * z_cam + X_cam = np.stack((x_cam, y_cam, z_cam), axis=-1) + return X_cam + +def compute_pointcloud(depthmap, hfov, camera_position, camera_rotation): + """Return a 3D point cloud corresponding to valid pixels of the depth map""" + R_cam2world, t_cam2world = compute_camera_pose_opencv_convention(camera_position, camera_rotation) + + X_cam = compute_pointmap(depthmap=depthmap, hfov=hfov) + valid_mask = (X_cam[:,:,2] != 0.0) + + X_cam = X_cam.reshape(-1, 3)[valid_mask.flatten()] + X_world = X_cam @ R_cam2world.T + t_cam2world.reshape(1, 3) + return X_world + +def compute_pointcloud_overlaps_scikit(pointcloud1, pointcloud2, distance_threshold, compute_symmetric=False): + """ + Compute 'overlapping' metrics based on a distance threshold between two point clouds. + """ + nbrs = NearestNeighbors(n_neighbors=1, algorithm = 'kd_tree').fit(pointcloud2) + distances, indices = nbrs.kneighbors(pointcloud1) + intersection1 = np.count_nonzero(distances.flatten() < distance_threshold) + + data = {"intersection1": intersection1, + "size1": len(pointcloud1)} + if compute_symmetric: + nbrs = NearestNeighbors(n_neighbors=1, algorithm = 'kd_tree').fit(pointcloud1) + distances, indices = nbrs.kneighbors(pointcloud2) + intersection2 = np.count_nonzero(distances.flatten() < distance_threshold) + data["intersection2"] = intersection2 + data["size2"] = len(pointcloud2) + + return data + +def _append_camera_parameters(observation, hfov, camera_location, camera_rotation): + """ + Add camera parameters to the observation dictionnary produced by Habitat-Sim + In-place modifications. + """ + R_cam2world, t_cam2world = compute_camera_pose_opencv_convention(camera_location, camera_rotation) + height, width = observation['depth'].shape + f, cu, cv = compute_camera_intrinsics(height, width, hfov) + K = np.asarray([[f, 0, cu], + [0, f, cv], + [0, 0, 1.0]]) + observation["camera_intrinsics"] = K + observation["t_cam2world"] = t_cam2world + observation["R_cam2world"] = R_cam2world + +def look_at(eye, center, up, return_cam2world=True): + """ + Return camera pose looking at a given center point. + Analogous of gluLookAt function, using OpenCV camera convention. + """ + z = center - eye + z /= np.linalg.norm(z, axis=-1, keepdims=True) + y = -up + y = y - np.sum(y * z, axis=-1, keepdims=True) * z + y /= np.linalg.norm(y, axis=-1, keepdims=True) + x = np.cross(y, z, axis=-1) + + if return_cam2world: + R = np.stack((x, y, z), axis=-1) + t = eye + else: + # World to camera transformation + # Transposed matrix + R = np.stack((x, y, z), axis=-2) + t = - np.einsum('...ij, ...j', R, eye) + return R, t + +def look_at_for_habitat(eye, center, up, return_cam2world=True): + R, t = look_at(eye, center, up) + orientation = quaternion.from_rotation_matrix(R @ R_OPENCV2HABITAT.T) + return orientation, t + +def generate_orientation_noise(pan_range, tilt_range, roll_range): + return (quaternion.from_rotation_vector(np.random.uniform(*pan_range) * DEG2RAD * habitat_sim.geo.UP) + * quaternion.from_rotation_vector(np.random.uniform(*tilt_range) * DEG2RAD * habitat_sim.geo.RIGHT) + * quaternion.from_rotation_vector(np.random.uniform(*roll_range) * DEG2RAD * habitat_sim.geo.FRONT)) + + +class NoNaviguableSpaceError(RuntimeError): + def __init__(self, *args): + super().__init__(*args) + +class MultiviewHabitatSimGenerator: + def __init__(self, + scene, + navmesh, + scene_dataset_config_file, + resolution = (240, 320), + views_count=2, + hfov = 60, + gpu_id = 0, + size = 10000, + minimum_covisibility = 0.5, + transform = None): + self.scene = scene + self.navmesh = navmesh + self.scene_dataset_config_file = scene_dataset_config_file + self.resolution = resolution + self.views_count = views_count + assert(self.views_count >= 1) + self.hfov = hfov + self.gpu_id = gpu_id + self.size = size + self.transform = transform + + # Noise added to camera orientation + self.pan_range = (-3, 3) + self.tilt_range = (-10, 10) + self.roll_range = (-5, 5) + + # Height range to sample cameras + self.height_range = (1.2, 1.8) + + # Random steps between the camera views + self.random_steps_count = 5 + self.random_step_variance = 2.0 + + # Minimum fraction of the scene which should be valid (well defined depth) + self.minimum_valid_fraction = 0.7 + + # Distance threshold to see to select pairs + self.distance_threshold = 0.05 + # Minimum IoU of a view point cloud with respect to the reference view to be kept. + self.minimum_covisibility = minimum_covisibility + + # Maximum number of retries. + self.max_attempts_count = 100 + + self.seed = None + self._lazy_initialization() + + def _lazy_initialization(self): + # Lazy random seeding and instantiation of the simulator to deal with multiprocessing properly + if self.seed == None: + # Re-seed numpy generator + np.random.seed() + self.seed = np.random.randint(2**32-1) + sim_cfg = habitat_sim.SimulatorConfiguration() + sim_cfg.scene_id = self.scene + if self.scene_dataset_config_file is not None and self.scene_dataset_config_file != "": + sim_cfg.scene_dataset_config_file = self.scene_dataset_config_file + sim_cfg.random_seed = self.seed + sim_cfg.load_semantic_mesh = False + sim_cfg.gpu_device_id = self.gpu_id + + depth_sensor_spec = habitat_sim.CameraSensorSpec() + depth_sensor_spec.uuid = "depth" + depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH + depth_sensor_spec.resolution = self.resolution + depth_sensor_spec.hfov = self.hfov + depth_sensor_spec.position = [0.0, 0.0, 0] + depth_sensor_spec.orientation + + rgb_sensor_spec = habitat_sim.CameraSensorSpec() + rgb_sensor_spec.uuid = "color" + rgb_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR + rgb_sensor_spec.resolution = self.resolution + rgb_sensor_spec.hfov = self.hfov + rgb_sensor_spec.position = [0.0, 0.0, 0] + agent_cfg = habitat_sim.agent.AgentConfiguration(sensor_specifications=[rgb_sensor_spec, depth_sensor_spec]) + + cfg = habitat_sim.Configuration(sim_cfg, [agent_cfg]) + self.sim = habitat_sim.Simulator(cfg) + if self.navmesh is not None and self.navmesh != "": + # Use pre-computed navmesh when available (usually better than those generated automatically) + self.sim.pathfinder.load_nav_mesh(self.navmesh) + + if not self.sim.pathfinder.is_loaded: + # Try to compute a navmesh + navmesh_settings = habitat_sim.NavMeshSettings() + navmesh_settings.set_defaults() + self.sim.recompute_navmesh(self.sim.pathfinder, navmesh_settings, True) + + # Ensure that the navmesh is not empty + if not self.sim.pathfinder.is_loaded: + raise NoNaviguableSpaceError(f"No naviguable location (scene: {self.scene} -- navmesh: {self.navmesh})") + + self.agent = self.sim.initialize_agent(agent_id=0) + + def close(self): + self.sim.close() + + def __del__(self): + self.sim.close() + + def __len__(self): + return self.size + + def sample_random_viewpoint(self): + """ Sample a random viewpoint using the navmesh """ + nav_point = self.sim.pathfinder.get_random_navigable_point() + + # Sample a random viewpoint height + viewpoint_height = np.random.uniform(*self.height_range) + viewpoint_position = nav_point + viewpoint_height * habitat_sim.geo.UP + viewpoint_orientation = quaternion.from_rotation_vector(np.random.uniform(0, 2 * np.pi) * habitat_sim.geo.UP) * generate_orientation_noise(self.pan_range, self.tilt_range, self.roll_range) + return viewpoint_position, viewpoint_orientation, nav_point + + def sample_other_random_viewpoint(self, observed_point, nav_point): + """ Sample a random viewpoint close to an existing one, using the navmesh and a reference observed point.""" + other_nav_point = nav_point + + walk_directions = self.random_step_variance * np.asarray([1,0,1]) + for i in range(self.random_steps_count): + temp = self.sim.pathfinder.snap_point(other_nav_point + walk_directions * np.random.normal(size=3)) + # Snapping may return nan when it fails + if not np.isnan(temp[0]): + other_nav_point = temp + + other_viewpoint_height = np.random.uniform(*self.height_range) + other_viewpoint_position = other_nav_point + other_viewpoint_height * habitat_sim.geo.UP + + # Set viewing direction towards the central point + rotation, position = look_at_for_habitat(eye=other_viewpoint_position, center=observed_point, up=habitat_sim.geo.UP, return_cam2world=True) + rotation = rotation * generate_orientation_noise(self.pan_range, self.tilt_range, self.roll_range) + return position, rotation, other_nav_point + + def is_other_pointcloud_overlapping(self, ref_pointcloud, other_pointcloud): + """ Check if a viewpoint is valid and overlaps significantly with a reference one. """ + # Observation + pixels_count = self.resolution[0] * self.resolution[1] + valid_fraction = len(other_pointcloud) / pixels_count + assert valid_fraction <= 1.0 and valid_fraction >= 0.0 + overlap = compute_pointcloud_overlaps_scikit(ref_pointcloud, other_pointcloud, self.distance_threshold, compute_symmetric=True) + covisibility = min(overlap["intersection1"] / pixels_count, overlap["intersection2"] / pixels_count) + is_valid = (valid_fraction >= self.minimum_valid_fraction) and (covisibility >= self.minimum_covisibility) + return is_valid, valid_fraction, covisibility + + def is_other_viewpoint_overlapping(self, ref_pointcloud, observation, position, rotation): + """ Check if a viewpoint is valid and overlaps significantly with a reference one. """ + # Observation + other_pointcloud = compute_pointcloud(observation['depth'], self.hfov, position, rotation) + return self.is_other_pointcloud_overlapping(ref_pointcloud, other_pointcloud) + + def render_viewpoint(self, viewpoint_position, viewpoint_orientation): + agent_state = habitat_sim.AgentState() + agent_state.position = viewpoint_position + agent_state.rotation = viewpoint_orientation + self.agent.set_state(agent_state) + viewpoint_observations = self.sim.get_sensor_observations(agent_ids=0) + _append_camera_parameters(viewpoint_observations, self.hfov, viewpoint_position, viewpoint_orientation) + return viewpoint_observations + + def __getitem__(self, useless_idx): + ref_position, ref_orientation, nav_point = self.sample_random_viewpoint() + ref_observations = self.render_viewpoint(ref_position, ref_orientation) + # Extract point cloud + ref_pointcloud = compute_pointcloud(depthmap=ref_observations['depth'], hfov=self.hfov, + camera_position=ref_position, camera_rotation=ref_orientation) + + pixels_count = self.resolution[0] * self.resolution[1] + ref_valid_fraction = len(ref_pointcloud) / pixels_count + assert ref_valid_fraction <= 1.0 and ref_valid_fraction >= 0.0 + if ref_valid_fraction < self.minimum_valid_fraction: + # This should produce a recursion error at some point when something is very wrong. + return self[0] + # Pick an reference observed point in the point cloud + observed_point = np.mean(ref_pointcloud, axis=0) + + # Add the first image as reference + viewpoints_observations = [ref_observations] + viewpoints_covisibility = [ref_valid_fraction] + viewpoints_positions = [ref_position] + viewpoints_orientations = [quaternion.as_float_array(ref_orientation)] + viewpoints_clouds = [ref_pointcloud] + viewpoints_valid_fractions = [ref_valid_fraction] + + for _ in range(self.views_count - 1): + # Generate an other viewpoint using some dummy random walk + successful_sampling = False + for sampling_attempt in range(self.max_attempts_count): + position, rotation, _ = self.sample_other_random_viewpoint(observed_point, nav_point) + # Observation + other_viewpoint_observations = self.render_viewpoint(position, rotation) + other_pointcloud = compute_pointcloud(other_viewpoint_observations['depth'], self.hfov, position, rotation) + + is_valid, valid_fraction, covisibility = self.is_other_pointcloud_overlapping(ref_pointcloud, other_pointcloud) + if is_valid: + successful_sampling = True + break + if not successful_sampling: + print("WARNING: Maximum number of attempts reached.") + # Dirty hack, try using a novel original viewpoint + return self[0] + viewpoints_observations.append(other_viewpoint_observations) + viewpoints_covisibility.append(covisibility) + viewpoints_positions.append(position) + viewpoints_orientations.append(quaternion.as_float_array(rotation)) # WXYZ convention for the quaternion encoding. + viewpoints_clouds.append(other_pointcloud) + viewpoints_valid_fractions.append(valid_fraction) + + # Estimate relations between all pairs of images + pairwise_visibility_ratios = np.ones((len(viewpoints_observations), len(viewpoints_observations))) + for i in range(len(viewpoints_observations)): + pairwise_visibility_ratios[i,i] = viewpoints_valid_fractions[i] + for j in range(i+1, len(viewpoints_observations)): + overlap = compute_pointcloud_overlaps_scikit(viewpoints_clouds[i], viewpoints_clouds[j], self.distance_threshold, compute_symmetric=True) + pairwise_visibility_ratios[i,j] = overlap['intersection1'] / pixels_count + pairwise_visibility_ratios[j,i] = overlap['intersection2'] / pixels_count + + # IoU is relative to the image 0 + data = {"observations": viewpoints_observations, + "positions": np.asarray(viewpoints_positions), + "orientations": np.asarray(viewpoints_orientations), + "covisibility_ratios": np.asarray(viewpoints_covisibility), + "valid_fractions": np.asarray(viewpoints_valid_fractions, dtype=float), + "pairwise_visibility_ratios": np.asarray(pairwise_visibility_ratios, dtype=float), + } + + if self.transform is not None: + data = self.transform(data) + return data + + def generate_random_spiral_trajectory(self, images_count = 100, max_radius=0.5, half_turns=5, use_constant_orientation=False): + """ + Return a list of images corresponding to a spiral trajectory from a random starting point. + Useful to generate nice visualisations. + Use an even number of half turns to get a nice "C1-continuous" loop effect + """ + ref_position, ref_orientation, navpoint = self.sample_random_viewpoint() + ref_observations = self.render_viewpoint(ref_position, ref_orientation) + ref_pointcloud = compute_pointcloud(depthmap=ref_observations['depth'], hfov=self.hfov, + camera_position=ref_position, camera_rotation=ref_orientation) + pixels_count = self.resolution[0] * self.resolution[1] + if len(ref_pointcloud) / pixels_count < self.minimum_valid_fraction: + # Dirty hack: ensure that the valid part of the image is significant + return self.generate_random_spiral_trajectory(images_count, max_radius, half_turns, use_constant_orientation) + + # Pick an observed point in the point cloud + observed_point = np.mean(ref_pointcloud, axis=0) + ref_R, ref_t = compute_camera_pose_opencv_convention(ref_position, ref_orientation) + + images = [] + is_valid = [] + # Spiral trajectory, use_constant orientation + for i, alpha in enumerate(np.linspace(0, 1, images_count)): + r = max_radius * np.abs(np.sin(alpha * np.pi)) # Increase then decrease the radius + theta = alpha * half_turns * np.pi + x = r * np.cos(theta) + y = r * np.sin(theta) + z = 0.0 + position = ref_position + (ref_R @ np.asarray([x, y, z]).reshape(3,1)).flatten() + if use_constant_orientation: + orientation = ref_orientation + else: + # trajectory looking at a mean point in front of the ref observation + orientation, position = look_at_for_habitat(eye=position, center=observed_point, up=habitat_sim.geo.UP) + observations = self.render_viewpoint(position, orientation) + images.append(observations['color'][...,:3]) + _is_valid, valid_fraction, iou = self.is_other_viewpoint_overlapping(ref_pointcloud, observations, position, orientation) + is_valid.append(_is_valid) + return images, np.all(is_valid) \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/datasets/habitat_sim/pack_metadata_files.py b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/pack_metadata_files.py new file mode 100644 index 0000000000000000000000000000000000000000..10672a01f7dd615d3b4df37781f7f6f97e753ba6 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/pack_metadata_files.py @@ -0,0 +1,69 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +""" +Utility script to pack metadata files of the dataset in order to be able to re-generate it elsewhere. +""" +import os +import glob +from tqdm import tqdm +import shutil +import json +from datasets.habitat_sim.paths import * +import argparse +import collections + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("input_dir") + parser.add_argument("output_dir") + args = parser.parse_args() + + input_dirname = args.input_dir + output_dirname = args.output_dir + + input_metadata_filenames = glob.iglob(f"{input_dirname}/**/metadata.json", recursive=True) + + images_count = collections.defaultdict(lambda : 0) + + os.makedirs(output_dirname) + for input_filename in tqdm(input_metadata_filenames): + # Ignore empty files + with open(input_filename, "r") as f: + original_metadata = json.load(f) + if "multiviews" not in original_metadata or len(original_metadata["multiviews"]) == 0: + print("No views in", input_filename) + continue + + relpath = os.path.relpath(input_filename, input_dirname) + print(relpath) + + # Copy metadata, while replacing scene paths by generic keys depending on the dataset, for portability. + # Data paths are sorted by decreasing length to avoid potential bugs due to paths starting by the same string pattern. + scenes_dataset_paths = dict(sorted(SCENES_DATASET.items(), key=lambda x: len(x[1]), reverse=True)) + metadata = dict() + for key, value in original_metadata.items(): + if key in ("scene_dataset_config_file", "scene", "navmesh") and value != "": + known_path = False + for dataset, dataset_path in scenes_dataset_paths.items(): + if value.startswith(dataset_path): + value = os.path.join(dataset, os.path.relpath(value, dataset_path)) + known_path = True + break + if not known_path: + raise KeyError("Unknown path:" + value) + metadata[key] = value + + # Compile some general statistics while packing data + scene_split = metadata["scene"].split("/") + upper_level = "/".join(scene_split[:2]) if scene_split[0] == "hm3d" else scene_split[0] + images_count[upper_level] += len(metadata["multiviews"]) + + output_filename = os.path.join(output_dirname, relpath) + os.makedirs(os.path.dirname(output_filename), exist_ok=True) + with open(output_filename, "w") as f: + json.dump(metadata, f) + + # Print statistics + print("Images count:") + for upper_level, count in images_count.items(): + print(f"- {upper_level}: {count}") \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/datasets/habitat_sim/paths.py b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/paths.py new file mode 100644 index 0000000000000000000000000000000000000000..4d63b5fa29c274ddfeae084734a35ba66d7edee8 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/datasets/habitat_sim/paths.py @@ -0,0 +1,129 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +""" +Paths to Habitat-Sim scenes +""" + +import os +import json +import collections +from tqdm import tqdm + + +# Hardcoded path to the different scene datasets +SCENES_DATASET = { + "hm3d": "./data/habitat-sim-data/scene_datasets/hm3d/", + "gibson": "./data/habitat-sim-data/scene_datasets/gibson/", + "habitat-test-scenes": "./data/habitat-sim/scene_datasets/habitat-test-scenes/", + "replica_cad_baked_lighting": "./data/habitat-sim/scene_datasets/replica_cad_baked_lighting/", + "replica_cad": "./data/habitat-sim/scene_datasets/replica_cad/", + "replica": "./data/habitat-sim/scene_datasets/ReplicaDataset/", + "scannet": "./data/habitat-sim/scene_datasets/scannet/" +} + +SceneData = collections.namedtuple("SceneData", ["scene_dataset_config_file", "scene", "navmesh", "output_dir"]) + +def list_replicacad_scenes(base_output_dir, base_path=SCENES_DATASET["replica_cad"]): + scene_dataset_config_file = os.path.join(base_path, "replicaCAD.scene_dataset_config.json") + scenes = [f"apt_{i}" for i in range(6)] + ["empty_stage"] + navmeshes = [f"navmeshes/apt_{i}_static_furniture.navmesh" for i in range(6)] + ["empty_stage.navmesh"] + scenes_data = [] + for idx in range(len(scenes)): + output_dir = os.path.join(base_output_dir, "ReplicaCAD", scenes[idx]) + # Add scene + data = SceneData(scene_dataset_config_file=scene_dataset_config_file, + scene = scenes[idx] + ".scene_instance.json", + navmesh = os.path.join(base_path, navmeshes[idx]), + output_dir = output_dir) + scenes_data.append(data) + return scenes_data + +def list_replica_cad_baked_lighting_scenes(base_output_dir, base_path=SCENES_DATASET["replica_cad_baked_lighting"]): + scene_dataset_config_file = os.path.join(base_path, "replicaCAD_baked.scene_dataset_config.json") + scenes = sum([[f"Baked_sc{i}_staging_{j:02}" for i in range(5)] for j in range(21)], []) + navmeshes = ""#[f"navmeshes/apt_{i}_static_furniture.navmesh" for i in range(6)] + ["empty_stage.navmesh"] + scenes_data = [] + for idx in range(len(scenes)): + output_dir = os.path.join(base_output_dir, "replica_cad_baked_lighting", scenes[idx]) + data = SceneData(scene_dataset_config_file=scene_dataset_config_file, + scene = scenes[idx], + navmesh = "", + output_dir = output_dir) + scenes_data.append(data) + return scenes_data + +def list_replica_scenes(base_output_dir, base_path): + scenes_data = [] + for scene_id in os.listdir(base_path): + scene = os.path.join(base_path, scene_id, "mesh.ply") + navmesh = os.path.join(base_path, scene_id, "habitat/mesh_preseg_semantic.navmesh") # Not sure if I should use it + scene_dataset_config_file = "" + output_dir = os.path.join(base_output_dir, scene_id) + # Add scene only if it does not exist already, or if exist_ok + data = SceneData(scene_dataset_config_file = scene_dataset_config_file, + scene = scene, + navmesh = navmesh, + output_dir = output_dir) + scenes_data.append(data) + return scenes_data + + +def list_scenes(base_output_dir, base_path): + """ + Generic method iterating through a base_path folder to find scenes. + """ + scenes_data = [] + for root, dirs, files in os.walk(base_path, followlinks=True): + folder_scenes_data = [] + for file in files: + name, ext = os.path.splitext(file) + if ext == ".glb": + scene = os.path.join(root, name + ".glb") + navmesh = os.path.join(root, name + ".navmesh") + if not os.path.exists(navmesh): + navmesh = "" + relpath = os.path.relpath(root, base_path) + output_dir = os.path.abspath(os.path.join(base_output_dir, relpath, name)) + data = SceneData(scene_dataset_config_file="", + scene = scene, + navmesh = navmesh, + output_dir = output_dir) + folder_scenes_data.append(data) + + # Specific check for HM3D: + # When two meshesxxxx.basis.glb and xxxx.glb are present, use the 'basis' version. + basis_scenes = [data.scene[:-len(".basis.glb")] for data in folder_scenes_data if data.scene.endswith(".basis.glb")] + if len(basis_scenes) != 0: + folder_scenes_data = [data for data in folder_scenes_data if not (data.scene[:-len(".glb")] in basis_scenes)] + + scenes_data.extend(folder_scenes_data) + return scenes_data + +def list_scenes_available(base_output_dir, scenes_dataset_paths=SCENES_DATASET): + scenes_data = [] + + # HM3D + for split in ("minival", "train", "val", "examples"): + scenes_data += list_scenes(base_output_dir=os.path.join(base_output_dir, f"hm3d/{split}/"), + base_path=f"{scenes_dataset_paths['hm3d']}/{split}") + + # Gibson + scenes_data += list_scenes(base_output_dir=os.path.join(base_output_dir, "gibson"), + base_path=scenes_dataset_paths["gibson"]) + + # Habitat test scenes (just a few) + scenes_data += list_scenes(base_output_dir=os.path.join(base_output_dir, "habitat-test-scenes"), + base_path=scenes_dataset_paths["habitat-test-scenes"]) + + # ReplicaCAD (baked lightning) + scenes_data += list_replica_cad_baked_lighting_scenes(base_output_dir=base_output_dir) + + # ScanNet + scenes_data += list_scenes(base_output_dir=os.path.join(base_output_dir, "scannet"), + base_path=scenes_dataset_paths["scannet"]) + + # Replica + list_replica_scenes(base_output_dir=os.path.join(base_output_dir, "replica"), + base_path=scenes_dataset_paths["replica"]) + return scenes_data diff --git a/submodules/mast3r/dust3r/croco/datasets/pairs_dataset.py b/submodules/mast3r/dust3r/croco/datasets/pairs_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..9f107526b34e154d9013a9a7a0bde3d5ff6f581c --- /dev/null +++ b/submodules/mast3r/dust3r/croco/datasets/pairs_dataset.py @@ -0,0 +1,109 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +import os +from torch.utils.data import Dataset +from PIL import Image + +from datasets.transforms import get_pair_transforms + +def load_image(impath): + return Image.open(impath) + +def load_pairs_from_cache_file(fname, root=''): + assert os.path.isfile(fname), "cannot parse pairs from {:s}, file does not exist".format(fname) + with open(fname, 'r') as fid: + lines = fid.read().strip().splitlines() + pairs = [ (os.path.join(root,l.split()[0]), os.path.join(root,l.split()[1])) for l in lines] + return pairs + +def load_pairs_from_list_file(fname, root=''): + assert os.path.isfile(fname), "cannot parse pairs from {:s}, file does not exist".format(fname) + with open(fname, 'r') as fid: + lines = fid.read().strip().splitlines() + pairs = [ (os.path.join(root,l+'_1.jpg'), os.path.join(root,l+'_2.jpg')) for l in lines if not l.startswith('#')] + return pairs + + +def write_cache_file(fname, pairs, root=''): + if len(root)>0: + if not root.endswith('/'): root+='/' + assert os.path.isdir(root) + s = '' + for im1, im2 in pairs: + if len(root)>0: + assert im1.startswith(root), im1 + assert im2.startswith(root), im2 + s += '{:s} {:s}\n'.format(im1[len(root):], im2[len(root):]) + with open(fname, 'w') as fid: + fid.write(s[:-1]) + +def parse_and_cache_all_pairs(dname, data_dir='./data/'): + if dname=='habitat_release': + dirname = os.path.join(data_dir, 'habitat_release') + assert os.path.isdir(dirname), "cannot find folder for habitat_release pairs: "+dirname + cache_file = os.path.join(dirname, 'pairs.txt') + assert not os.path.isfile(cache_file), "cache file already exists: "+cache_file + + print('Parsing pairs for dataset: '+dname) + pairs = [] + for root, dirs, files in os.walk(dirname): + if 'val' in root: continue + dirs.sort() + pairs += [ (os.path.join(root,f), os.path.join(root,f[:-len('_1.jpeg')]+'_2.jpeg')) for f in sorted(files) if f.endswith('_1.jpeg')] + print('Found {:,} pairs'.format(len(pairs))) + print('Writing cache to: '+cache_file) + write_cache_file(cache_file, pairs, root=dirname) + + else: + raise NotImplementedError('Unknown dataset: '+dname) + +def dnames_to_image_pairs(dnames, data_dir='./data/'): + """ + dnames: list of datasets with image pairs, separated by + + """ + all_pairs = [] + for dname in dnames.split('+'): + if dname=='habitat_release': + dirname = os.path.join(data_dir, 'habitat_release') + assert os.path.isdir(dirname), "cannot find folder for habitat_release pairs: "+dirname + cache_file = os.path.join(dirname, 'pairs.txt') + assert os.path.isfile(cache_file), "cannot find cache file for habitat_release pairs, please first create the cache file, see instructions. "+cache_file + pairs = load_pairs_from_cache_file(cache_file, root=dirname) + elif dname in ['ARKitScenes', 'MegaDepth', '3DStreetView', 'IndoorVL']: + dirname = os.path.join(data_dir, dname+'_crops') + assert os.path.isdir(dirname), "cannot find folder for {:s} pairs: {:s}".format(dname, dirname) + list_file = os.path.join(dirname, 'listing.txt') + assert os.path.isfile(list_file), "cannot find list file for {:s} pairs, see instructions. {:s}".format(dname, list_file) + pairs = load_pairs_from_list_file(list_file, root=dirname) + print(' {:s}: {:,} pairs'.format(dname, len(pairs))) + all_pairs += pairs + if '+' in dnames: print(' Total: {:,} pairs'.format(len(all_pairs))) + return all_pairs + + +class PairsDataset(Dataset): + + def __init__(self, dnames, trfs='', totensor=True, normalize=True, data_dir='./data/'): + super().__init__() + self.image_pairs = dnames_to_image_pairs(dnames, data_dir=data_dir) + self.transforms = get_pair_transforms(transform_str=trfs, totensor=totensor, normalize=normalize) + + def __len__(self): + return len(self.image_pairs) + + def __getitem__(self, index): + im1path, im2path = self.image_pairs[index] + im1 = load_image(im1path) + im2 = load_image(im2path) + if self.transforms is not None: im1, im2 = self.transforms(im1, im2) + return im1, im2 + + +if __name__=="__main__": + import argparse + parser = argparse.ArgumentParser(prog="Computing and caching list of pairs for a given dataset") + parser.add_argument('--data_dir', default='./data/', type=str, help="path where data are stored") + parser.add_argument('--dataset', default='habitat_release', type=str, help="name of the dataset") + args = parser.parse_args() + parse_and_cache_all_pairs(dname=args.dataset, data_dir=args.data_dir) diff --git a/submodules/mast3r/dust3r/croco/datasets/transforms.py b/submodules/mast3r/dust3r/croco/datasets/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..216bac61f8254fd50e7f269ee80301f250a2d11e --- /dev/null +++ b/submodules/mast3r/dust3r/croco/datasets/transforms.py @@ -0,0 +1,95 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +import torch +import torchvision.transforms +import torchvision.transforms.functional as F + +# "Pair": apply a transform on a pair +# "Both": apply the exact same transform to both images + +class ComposePair(torchvision.transforms.Compose): + def __call__(self, img1, img2): + for t in self.transforms: + img1, img2 = t(img1, img2) + return img1, img2 + +class NormalizeBoth(torchvision.transforms.Normalize): + def forward(self, img1, img2): + img1 = super().forward(img1) + img2 = super().forward(img2) + return img1, img2 + +class ToTensorBoth(torchvision.transforms.ToTensor): + def __call__(self, img1, img2): + img1 = super().__call__(img1) + img2 = super().__call__(img2) + return img1, img2 + +class RandomCropPair(torchvision.transforms.RandomCrop): + # the crop will be intentionally different for the two images with this class + def forward(self, img1, img2): + img1 = super().forward(img1) + img2 = super().forward(img2) + return img1, img2 + +class ColorJitterPair(torchvision.transforms.ColorJitter): + # can be symmetric (same for both images) or assymetric (different jitter params for each image) depending on assymetric_prob + def __init__(self, assymetric_prob, **kwargs): + super().__init__(**kwargs) + self.assymetric_prob = assymetric_prob + def jitter_one(self, img, fn_idx, brightness_factor, contrast_factor, saturation_factor, hue_factor): + for fn_id in fn_idx: + if fn_id == 0 and brightness_factor is not None: + img = F.adjust_brightness(img, brightness_factor) + elif fn_id == 1 and contrast_factor is not None: + img = F.adjust_contrast(img, contrast_factor) + elif fn_id == 2 and saturation_factor is not None: + img = F.adjust_saturation(img, saturation_factor) + elif fn_id == 3 and hue_factor is not None: + img = F.adjust_hue(img, hue_factor) + return img + + def forward(self, img1, img2): + + fn_idx, brightness_factor, contrast_factor, saturation_factor, hue_factor = self.get_params( + self.brightness, self.contrast, self.saturation, self.hue + ) + img1 = self.jitter_one(img1, fn_idx, brightness_factor, contrast_factor, saturation_factor, hue_factor) + if torch.rand(1) < self.assymetric_prob: # assymetric: + fn_idx, brightness_factor, contrast_factor, saturation_factor, hue_factor = self.get_params( + self.brightness, self.contrast, self.saturation, self.hue + ) + img2 = self.jitter_one(img2, fn_idx, brightness_factor, contrast_factor, saturation_factor, hue_factor) + return img1, img2 + +def get_pair_transforms(transform_str, totensor=True, normalize=True): + # transform_str is eg crop224+color + trfs = [] + for s in transform_str.split('+'): + if s.startswith('crop'): + size = int(s[len('crop'):]) + trfs.append(RandomCropPair(size)) + elif s=='acolor': + trfs.append(ColorJitterPair(assymetric_prob=1.0, brightness=(0.6, 1.4), contrast=(0.6, 1.4), saturation=(0.6, 1.4), hue=0.0)) + elif s=='': # if transform_str was "" + pass + else: + raise NotImplementedError('Unknown augmentation: '+s) + + if totensor: + trfs.append( ToTensorBoth() ) + if normalize: + trfs.append( NormalizeBoth(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ) + + if len(trfs)==0: + return None + elif len(trfs)==1: + return trfs + else: + return ComposePair(trfs) + + + + + diff --git a/submodules/mast3r/dust3r/croco/demo.py b/submodules/mast3r/dust3r/croco/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..91b80ccc5c98c18e20d1ce782511aa824ef28f77 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/demo.py @@ -0,0 +1,55 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +import torch +from models.croco import CroCoNet +from PIL import Image +import torchvision.transforms +from torchvision.transforms import ToTensor, Normalize, Compose + +def main(): + device = torch.device('cuda:0' if torch.cuda.is_available() and torch.cuda.device_count()>0 else 'cpu') + + # load 224x224 images and transform them to tensor + imagenet_mean = [0.485, 0.456, 0.406] + imagenet_mean_tensor = torch.tensor(imagenet_mean).view(1,3,1,1).to(device, non_blocking=True) + imagenet_std = [0.229, 0.224, 0.225] + imagenet_std_tensor = torch.tensor(imagenet_std).view(1,3,1,1).to(device, non_blocking=True) + trfs = Compose([ToTensor(), Normalize(mean=imagenet_mean, std=imagenet_std)]) + image1 = trfs(Image.open('assets/Chateau1.png').convert('RGB')).to(device, non_blocking=True).unsqueeze(0) + image2 = trfs(Image.open('assets/Chateau2.png').convert('RGB')).to(device, non_blocking=True).unsqueeze(0) + + # load model + ckpt = torch.load('pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth', 'cpu') + model = CroCoNet( **ckpt.get('croco_kwargs',{})).to(device) + model.eval() + msg = model.load_state_dict(ckpt['model'], strict=True) + + # forward + with torch.inference_mode(): + out, mask, target = model(image1, image2) + + # the output is normalized, thus use the mean/std of the actual image to go back to RGB space + patchified = model.patchify(image1) + mean = patchified.mean(dim=-1, keepdim=True) + var = patchified.var(dim=-1, keepdim=True) + decoded_image = model.unpatchify(out * (var + 1.e-6)**.5 + mean) + # undo imagenet normalization, prepare masked image + decoded_image = decoded_image * imagenet_std_tensor + imagenet_mean_tensor + input_image = image1 * imagenet_std_tensor + imagenet_mean_tensor + ref_image = image2 * imagenet_std_tensor + imagenet_mean_tensor + image_masks = model.unpatchify(model.patchify(torch.ones_like(ref_image)) * mask[:,:,None]) + masked_input_image = ((1 - image_masks) * input_image) + + # make visualization + visualization = torch.cat((ref_image, masked_input_image, decoded_image, input_image), dim=3) # 4*(B, 3, H, W) -> B, 3, H, W*4 + B, C, H, W = visualization.shape + visualization = visualization.permute(1, 0, 2, 3).reshape(C, B*H, W) + visualization = torchvision.transforms.functional.to_pil_image(torch.clamp(visualization, 0, 1)) + fname = "demo_output.png" + visualization.save(fname) + print('Visualization save in '+fname) + + +if __name__=="__main__": + main() diff --git a/submodules/mast3r/dust3r/croco/interactive_demo.ipynb b/submodules/mast3r/dust3r/croco/interactive_demo.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..6cfc960af5baac9a69029c29a16eea4e24123a71 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/interactive_demo.ipynb @@ -0,0 +1,271 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Interactive demo of Cross-view Completion." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Copyright (C) 2022-present Naver Corporation. All rights reserved.\n", + "# Licensed under CC BY-NC-SA 4.0 (non-commercial use only)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import numpy as np\n", + "from models.croco import CroCoNet\n", + "from ipywidgets import interact, interactive, fixed, interact_manual\n", + "import ipywidgets as widgets\n", + "import matplotlib.pyplot as plt\n", + "import quaternion\n", + "import models.masking" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load CroCo model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ckpt = torch.load('pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth', 'cpu')\n", + "model = CroCoNet( **ckpt.get('croco_kwargs',{}))\n", + "msg = model.load_state_dict(ckpt['model'], strict=True)\n", + "use_gpu = torch.cuda.is_available() and torch.cuda.device_count()>0\n", + "device = torch.device('cuda:0' if use_gpu else 'cpu')\n", + "model = model.eval()\n", + "model = model.to(device=device)\n", + "print(msg)\n", + "\n", + "def process_images(ref_image, target_image, masking_ratio, reconstruct_unmasked_patches=False):\n", + " \"\"\"\n", + " Perform Cross-View completion using two input images, specified using Numpy arrays.\n", + " \"\"\"\n", + " # Replace the mask generator\n", + " model.mask_generator = models.masking.RandomMask(model.patch_embed.num_patches, masking_ratio)\n", + "\n", + " # ImageNet-1k color normalization\n", + " imagenet_mean = torch.as_tensor([0.485, 0.456, 0.406]).reshape(1,3,1,1).to(device)\n", + " imagenet_std = torch.as_tensor([0.229, 0.224, 0.225]).reshape(1,3,1,1).to(device)\n", + "\n", + " normalize_input_colors = True\n", + " is_output_normalized = True\n", + " with torch.no_grad():\n", + " # Cast data to torch\n", + " target_image = (torch.as_tensor(target_image, dtype=torch.float, device=device).permute(2,0,1) / 255)[None]\n", + " ref_image = (torch.as_tensor(ref_image, dtype=torch.float, device=device).permute(2,0,1) / 255)[None]\n", + "\n", + " if normalize_input_colors:\n", + " ref_image = (ref_image - imagenet_mean) / imagenet_std\n", + " target_image = (target_image - imagenet_mean) / imagenet_std\n", + "\n", + " out, mask, _ = model(target_image, ref_image)\n", + " # # get target\n", + " if not is_output_normalized:\n", + " predicted_image = model.unpatchify(out)\n", + " else:\n", + " # The output only contains higher order information,\n", + " # we retrieve mean and standard deviation from the actual target image\n", + " patchified = model.patchify(target_image)\n", + " mean = patchified.mean(dim=-1, keepdim=True)\n", + " var = patchified.var(dim=-1, keepdim=True)\n", + " pred_renorm = out * (var + 1.e-6)**.5 + mean\n", + " predicted_image = model.unpatchify(pred_renorm)\n", + "\n", + " image_masks = model.unpatchify(model.patchify(torch.ones_like(ref_image)) * mask[:,:,None])\n", + " masked_target_image = (1 - image_masks) * target_image\n", + " \n", + " if not reconstruct_unmasked_patches:\n", + " # Replace unmasked patches by their actual values\n", + " predicted_image = predicted_image * image_masks + masked_target_image\n", + "\n", + " # Unapply color normalization\n", + " if normalize_input_colors:\n", + " predicted_image = predicted_image * imagenet_std + imagenet_mean\n", + " masked_target_image = masked_target_image * imagenet_std + imagenet_mean\n", + " \n", + " # Cast to Numpy\n", + " masked_target_image = np.asarray(torch.clamp(masked_target_image.squeeze(0).permute(1,2,0) * 255, 0, 255).cpu().numpy(), dtype=np.uint8)\n", + " predicted_image = np.asarray(torch.clamp(predicted_image.squeeze(0).permute(1,2,0) * 255, 0, 255).cpu().numpy(), dtype=np.uint8)\n", + " return masked_target_image, predicted_image" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Use the Habitat simulator to render images from arbitrary viewpoints (requires habitat_sim to be installed)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"MAGNUM_LOG\"]=\"quiet\"\n", + "os.environ[\"HABITAT_SIM_LOG\"]=\"quiet\"\n", + "import habitat_sim\n", + "\n", + "scene = \"habitat-sim-data/scene_datasets/habitat-test-scenes/skokloster-castle.glb\"\n", + "navmesh = \"habitat-sim-data/scene_datasets/habitat-test-scenes/skokloster-castle.navmesh\"\n", + "\n", + "sim_cfg = habitat_sim.SimulatorConfiguration()\n", + "if use_gpu: sim_cfg.gpu_device_id = 0\n", + "sim_cfg.scene_id = scene\n", + "sim_cfg.load_semantic_mesh = False\n", + "rgb_sensor_spec = habitat_sim.CameraSensorSpec()\n", + "rgb_sensor_spec.uuid = \"color\"\n", + "rgb_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR\n", + "rgb_sensor_spec.resolution = (224,224)\n", + "rgb_sensor_spec.hfov = 56.56\n", + "rgb_sensor_spec.position = [0.0, 0.0, 0.0]\n", + "rgb_sensor_spec.orientation = [0, 0, 0]\n", + "agent_cfg = habitat_sim.agent.AgentConfiguration(sensor_specifications=[rgb_sensor_spec])\n", + "\n", + "\n", + "cfg = habitat_sim.Configuration(sim_cfg, [agent_cfg])\n", + "sim = habitat_sim.Simulator(cfg)\n", + "if navmesh is not None:\n", + " sim.pathfinder.load_nav_mesh(navmesh)\n", + "agent = sim.initialize_agent(agent_id=0)\n", + "\n", + "def sample_random_viewpoint():\n", + " \"\"\" Sample a random viewpoint using the navmesh \"\"\"\n", + " nav_point = sim.pathfinder.get_random_navigable_point()\n", + " # Sample a random viewpoint height\n", + " viewpoint_height = np.random.uniform(1.0, 1.6)\n", + " viewpoint_position = nav_point + viewpoint_height * habitat_sim.geo.UP\n", + " viewpoint_orientation = quaternion.from_rotation_vector(np.random.uniform(-np.pi, np.pi) * habitat_sim.geo.UP)\n", + " return viewpoint_position, viewpoint_orientation\n", + "\n", + "def render_viewpoint(position, orientation):\n", + " agent_state = habitat_sim.AgentState()\n", + " agent_state.position = position\n", + " agent_state.rotation = orientation\n", + " agent.set_state(agent_state)\n", + " viewpoint_observations = sim.get_sensor_observations(agent_ids=0)\n", + " image = viewpoint_observations['color'][:,:,:3]\n", + " image = np.asarray(np.clip(1.5 * np.asarray(image, dtype=float), 0, 255), dtype=np.uint8)\n", + " return image" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Sample a random reference view" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ref_position, ref_orientation = sample_random_viewpoint()\n", + "ref_image = render_viewpoint(ref_position, ref_orientation)\n", + "plt.clf()\n", + "fig, axes = plt.subplots(1,1, squeeze=False, num=1)\n", + "axes[0,0].imshow(ref_image)\n", + "for ax in axes.flatten():\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Interactive cross-view completion using CroCo" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "reconstruct_unmasked_patches = False\n", + "\n", + "def show_demo(masking_ratio, x, y, z, panorama, elevation):\n", + " R = quaternion.as_rotation_matrix(ref_orientation)\n", + " target_position = ref_position + x * R[:,0] + y * R[:,1] + z * R[:,2]\n", + " target_orientation = (ref_orientation\n", + " * quaternion.from_rotation_vector(-elevation * np.pi/180 * habitat_sim.geo.LEFT) \n", + " * quaternion.from_rotation_vector(-panorama * np.pi/180 * habitat_sim.geo.UP))\n", + " \n", + " ref_image = render_viewpoint(ref_position, ref_orientation)\n", + " target_image = render_viewpoint(target_position, target_orientation)\n", + "\n", + " masked_target_image, predicted_image = process_images(ref_image, target_image, masking_ratio, reconstruct_unmasked_patches)\n", + "\n", + " fig, axes = plt.subplots(1,4, squeeze=True, dpi=300)\n", + " axes[0].imshow(ref_image)\n", + " axes[0].set_xlabel(\"Reference\")\n", + " axes[1].imshow(masked_target_image)\n", + " axes[1].set_xlabel(\"Masked target\")\n", + " axes[2].imshow(predicted_image)\n", + " axes[2].set_xlabel(\"Reconstruction\") \n", + " axes[3].imshow(target_image)\n", + " axes[3].set_xlabel(\"Target\")\n", + " for ax in axes.flatten():\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + "\n", + "interact(show_demo,\n", + " masking_ratio=widgets.FloatSlider(description='masking', value=0.9, min=0.0, max=1.0),\n", + " x=widgets.FloatSlider(value=0.0, min=-0.5, max=0.5, step=0.05),\n", + " y=widgets.FloatSlider(value=0.0, min=-0.5, max=0.5, step=0.05),\n", + " z=widgets.FloatSlider(value=0.0, min=-0.5, max=0.5, step=0.05),\n", + " panorama=widgets.FloatSlider(value=0.0, min=-20, max=20, step=0.5),\n", + " elevation=widgets.FloatSlider(value=0.0, min=-20, max=20, step=0.5));" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.13" + }, + "vscode": { + "interpreter": { + "hash": "f9237820cd248d7e07cb4fb9f0e4508a85d642f19d831560c0a4b61f3e907e67" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/submodules/mast3r/dust3r/croco/models/blocks.py b/submodules/mast3r/dust3r/croco/models/blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..18133524f0ae265b0bd8d062d7c9eeaa63858a9b --- /dev/null +++ b/submodules/mast3r/dust3r/croco/models/blocks.py @@ -0,0 +1,241 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + + +# -------------------------------------------------------- +# Main encoder/decoder blocks +# -------------------------------------------------------- +# References: +# timm +# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/helpers.py +# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/drop.py +# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/mlp.py +# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/patch_embed.py + + +import torch +import torch.nn as nn + +from itertools import repeat +import collections.abc + + +def _ntuple(n): + def parse(x): + if isinstance(x, collections.abc.Iterable) and not isinstance(x, str): + return x + return tuple(repeat(x, n)) + return parse +to_2tuple = _ntuple(2) + +def drop_path(x, drop_prob: float = 0., training: bool = False, scale_by_keep: bool = True): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = x.new_empty(shape).bernoulli_(keep_prob) + if keep_prob > 0.0 and scale_by_keep: + random_tensor.div_(keep_prob) + return x * random_tensor + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + def __init__(self, drop_prob: float = 0., scale_by_keep: bool = True): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + self.scale_by_keep = scale_by_keep + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training, self.scale_by_keep) + + def extra_repr(self): + return f'drop_prob={round(self.drop_prob,3):0.3f}' + +class Mlp(nn.Module): + """ MLP as used in Vision Transformer, MLP-Mixer and related networks""" + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, bias=True, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + bias = to_2tuple(bias) + drop_probs = to_2tuple(drop) + + self.fc1 = nn.Linear(in_features, hidden_features, bias=bias[0]) + self.act = act_layer() + self.drop1 = nn.Dropout(drop_probs[0]) + self.fc2 = nn.Linear(hidden_features, out_features, bias=bias[1]) + self.drop2 = nn.Dropout(drop_probs[1]) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop1(x) + x = self.fc2(x) + x = self.drop2(x) + return x + +class Attention(nn.Module): + + def __init__(self, dim, rope=None, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0.): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + self.rope = rope + + def forward(self, x, xpos): + B, N, C = x.shape + + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).transpose(1,3) + q, k, v = [qkv[:,:,i] for i in range(3)] + # q,k,v = qkv.unbind(2) # make torchscript happy (cannot use tensor as tuple) + + if self.rope is not None: + q = self.rope(q, xpos) + k = self.rope(k, xpos) + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + +class Block(nn.Module): + + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, rope=None): + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention(dim, rope=rope, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x, xpos): + x = x + self.drop_path(self.attn(self.norm1(x), xpos)) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + +class CrossAttention(nn.Module): + + def __init__(self, dim, rope=None, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0.): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + + self.projq = nn.Linear(dim, dim, bias=qkv_bias) + self.projk = nn.Linear(dim, dim, bias=qkv_bias) + self.projv = nn.Linear(dim, dim, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + self.rope = rope + + def forward(self, query, key, value, qpos, kpos): + B, Nq, C = query.shape + Nk = key.shape[1] + Nv = value.shape[1] + + q = self.projq(query).reshape(B,Nq,self.num_heads, C// self.num_heads).permute(0, 2, 1, 3) + k = self.projk(key).reshape(B,Nk,self.num_heads, C// self.num_heads).permute(0, 2, 1, 3) + v = self.projv(value).reshape(B,Nv,self.num_heads, C// self.num_heads).permute(0, 2, 1, 3) + + if self.rope is not None: + q = self.rope(q, qpos) + k = self.rope(k, kpos) + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, Nq, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + +class DecoderBlock(nn.Module): + + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, norm_mem=True, rope=None): + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention(dim, rope=rope, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) + self.cross_attn = CrossAttention(dim, rope=rope, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + self.norm3 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + self.norm_y = norm_layer(dim) if norm_mem else nn.Identity() + + def forward(self, x, y, xpos, ypos): + x = x + self.drop_path(self.attn(self.norm1(x), xpos)) + y_ = self.norm_y(y) + x = x + self.drop_path(self.cross_attn(self.norm2(x), y_, y_, xpos, ypos)) + x = x + self.drop_path(self.mlp(self.norm3(x))) + return x, y + + +# patch embedding +class PositionGetter(object): + """ return positions of patches """ + + def __init__(self): + self.cache_positions = {} + + def __call__(self, b, h, w, device): + if not (h,w) in self.cache_positions: + x = torch.arange(w, device=device) + y = torch.arange(h, device=device) + self.cache_positions[h,w] = torch.cartesian_prod(y, x) # (h, w, 2) + pos = self.cache_positions[h,w].view(1, h*w, 2).expand(b, -1, 2).clone() + return pos + +class PatchEmbed(nn.Module): + """ just adding _init_weights + position getter compared to timm.models.layers.patch_embed.PatchEmbed""" + + def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + self.img_size = img_size + self.patch_size = patch_size + self.grid_size = (img_size[0] // patch_size[0], img_size[1] // patch_size[1]) + self.num_patches = self.grid_size[0] * self.grid_size[1] + self.flatten = flatten + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + self.position_getter = PositionGetter() + + def forward(self, x): + B, C, H, W = x.shape + torch._assert(H == self.img_size[0], f"Input image height ({H}) doesn't match model ({self.img_size[0]}).") + torch._assert(W == self.img_size[1], f"Input image width ({W}) doesn't match model ({self.img_size[1]}).") + x = self.proj(x) + pos = self.position_getter(B, x.size(2), x.size(3), x.device) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x, pos + + def _init_weights(self): + w = self.proj.weight.data + torch.nn.init.xavier_uniform_(w.view([w.shape[0], -1])) + diff --git a/submodules/mast3r/dust3r/croco/models/criterion.py b/submodules/mast3r/dust3r/croco/models/criterion.py new file mode 100644 index 0000000000000000000000000000000000000000..11696c40865344490f23796ea45e8fbd5e654731 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/models/criterion.py @@ -0,0 +1,37 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Criterion to train CroCo +# -------------------------------------------------------- +# References: +# MAE: https://github.com/facebookresearch/mae +# -------------------------------------------------------- + +import torch + +class MaskedMSE(torch.nn.Module): + + def __init__(self, norm_pix_loss=False, masked=True): + """ + norm_pix_loss: normalize each patch by their pixel mean and variance + masked: compute loss over the masked patches only + """ + super().__init__() + self.norm_pix_loss = norm_pix_loss + self.masked = masked + + def forward(self, pred, mask, target): + + if self.norm_pix_loss: + mean = target.mean(dim=-1, keepdim=True) + var = target.var(dim=-1, keepdim=True) + target = (target - mean) / (var + 1.e-6)**.5 + + loss = (pred - target) ** 2 + loss = loss.mean(dim=-1) # [N, L], mean loss per patch + if self.masked: + loss = (loss * mask).sum() / mask.sum() # mean loss on masked patches + else: + loss = loss.mean() # mean loss + return loss diff --git a/submodules/mast3r/dust3r/croco/models/croco.py b/submodules/mast3r/dust3r/croco/models/croco.py new file mode 100644 index 0000000000000000000000000000000000000000..14c68634152d75555b4c35c25af268394c5821fe --- /dev/null +++ b/submodules/mast3r/dust3r/croco/models/croco.py @@ -0,0 +1,249 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + + +# -------------------------------------------------------- +# CroCo model during pretraining +# -------------------------------------------------------- + + + +import torch +import torch.nn as nn +torch.backends.cuda.matmul.allow_tf32 = True # for gpu >= Ampere and pytorch >= 1.12 +from functools import partial + +from models.blocks import Block, DecoderBlock, PatchEmbed +from models.pos_embed import get_2d_sincos_pos_embed, RoPE2D +from models.masking import RandomMask + + +class CroCoNet(nn.Module): + + def __init__(self, + img_size=224, # input image size + patch_size=16, # patch_size + mask_ratio=0.9, # ratios of masked tokens + enc_embed_dim=768, # encoder feature dimension + enc_depth=12, # encoder depth + enc_num_heads=12, # encoder number of heads in the transformer block + dec_embed_dim=512, # decoder feature dimension + dec_depth=8, # decoder depth + dec_num_heads=16, # decoder number of heads in the transformer block + mlp_ratio=4, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + norm_im2_in_dec=True, # whether to apply normalization of the 'memory' = (second image) in the decoder + pos_embed='cosine', # positional embedding (either cosine or RoPE100) + ): + + super(CroCoNet, self).__init__() + + # patch embeddings (with initialization done as in MAE) + self._set_patch_embed(img_size, patch_size, enc_embed_dim) + + # mask generations + self._set_mask_generator(self.patch_embed.num_patches, mask_ratio) + + self.pos_embed = pos_embed + if pos_embed=='cosine': + # positional embedding of the encoder + enc_pos_embed = get_2d_sincos_pos_embed(enc_embed_dim, int(self.patch_embed.num_patches**.5), n_cls_token=0) + self.register_buffer('enc_pos_embed', torch.from_numpy(enc_pos_embed).float()) + # positional embedding of the decoder + dec_pos_embed = get_2d_sincos_pos_embed(dec_embed_dim, int(self.patch_embed.num_patches**.5), n_cls_token=0) + self.register_buffer('dec_pos_embed', torch.from_numpy(dec_pos_embed).float()) + # pos embedding in each block + self.rope = None # nothing for cosine + elif pos_embed.startswith('RoPE'): # eg RoPE100 + self.enc_pos_embed = None # nothing to add in the encoder with RoPE + self.dec_pos_embed = None # nothing to add in the decoder with RoPE + if RoPE2D is None: raise ImportError("Cannot find cuRoPE2D, please install it following the README instructions") + freq = float(pos_embed[len('RoPE'):]) + self.rope = RoPE2D(freq=freq) + else: + raise NotImplementedError('Unknown pos_embed '+pos_embed) + + # transformer for the encoder + self.enc_depth = enc_depth + self.enc_embed_dim = enc_embed_dim + self.enc_blocks = nn.ModuleList([ + Block(enc_embed_dim, enc_num_heads, mlp_ratio, qkv_bias=True, norm_layer=norm_layer, rope=self.rope) + for i in range(enc_depth)]) + self.enc_norm = norm_layer(enc_embed_dim) + + # masked tokens + self._set_mask_token(dec_embed_dim) + + # decoder + self._set_decoder(enc_embed_dim, dec_embed_dim, dec_num_heads, dec_depth, mlp_ratio, norm_layer, norm_im2_in_dec) + + # prediction head + self._set_prediction_head(dec_embed_dim, patch_size) + + # initializer weights + self.initialize_weights() + + def _set_patch_embed(self, img_size=224, patch_size=16, enc_embed_dim=768): + self.patch_embed = PatchEmbed(img_size, patch_size, 3, enc_embed_dim) + + def _set_mask_generator(self, num_patches, mask_ratio): + self.mask_generator = RandomMask(num_patches, mask_ratio) + + def _set_mask_token(self, dec_embed_dim): + self.mask_token = nn.Parameter(torch.zeros(1, 1, dec_embed_dim)) + + def _set_decoder(self, enc_embed_dim, dec_embed_dim, dec_num_heads, dec_depth, mlp_ratio, norm_layer, norm_im2_in_dec): + self.dec_depth = dec_depth + self.dec_embed_dim = dec_embed_dim + # transfer from encoder to decoder + self.decoder_embed = nn.Linear(enc_embed_dim, dec_embed_dim, bias=True) + # transformer for the decoder + self.dec_blocks = nn.ModuleList([ + DecoderBlock(dec_embed_dim, dec_num_heads, mlp_ratio=mlp_ratio, qkv_bias=True, norm_layer=norm_layer, norm_mem=norm_im2_in_dec, rope=self.rope) + for i in range(dec_depth)]) + # final norm layer + self.dec_norm = norm_layer(dec_embed_dim) + + def _set_prediction_head(self, dec_embed_dim, patch_size): + self.prediction_head = nn.Linear(dec_embed_dim, patch_size**2 * 3, bias=True) + + + def initialize_weights(self): + # patch embed + self.patch_embed._init_weights() + # mask tokens + if self.mask_token is not None: torch.nn.init.normal_(self.mask_token, std=.02) + # linears and layer norms + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + # we use xavier_uniform following official JAX ViT: + torch.nn.init.xavier_uniform_(m.weight) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def _encode_image(self, image, do_mask=False, return_all_blocks=False): + """ + image has B x 3 x img_size x img_size + do_mask: whether to perform masking or not + return_all_blocks: if True, return the features at the end of every block + instead of just the features from the last block (eg for some prediction heads) + """ + # embed the image into patches (x has size B x Npatches x C) + # and get position if each return patch (pos has size B x Npatches x 2) + x, pos = self.patch_embed(image) + # add positional embedding without cls token + if self.enc_pos_embed is not None: + x = x + self.enc_pos_embed[None,...] + # apply masking + B,N,C = x.size() + if do_mask: + masks = self.mask_generator(x) + x = x[~masks].view(B, -1, C) + posvis = pos[~masks].view(B, -1, 2) + else: + B,N,C = x.size() + masks = torch.zeros((B,N), dtype=bool) + posvis = pos + # now apply the transformer encoder and normalization + if return_all_blocks: + out = [] + for blk in self.enc_blocks: + x = blk(x, posvis) + out.append(x) + out[-1] = self.enc_norm(out[-1]) + return out, pos, masks + else: + for blk in self.enc_blocks: + x = blk(x, posvis) + x = self.enc_norm(x) + return x, pos, masks + + def _decoder(self, feat1, pos1, masks1, feat2, pos2, return_all_blocks=False): + """ + return_all_blocks: if True, return the features at the end of every block + instead of just the features from the last block (eg for some prediction heads) + + masks1 can be None => assume image1 fully visible + """ + # encoder to decoder layer + visf1 = self.decoder_embed(feat1) + f2 = self.decoder_embed(feat2) + # append masked tokens to the sequence + B,Nenc,C = visf1.size() + if masks1 is None: # downstreams + f1_ = visf1 + else: # pretraining + Ntotal = masks1.size(1) + f1_ = self.mask_token.repeat(B, Ntotal, 1).to(dtype=visf1.dtype) + f1_[~masks1] = visf1.view(B * Nenc, C) + # add positional embedding + if self.dec_pos_embed is not None: + f1_ = f1_ + self.dec_pos_embed + f2 = f2 + self.dec_pos_embed + # apply Transformer blocks + out = f1_ + out2 = f2 + if return_all_blocks: + _out, out = out, [] + for blk in self.dec_blocks: + _out, out2 = blk(_out, out2, pos1, pos2) + out.append(_out) + out[-1] = self.dec_norm(out[-1]) + else: + for blk in self.dec_blocks: + out, out2 = blk(out, out2, pos1, pos2) + out = self.dec_norm(out) + return out + + def patchify(self, imgs): + """ + imgs: (B, 3, H, W) + x: (B, L, patch_size**2 *3) + """ + p = self.patch_embed.patch_size[0] + assert imgs.shape[2] == imgs.shape[3] and imgs.shape[2] % p == 0 + + h = w = imgs.shape[2] // p + x = imgs.reshape(shape=(imgs.shape[0], 3, h, p, w, p)) + x = torch.einsum('nchpwq->nhwpqc', x) + x = x.reshape(shape=(imgs.shape[0], h * w, p**2 * 3)) + + return x + + def unpatchify(self, x, channels=3): + """ + x: (N, L, patch_size**2 *channels) + imgs: (N, 3, H, W) + """ + patch_size = self.patch_embed.patch_size[0] + h = w = int(x.shape[1]**.5) + assert h * w == x.shape[1] + x = x.reshape(shape=(x.shape[0], h, w, patch_size, patch_size, channels)) + x = torch.einsum('nhwpqc->nchpwq', x) + imgs = x.reshape(shape=(x.shape[0], channels, h * patch_size, h * patch_size)) + return imgs + + def forward(self, img1, img2): + """ + img1: tensor of size B x 3 x img_size x img_size + img2: tensor of size B x 3 x img_size x img_size + + out will be B x N x (3*patch_size*patch_size) + masks are also returned as B x N just in case + """ + # encoder of the masked first image + feat1, pos1, mask1 = self._encode_image(img1, do_mask=True) + # encoder of the second image + feat2, pos2, _ = self._encode_image(img2, do_mask=False) + # decoder + decfeat = self._decoder(feat1, pos1, mask1, feat2, pos2) + # prediction head + out = self.prediction_head(decfeat) + # get target + target = self.patchify(img1) + return out, mask1, target diff --git a/submodules/mast3r/dust3r/croco/models/croco_downstream.py b/submodules/mast3r/dust3r/croco/models/croco_downstream.py new file mode 100644 index 0000000000000000000000000000000000000000..159dfff4d2c1461bc235e21441b57ce1e2088f76 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/models/croco_downstream.py @@ -0,0 +1,122 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# CroCo model for downstream tasks +# -------------------------------------------------------- + +import torch + +from .croco import CroCoNet + + +def croco_args_from_ckpt(ckpt): + if 'croco_kwargs' in ckpt: # CroCo v2 released models + return ckpt['croco_kwargs'] + elif 'args' in ckpt and hasattr(ckpt['args'], 'model'): # pretrained using the official code release + s = ckpt['args'].model # eg "CroCoNet(enc_embed_dim=1024, enc_num_heads=16, enc_depth=24)" + assert s.startswith('CroCoNet(') + return eval('dict'+s[len('CroCoNet'):]) # transform it into the string of a dictionary and evaluate it + else: # CroCo v1 released models + return dict() + +class CroCoDownstreamMonocularEncoder(CroCoNet): + + def __init__(self, + head, + **kwargs): + """ Build network for monocular downstream task, only using the encoder. + It takes an extra argument head, that is called with the features + and a dictionary img_info containing 'width' and 'height' keys + The head is setup with the croconet arguments in this init function + NOTE: It works by *calling super().__init__() but with redefined setters + + """ + super(CroCoDownstreamMonocularEncoder, self).__init__(**kwargs) + head.setup(self) + self.head = head + + def _set_mask_generator(self, *args, **kwargs): + """ No mask generator """ + return + + def _set_mask_token(self, *args, **kwargs): + """ No mask token """ + self.mask_token = None + return + + def _set_decoder(self, *args, **kwargs): + """ No decoder """ + return + + def _set_prediction_head(self, *args, **kwargs): + """ No 'prediction head' for downstream tasks.""" + return + + def forward(self, img): + """ + img if of size batch_size x 3 x h x w + """ + B, C, H, W = img.size() + img_info = {'height': H, 'width': W} + need_all_layers = hasattr(self.head, 'return_all_blocks') and self.head.return_all_blocks + out, _, _ = self._encode_image(img, do_mask=False, return_all_blocks=need_all_layers) + return self.head(out, img_info) + + +class CroCoDownstreamBinocular(CroCoNet): + + def __init__(self, + head, + **kwargs): + """ Build network for binocular downstream task + It takes an extra argument head, that is called with the features + and a dictionary img_info containing 'width' and 'height' keys + The head is setup with the croconet arguments in this init function + """ + super(CroCoDownstreamBinocular, self).__init__(**kwargs) + head.setup(self) + self.head = head + + def _set_mask_generator(self, *args, **kwargs): + """ No mask generator """ + return + + def _set_mask_token(self, *args, **kwargs): + """ No mask token """ + self.mask_token = None + return + + def _set_prediction_head(self, *args, **kwargs): + """ No prediction head for downstream tasks, define your own head """ + return + + def encode_image_pairs(self, img1, img2, return_all_blocks=False): + """ run encoder for a pair of images + it is actually ~5% faster to concatenate the images along the batch dimension + than to encode them separately + """ + ## the two commented lines below is the naive version with separate encoding + #out, pos, _ = self._encode_image(img1, do_mask=False, return_all_blocks=return_all_blocks) + #out2, pos2, _ = self._encode_image(img2, do_mask=False, return_all_blocks=False) + ## and now the faster version + out, pos, _ = self._encode_image( torch.cat( (img1,img2), dim=0), do_mask=False, return_all_blocks=return_all_blocks ) + if return_all_blocks: + out,out2 = list(map(list, zip(*[o.chunk(2, dim=0) for o in out]))) + out2 = out2[-1] + else: + out,out2 = out.chunk(2, dim=0) + pos,pos2 = pos.chunk(2, dim=0) + return out, out2, pos, pos2 + + def forward(self, img1, img2): + B, C, H, W = img1.size() + img_info = {'height': H, 'width': W} + return_all_blocks = hasattr(self.head, 'return_all_blocks') and self.head.return_all_blocks + out, out2, pos, pos2 = self.encode_image_pairs(img1, img2, return_all_blocks=return_all_blocks) + if return_all_blocks: + decout = self._decoder(out[-1], pos, None, out2, pos2, return_all_blocks=return_all_blocks) + decout = out+decout + else: + decout = self._decoder(out, pos, None, out2, pos2, return_all_blocks=return_all_blocks) + return self.head(decout, img_info) \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/models/curope/__init__.py b/submodules/mast3r/dust3r/croco/models/curope/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..25e3d48a162760260826080f6366838e83e26878 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/models/curope/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +from .curope2d import cuRoPE2D diff --git a/submodules/mast3r/dust3r/croco/models/curope/curope.cpp b/submodules/mast3r/dust3r/croco/models/curope/curope.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8fe9058e05aa1bf3f37b0d970edc7312bc68455b --- /dev/null +++ b/submodules/mast3r/dust3r/croco/models/curope/curope.cpp @@ -0,0 +1,69 @@ +/* + Copyright (C) 2022-present Naver Corporation. All rights reserved. + Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +*/ + +#include + +// forward declaration +void rope_2d_cuda( torch::Tensor tokens, const torch::Tensor pos, const float base, const float fwd ); + +void rope_2d_cpu( torch::Tensor tokens, const torch::Tensor positions, const float base, const float fwd ) +{ + const int B = tokens.size(0); + const int N = tokens.size(1); + const int H = tokens.size(2); + const int D = tokens.size(3) / 4; + + auto tok = tokens.accessor(); + auto pos = positions.accessor(); + + for (int b = 0; b < B; b++) { + for (int x = 0; x < 2; x++) { // y and then x (2d) + for (int n = 0; n < N; n++) { + + // grab the token position + const int p = pos[b][n][x]; + + for (int h = 0; h < H; h++) { + for (int d = 0; d < D; d++) { + // grab the two values + float u = tok[b][n][h][d+0+x*2*D]; + float v = tok[b][n][h][d+D+x*2*D]; + + // grab the cos,sin + const float inv_freq = fwd * p / powf(base, d/float(D)); + float c = cosf(inv_freq); + float s = sinf(inv_freq); + + // write the result + tok[b][n][h][d+0+x*2*D] = u*c - v*s; + tok[b][n][h][d+D+x*2*D] = v*c + u*s; + } + } + } + } + } +} + +void rope_2d( torch::Tensor tokens, // B,N,H,D + const torch::Tensor positions, // B,N,2 + const float base, + const float fwd ) +{ + TORCH_CHECK(tokens.dim() == 4, "tokens must have 4 dimensions"); + TORCH_CHECK(positions.dim() == 3, "positions must have 3 dimensions"); + TORCH_CHECK(tokens.size(0) == positions.size(0), "batch size differs between tokens & positions"); + TORCH_CHECK(tokens.size(1) == positions.size(1), "seq_length differs between tokens & positions"); + TORCH_CHECK(positions.size(2) == 2, "positions.shape[2] must be equal to 2"); + TORCH_CHECK(tokens.is_cuda() == positions.is_cuda(), "tokens and positions are not on the same device" ); + + if (tokens.is_cuda()) + rope_2d_cuda( tokens, positions, base, fwd ); + else + rope_2d_cpu( tokens, positions, base, fwd ); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("rope_2d", &rope_2d, "RoPE 2d forward/backward"); +} diff --git a/submodules/mast3r/dust3r/croco/models/curope/curope2d.py b/submodules/mast3r/dust3r/croco/models/curope/curope2d.py new file mode 100644 index 0000000000000000000000000000000000000000..a49c12f8c529e9a889b5ac20c5767158f238e17d --- /dev/null +++ b/submodules/mast3r/dust3r/croco/models/curope/curope2d.py @@ -0,0 +1,40 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +import torch + +try: + import curope as _kernels # run `python setup.py install` +except ModuleNotFoundError: + from . import curope as _kernels # run `python setup.py build_ext --inplace` + + +class cuRoPE2D_func (torch.autograd.Function): + + @staticmethod + def forward(ctx, tokens, positions, base, F0=1): + ctx.save_for_backward(positions) + ctx.saved_base = base + ctx.saved_F0 = F0 + # tokens = tokens.clone() # uncomment this if inplace doesn't work + _kernels.rope_2d( tokens, positions, base, F0 ) + ctx.mark_dirty(tokens) + return tokens + + @staticmethod + def backward(ctx, grad_res): + positions, base, F0 = ctx.saved_tensors[0], ctx.saved_base, ctx.saved_F0 + _kernels.rope_2d( grad_res, positions, base, -F0 ) + ctx.mark_dirty(grad_res) + return grad_res, None, None, None + + +class cuRoPE2D(torch.nn.Module): + def __init__(self, freq=100.0, F0=1.0): + super().__init__() + self.base = freq + self.F0 = F0 + + def forward(self, tokens, positions): + cuRoPE2D_func.apply( tokens.transpose(1,2), positions, self.base, self.F0 ) + return tokens \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/models/curope/kernels.cu b/submodules/mast3r/dust3r/croco/models/curope/kernels.cu new file mode 100644 index 0000000000000000000000000000000000000000..7156cd1bb935cb1f0be45e58add53f9c21505c20 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/models/curope/kernels.cu @@ -0,0 +1,108 @@ +/* + Copyright (C) 2022-present Naver Corporation. All rights reserved. + Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +*/ + +#include +#include +#include +#include + +#define CHECK_CUDA(tensor) {\ + TORCH_CHECK((tensor).is_cuda(), #tensor " is not in cuda memory"); \ + TORCH_CHECK((tensor).is_contiguous(), #tensor " is not contiguous"); } +void CHECK_KERNEL() {auto error = cudaGetLastError(); TORCH_CHECK( error == cudaSuccess, cudaGetErrorString(error));} + + +template < typename scalar_t > +__global__ void rope_2d_cuda_kernel( + //scalar_t* __restrict__ tokens, + torch::PackedTensorAccessor32 tokens, + const int64_t* __restrict__ pos, + const float base, + const float fwd ) + // const int N, const int H, const int D ) +{ + // tokens shape = (B, N, H, D) + const int N = tokens.size(1); + const int H = tokens.size(2); + const int D = tokens.size(3); + + // each block update a single token, for all heads + // each thread takes care of a single output + extern __shared__ float shared[]; + float* shared_inv_freq = shared + D; + + const int b = blockIdx.x / N; + const int n = blockIdx.x % N; + + const int Q = D / 4; + // one token = [0..Q : Q..2Q : 2Q..3Q : 3Q..D] + // u_Y v_Y u_X v_X + + // shared memory: first, compute inv_freq + if (threadIdx.x < Q) + shared_inv_freq[threadIdx.x] = fwd / powf(base, threadIdx.x/float(Q)); + __syncthreads(); + + // start of X or Y part + const int X = threadIdx.x < D/2 ? 0 : 1; + const int m = (X*D/2) + (threadIdx.x % Q); // index of u_Y or u_X + + // grab the cos,sin appropriate for me + const float freq = pos[blockIdx.x*2+X] * shared_inv_freq[threadIdx.x % Q]; + const float cos = cosf(freq); + const float sin = sinf(freq); + /* + float* shared_cos_sin = shared + D + D/4; + if ((threadIdx.x % (D/2)) < Q) + shared_cos_sin[m+0] = cosf(freq); + else + shared_cos_sin[m+Q] = sinf(freq); + __syncthreads(); + const float cos = shared_cos_sin[m+0]; + const float sin = shared_cos_sin[m+Q]; + */ + + for (int h = 0; h < H; h++) + { + // then, load all the token for this head in shared memory + shared[threadIdx.x] = tokens[b][n][h][threadIdx.x]; + __syncthreads(); + + const float u = shared[m]; + const float v = shared[m+Q]; + + // write output + if ((threadIdx.x % (D/2)) < Q) + tokens[b][n][h][threadIdx.x] = u*cos - v*sin; + else + tokens[b][n][h][threadIdx.x] = v*cos + u*sin; + } +} + +void rope_2d_cuda( torch::Tensor tokens, const torch::Tensor pos, const float base, const float fwd ) +{ + const int B = tokens.size(0); // batch size + const int N = tokens.size(1); // sequence length + const int H = tokens.size(2); // number of heads + const int D = tokens.size(3); // dimension per head + + TORCH_CHECK(tokens.stride(3) == 1 && tokens.stride(2) == D, "tokens are not contiguous"); + TORCH_CHECK(pos.is_contiguous(), "positions are not contiguous"); + TORCH_CHECK(pos.size(0) == B && pos.size(1) == N && pos.size(2) == 2, "bad pos.shape"); + TORCH_CHECK(D % 4 == 0, "token dim must be multiple of 4"); + + // one block for each layer, one thread per local-max + const int THREADS_PER_BLOCK = D; + const int N_BLOCKS = B * N; // each block takes care of H*D values + const int SHARED_MEM = sizeof(float) * (D + D/4); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF(tokens.type(), "rope_2d_cuda", ([&] { + rope_2d_cuda_kernel <<>> ( + //tokens.data_ptr(), + tokens.packed_accessor32(), + pos.data_ptr(), + base, fwd); //, N, H, D ); + })); +} diff --git a/submodules/mast3r/dust3r/croco/models/curope/setup.py b/submodules/mast3r/dust3r/croco/models/curope/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..230632ed05e309200e8f93a3a852072333975009 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/models/curope/setup.py @@ -0,0 +1,34 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +from setuptools import setup +from torch import cuda +from torch.utils.cpp_extension import BuildExtension, CUDAExtension + +# compile for all possible CUDA architectures +all_cuda_archs = cuda.get_gencode_flags().replace('compute=','arch=').split() +# alternatively, you can list cuda archs that you want, eg: +# all_cuda_archs = [ + # '-gencode', 'arch=compute_70,code=sm_70', + # '-gencode', 'arch=compute_75,code=sm_75', + # '-gencode', 'arch=compute_80,code=sm_80', + # '-gencode', 'arch=compute_86,code=sm_86' +# ] + +setup( + name = 'curope', + ext_modules = [ + CUDAExtension( + name='curope', + sources=[ + "curope.cpp", + "kernels.cu", + ], + extra_compile_args = dict( + nvcc=['-O3','--ptxas-options=-v',"--use_fast_math"]+all_cuda_archs, + cxx=['-O3']) + ) + ], + cmdclass = { + 'build_ext': BuildExtension + }) diff --git a/submodules/mast3r/dust3r/croco/models/dpt_block.py b/submodules/mast3r/dust3r/croco/models/dpt_block.py new file mode 100644 index 0000000000000000000000000000000000000000..d4ddfb74e2769ceca88720d4c730e00afd71c763 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/models/dpt_block.py @@ -0,0 +1,450 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# DPT head for ViTs +# -------------------------------------------------------- +# References: +# https://github.com/isl-org/DPT +# https://github.com/EPFL-VILAB/MultiMAE/blob/main/multimae/output_adapters.py + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange, repeat +from typing import Union, Tuple, Iterable, List, Optional, Dict + +def pair(t): + return t if isinstance(t, tuple) else (t, t) + +def make_scratch(in_shape, out_shape, groups=1, expand=False): + scratch = nn.Module() + + out_shape1 = out_shape + out_shape2 = out_shape + out_shape3 = out_shape + out_shape4 = out_shape + if expand == True: + out_shape1 = out_shape + out_shape2 = out_shape * 2 + out_shape3 = out_shape * 4 + out_shape4 = out_shape * 8 + + scratch.layer1_rn = nn.Conv2d( + in_shape[0], + out_shape1, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer2_rn = nn.Conv2d( + in_shape[1], + out_shape2, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer3_rn = nn.Conv2d( + in_shape[2], + out_shape3, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer4_rn = nn.Conv2d( + in_shape[3], + out_shape4, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + + scratch.layer_rn = nn.ModuleList([ + scratch.layer1_rn, + scratch.layer2_rn, + scratch.layer3_rn, + scratch.layer4_rn, + ]) + + return scratch + +class ResidualConvUnit_custom(nn.Module): + """Residual convolution module.""" + + def __init__(self, features, activation, bn): + """Init. + Args: + features (int): number of features + """ + super().__init__() + + self.bn = bn + + self.groups = 1 + + self.conv1 = nn.Conv2d( + features, + features, + kernel_size=3, + stride=1, + padding=1, + bias=not self.bn, + groups=self.groups, + ) + + self.conv2 = nn.Conv2d( + features, + features, + kernel_size=3, + stride=1, + padding=1, + bias=not self.bn, + groups=self.groups, + ) + + if self.bn == True: + self.bn1 = nn.BatchNorm2d(features) + self.bn2 = nn.BatchNorm2d(features) + + self.activation = activation + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, x): + """Forward pass. + Args: + x (tensor): input + Returns: + tensor: output + """ + + out = self.activation(x) + out = self.conv1(out) + if self.bn == True: + out = self.bn1(out) + + out = self.activation(out) + out = self.conv2(out) + if self.bn == True: + out = self.bn2(out) + + if self.groups > 1: + out = self.conv_merge(out) + + return self.skip_add.add(out, x) + +class FeatureFusionBlock_custom(nn.Module): + """Feature fusion block.""" + + def __init__( + self, + features, + activation, + deconv=False, + bn=False, + expand=False, + align_corners=True, + width_ratio=1, + ): + """Init. + Args: + features (int): number of features + """ + super(FeatureFusionBlock_custom, self).__init__() + self.width_ratio = width_ratio + + self.deconv = deconv + self.align_corners = align_corners + + self.groups = 1 + + self.expand = expand + out_features = features + if self.expand == True: + out_features = features // 2 + + self.out_conv = nn.Conv2d( + features, + out_features, + kernel_size=1, + stride=1, + padding=0, + bias=True, + groups=1, + ) + + self.resConfUnit1 = ResidualConvUnit_custom(features, activation, bn) + self.resConfUnit2 = ResidualConvUnit_custom(features, activation, bn) + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, *xs): + """Forward pass. + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + res = self.resConfUnit1(xs[1]) + if self.width_ratio != 1: + res = F.interpolate(res, size=(output.shape[2], output.shape[3]), mode='bilinear') + + output = self.skip_add.add(output, res) + # output += res + + output = self.resConfUnit2(output) + + if self.width_ratio != 1: + # and output.shape[3] < self.width_ratio * output.shape[2] + #size=(image.shape[]) + if (output.shape[3] / output.shape[2]) < (2 / 3) * self.width_ratio: + shape = 3 * output.shape[3] + else: + shape = int(self.width_ratio * 2 * output.shape[2]) + output = F.interpolate(output, size=(2* output.shape[2], shape), mode='bilinear') + else: + output = nn.functional.interpolate(output, scale_factor=2, + mode="bilinear", align_corners=self.align_corners) + output = self.out_conv(output) + return output + +def make_fusion_block(features, use_bn, width_ratio=1): + return FeatureFusionBlock_custom( + features, + nn.ReLU(False), + deconv=False, + bn=use_bn, + expand=False, + align_corners=True, + width_ratio=width_ratio, + ) + +class Interpolate(nn.Module): + """Interpolation module.""" + + def __init__(self, scale_factor, mode, align_corners=False): + """Init. + Args: + scale_factor (float): scaling + mode (str): interpolation mode + """ + super(Interpolate, self).__init__() + + self.interp = nn.functional.interpolate + self.scale_factor = scale_factor + self.mode = mode + self.align_corners = align_corners + + def forward(self, x): + """Forward pass. + Args: + x (tensor): input + Returns: + tensor: interpolated data + """ + + x = self.interp( + x, + scale_factor=self.scale_factor, + mode=self.mode, + align_corners=self.align_corners, + ) + + return x + +class DPTOutputAdapter(nn.Module): + """DPT output adapter. + + :param num_cahnnels: Number of output channels + :param stride_level: tride level compared to the full-sized image. + E.g. 4 for 1/4th the size of the image. + :param patch_size_full: Int or tuple of the patch size over the full image size. + Patch size for smaller inputs will be computed accordingly. + :param hooks: Index of intermediate layers + :param layer_dims: Dimension of intermediate layers + :param feature_dim: Feature dimension + :param last_dim: out_channels/in_channels for the last two Conv2d when head_type == regression + :param use_bn: If set to True, activates batch norm + :param dim_tokens_enc: Dimension of tokens coming from encoder + """ + + def __init__(self, + num_channels: int = 1, + stride_level: int = 1, + patch_size: Union[int, Tuple[int, int]] = 16, + main_tasks: Iterable[str] = ('rgb',), + hooks: List[int] = [2, 5, 8, 11], + layer_dims: List[int] = [96, 192, 384, 768], + feature_dim: int = 256, + last_dim: int = 32, + use_bn: bool = False, + dim_tokens_enc: Optional[int] = None, + head_type: str = 'regression', + output_width_ratio=1, + **kwargs): + super().__init__() + self.num_channels = num_channels + self.stride_level = stride_level + self.patch_size = pair(patch_size) + self.main_tasks = main_tasks + self.hooks = hooks + self.layer_dims = layer_dims + self.feature_dim = feature_dim + self.dim_tokens_enc = dim_tokens_enc * len(self.main_tasks) if dim_tokens_enc is not None else None + self.head_type = head_type + + # Actual patch height and width, taking into account stride of input + self.P_H = max(1, self.patch_size[0] // stride_level) + self.P_W = max(1, self.patch_size[1] // stride_level) + + self.scratch = make_scratch(layer_dims, feature_dim, groups=1, expand=False) + + self.scratch.refinenet1 = make_fusion_block(feature_dim, use_bn, output_width_ratio) + self.scratch.refinenet2 = make_fusion_block(feature_dim, use_bn, output_width_ratio) + self.scratch.refinenet3 = make_fusion_block(feature_dim, use_bn, output_width_ratio) + self.scratch.refinenet4 = make_fusion_block(feature_dim, use_bn, output_width_ratio) + + if self.head_type == 'regression': + # The "DPTDepthModel" head + self.head = nn.Sequential( + nn.Conv2d(feature_dim, feature_dim // 2, kernel_size=3, stride=1, padding=1), + Interpolate(scale_factor=2, mode="bilinear", align_corners=True), + nn.Conv2d(feature_dim // 2, last_dim, kernel_size=3, stride=1, padding=1), + nn.ReLU(True), + nn.Conv2d(last_dim, self.num_channels, kernel_size=1, stride=1, padding=0) + ) + elif self.head_type == 'semseg': + # The "DPTSegmentationModel" head + self.head = nn.Sequential( + nn.Conv2d(feature_dim, feature_dim, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(feature_dim) if use_bn else nn.Identity(), + nn.ReLU(True), + nn.Dropout(0.1, False), + nn.Conv2d(feature_dim, self.num_channels, kernel_size=1), + Interpolate(scale_factor=2, mode="bilinear", align_corners=True), + ) + else: + raise ValueError('DPT head_type must be "regression" or "semseg".') + + if self.dim_tokens_enc is not None: + self.init(dim_tokens_enc=dim_tokens_enc) + + def init(self, dim_tokens_enc=768): + """ + Initialize parts of decoder that are dependent on dimension of encoder tokens. + Should be called when setting up MultiMAE. + + :param dim_tokens_enc: Dimension of tokens coming from encoder + """ + #print(dim_tokens_enc) + + # Set up activation postprocessing layers + if isinstance(dim_tokens_enc, int): + dim_tokens_enc = 4 * [dim_tokens_enc] + + self.dim_tokens_enc = [dt * len(self.main_tasks) for dt in dim_tokens_enc] + + self.act_1_postprocess = nn.Sequential( + nn.Conv2d( + in_channels=self.dim_tokens_enc[0], + out_channels=self.layer_dims[0], + kernel_size=1, stride=1, padding=0, + ), + nn.ConvTranspose2d( + in_channels=self.layer_dims[0], + out_channels=self.layer_dims[0], + kernel_size=4, stride=4, padding=0, + bias=True, dilation=1, groups=1, + ) + ) + + self.act_2_postprocess = nn.Sequential( + nn.Conv2d( + in_channels=self.dim_tokens_enc[1], + out_channels=self.layer_dims[1], + kernel_size=1, stride=1, padding=0, + ), + nn.ConvTranspose2d( + in_channels=self.layer_dims[1], + out_channels=self.layer_dims[1], + kernel_size=2, stride=2, padding=0, + bias=True, dilation=1, groups=1, + ) + ) + + self.act_3_postprocess = nn.Sequential( + nn.Conv2d( + in_channels=self.dim_tokens_enc[2], + out_channels=self.layer_dims[2], + kernel_size=1, stride=1, padding=0, + ) + ) + + self.act_4_postprocess = nn.Sequential( + nn.Conv2d( + in_channels=self.dim_tokens_enc[3], + out_channels=self.layer_dims[3], + kernel_size=1, stride=1, padding=0, + ), + nn.Conv2d( + in_channels=self.layer_dims[3], + out_channels=self.layer_dims[3], + kernel_size=3, stride=2, padding=1, + ) + ) + + self.act_postprocess = nn.ModuleList([ + self.act_1_postprocess, + self.act_2_postprocess, + self.act_3_postprocess, + self.act_4_postprocess + ]) + + def adapt_tokens(self, encoder_tokens): + # Adapt tokens + x = [] + x.append(encoder_tokens[:, :]) + x = torch.cat(x, dim=-1) + return x + + def forward(self, encoder_tokens: List[torch.Tensor], image_size): + #input_info: Dict): + assert self.dim_tokens_enc is not None, 'Need to call init(dim_tokens_enc) function first' + H, W = image_size + + # Number of patches in height and width + N_H = H // (self.stride_level * self.P_H) + N_W = W // (self.stride_level * self.P_W) + + # Hook decoder onto 4 layers from specified ViT layers + layers = [encoder_tokens[hook] for hook in self.hooks] + + # Extract only task-relevant tokens and ignore global tokens. + layers = [self.adapt_tokens(l) for l in layers] + + # Reshape tokens to spatial representation + layers = [rearrange(l, 'b (nh nw) c -> b c nh nw', nh=N_H, nw=N_W) for l in layers] + + layers = [self.act_postprocess[idx](l) for idx, l in enumerate(layers)] + # Project layers to chosen feature dim + layers = [self.scratch.layer_rn[idx](l) for idx, l in enumerate(layers)] + + # Fuse layers using refinement stages + path_4 = self.scratch.refinenet4(layers[3]) + path_3 = self.scratch.refinenet3(path_4, layers[2]) + path_2 = self.scratch.refinenet2(path_3, layers[1]) + path_1 = self.scratch.refinenet1(path_2, layers[0]) + + # Output head + out = self.head(path_1) + + return out diff --git a/submodules/mast3r/dust3r/croco/models/head_downstream.py b/submodules/mast3r/dust3r/croco/models/head_downstream.py new file mode 100644 index 0000000000000000000000000000000000000000..bd40c91ba244d6c3522c6efd4ed4d724b7bdc650 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/models/head_downstream.py @@ -0,0 +1,58 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# Heads for downstream tasks +# -------------------------------------------------------- + +""" +A head is a module where the __init__ defines only the head hyperparameters. +A method setup(croconet) takes a CroCoNet and set all layers according to the head and croconet attributes. +The forward takes the features as well as a dictionary img_info containing the keys 'width' and 'height' +""" + +import torch +import torch.nn as nn +from .dpt_block import DPTOutputAdapter + + +class PixelwiseTaskWithDPT(nn.Module): + """ DPT module for CroCo. + by default, hooks_idx will be equal to: + * for encoder-only: 4 equally spread layers + * for encoder+decoder: last encoder + 3 equally spread layers of the decoder + """ + + def __init__(self, *, hooks_idx=None, layer_dims=[96,192,384,768], + output_width_ratio=1, num_channels=1, postprocess=None, **kwargs): + super(PixelwiseTaskWithDPT, self).__init__() + self.return_all_blocks = True # backbone needs to return all layers + self.postprocess = postprocess + self.output_width_ratio = output_width_ratio + self.num_channels = num_channels + self.hooks_idx = hooks_idx + self.layer_dims = layer_dims + + def setup(self, croconet): + dpt_args = {'output_width_ratio': self.output_width_ratio, 'num_channels': self.num_channels} + if self.hooks_idx is None: + if hasattr(croconet, 'dec_blocks'): # encoder + decoder + step = {8: 3, 12: 4, 24: 8}[croconet.dec_depth] + hooks_idx = [croconet.dec_depth+croconet.enc_depth-1-i*step for i in range(3,-1,-1)] + else: # encoder only + step = croconet.enc_depth//4 + hooks_idx = [croconet.enc_depth-1-i*step for i in range(3,-1,-1)] + self.hooks_idx = hooks_idx + print(f' PixelwiseTaskWithDPT: automatically setting hook_idxs={self.hooks_idx}') + dpt_args['hooks'] = self.hooks_idx + dpt_args['layer_dims'] = self.layer_dims + self.dpt = DPTOutputAdapter(**dpt_args) + dim_tokens = [croconet.enc_embed_dim if hook0: + pos_embed = np.concatenate([np.zeros([n_cls_token, embed_dim]), pos_embed], axis=0) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + assert embed_dim % 2 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2) + emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2) + + emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) + return emb + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=float) + omega /= embed_dim / 2. + omega = 1. / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +# -------------------------------------------------------- +# Interpolate position embeddings for high-resolution +# References: +# MAE: https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py +# DeiT: https://github.com/facebookresearch/deit +# -------------------------------------------------------- +def interpolate_pos_embed(model, checkpoint_model): + if 'pos_embed' in checkpoint_model: + pos_embed_checkpoint = checkpoint_model['pos_embed'] + embedding_size = pos_embed_checkpoint.shape[-1] + num_patches = model.patch_embed.num_patches + num_extra_tokens = model.pos_embed.shape[-2] - num_patches + # height (== width) for the checkpoint position embedding + orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5) + # height (== width) for the new position embedding + new_size = int(num_patches ** 0.5) + # class_token and dist_token are kept unchanged + if orig_size != new_size: + print("Position interpolate from %dx%d to %dx%d" % (orig_size, orig_size, new_size, new_size)) + extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens] + # only the position tokens are interpolated + pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:] + pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2) + pos_tokens = torch.nn.functional.interpolate( + pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False) + pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2) + new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1) + checkpoint_model['pos_embed'] = new_pos_embed + + +#---------------------------------------------------------- +# RoPE2D: RoPE implementation in 2D +#---------------------------------------------------------- + +try: + from models.curope import cuRoPE2D + RoPE2D = cuRoPE2D +except ImportError: + print('Warning, cannot find cuda-compiled version of RoPE2D, using a slow pytorch version instead') + + class RoPE2D(torch.nn.Module): + + def __init__(self, freq=100.0, F0=1.0): + super().__init__() + self.base = freq + self.F0 = F0 + self.cache = {} + + def get_cos_sin(self, D, seq_len, device, dtype): + if (D,seq_len,device,dtype) not in self.cache: + inv_freq = 1.0 / (self.base ** (torch.arange(0, D, 2).float().to(device) / D)) + t = torch.arange(seq_len, device=device, dtype=inv_freq.dtype) + freqs = torch.einsum("i,j->ij", t, inv_freq).to(dtype) + freqs = torch.cat((freqs, freqs), dim=-1) + cos = freqs.cos() # (Seq, Dim) + sin = freqs.sin() + self.cache[D,seq_len,device,dtype] = (cos,sin) + return self.cache[D,seq_len,device,dtype] + + @staticmethod + def rotate_half(x): + x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + def apply_rope1d(self, tokens, pos1d, cos, sin): + assert pos1d.ndim==2 + cos = torch.nn.functional.embedding(pos1d, cos)[:, None, :, :] + sin = torch.nn.functional.embedding(pos1d, sin)[:, None, :, :] + return (tokens * cos) + (self.rotate_half(tokens) * sin) + + def forward(self, tokens, positions): + """ + input: + * tokens: batch_size x nheads x ntokens x dim + * positions: batch_size x ntokens x 2 (y and x position of each token) + output: + * tokens after appplying RoPE2D (batch_size x nheads x ntokens x dim) + """ + assert tokens.size(3)%2==0, "number of dimensions should be a multiple of two" + D = tokens.size(3) // 2 + assert positions.ndim==3 and positions.shape[-1] == 2 # Batch, Seq, 2 + cos, sin = self.get_cos_sin(D, int(positions.max())+1, tokens.device, tokens.dtype) + # split features into two along the feature dimension, and apply rope1d on each half + y, x = tokens.chunk(2, dim=-1) + y = self.apply_rope1d(y, positions[:,:,0], cos, sin) + x = self.apply_rope1d(x, positions[:,:,1], cos, sin) + tokens = torch.cat((y, x), dim=-1) + return tokens \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/pretrain.py b/submodules/mast3r/dust3r/croco/pretrain.py new file mode 100644 index 0000000000000000000000000000000000000000..2c45e488015ef5380c71d0381ff453fdb860759e --- /dev/null +++ b/submodules/mast3r/dust3r/croco/pretrain.py @@ -0,0 +1,254 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Pre-training CroCo +# -------------------------------------------------------- +# References: +# MAE: https://github.com/facebookresearch/mae +# DeiT: https://github.com/facebookresearch/deit +# BEiT: https://github.com/microsoft/unilm/tree/master/beit +# -------------------------------------------------------- +import argparse +import datetime +import json +import numpy as np +import os +import sys +import time +import math +from pathlib import Path +from typing import Iterable + +import torch +import torch.distributed as dist +import torch.backends.cudnn as cudnn +from torch.utils.tensorboard import SummaryWriter +import torchvision.transforms as transforms +import torchvision.datasets as datasets + +import utils.misc as misc +from utils.misc import NativeScalerWithGradNormCount as NativeScaler +from models.croco import CroCoNet +from models.criterion import MaskedMSE +from datasets.pairs_dataset import PairsDataset + + +def get_args_parser(): + parser = argparse.ArgumentParser('CroCo pre-training', add_help=False) + # model and criterion + parser.add_argument('--model', default='CroCoNet()', type=str, help="string containing the model to build") + parser.add_argument('--norm_pix_loss', default=1, choices=[0,1], help="apply per-patch mean/std normalization before applying the loss") + # dataset + parser.add_argument('--dataset', default='habitat_release', type=str, help="training set") + parser.add_argument('--transforms', default='crop224+acolor', type=str, help="transforms to apply") # in the paper, we also use some homography and rotation, but find later that they were not useful or even harmful + # training + parser.add_argument('--seed', default=0, type=int, help="Random seed") + parser.add_argument('--batch_size', default=64, type=int, help="Batch size per GPU (effective batch size is batch_size * accum_iter * # gpus") + parser.add_argument('--epochs', default=800, type=int, help="Maximum number of epochs for the scheduler") + parser.add_argument('--max_epoch', default=400, type=int, help="Stop training at this epoch") + parser.add_argument('--accum_iter', default=1, type=int, help="Accumulate gradient iterations (for increasing the effective batch size under memory constraints)") + parser.add_argument('--weight_decay', type=float, default=0.05, help="weight decay (default: 0.05)") + parser.add_argument('--lr', type=float, default=None, metavar='LR', help='learning rate (absolute lr)') + parser.add_argument('--blr', type=float, default=1.5e-4, metavar='LR', help='base learning rate: absolute_lr = base_lr * total_batch_size / 256') + parser.add_argument('--min_lr', type=float, default=0., metavar='LR', help='lower lr bound for cyclic schedulers that hit 0') + parser.add_argument('--warmup_epochs', type=int, default=40, metavar='N', help='epochs to warmup LR') + parser.add_argument('--amp', type=int, default=1, choices=[0,1], help="Use Automatic Mixed Precision for pretraining") + # others + parser.add_argument('--num_workers', default=8, type=int) + parser.add_argument('--world_size', default=1, type=int, help='number of distributed processes') + parser.add_argument('--local_rank', default=-1, type=int) + parser.add_argument('--dist_url', default='env://', help='url used to set up distributed training') + parser.add_argument('--save_freq', default=1, type=int, help='frequence (number of epochs) to save checkpoint in checkpoint-last.pth') + parser.add_argument('--keep_freq', default=20, type=int, help='frequence (number of epochs) to save checkpoint in checkpoint-%d.pth') + parser.add_argument('--print_freq', default=20, type=int, help='frequence (number of iterations) to print infos while training') + # paths + parser.add_argument('--output_dir', default='./output/', type=str, help="path where to save the output") + parser.add_argument('--data_dir', default='./data/', type=str, help="path where data are stored") + return parser + + + + +def main(args): + misc.init_distributed_mode(args) + global_rank = misc.get_rank() + world_size = misc.get_world_size() + + print("output_dir: "+args.output_dir) + if args.output_dir: + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + + # auto resume + last_ckpt_fname = os.path.join(args.output_dir, f'checkpoint-last.pth') + args.resume = last_ckpt_fname if os.path.isfile(last_ckpt_fname) else None + + print('job dir: {}'.format(os.path.dirname(os.path.realpath(__file__)))) + print("{}".format(args).replace(', ', ',\n')) + + device = "cuda" if torch.cuda.is_available() else "cpu" + device = torch.device(device) + + # fix the seed + seed = args.seed + misc.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + + cudnn.benchmark = True + + ## training dataset and loader + print('Building dataset for {:s} with transforms {:s}'.format(args.dataset, args.transforms)) + dataset = PairsDataset(args.dataset, trfs=args.transforms, data_dir=args.data_dir) + if world_size>1: + sampler_train = torch.utils.data.DistributedSampler( + dataset, num_replicas=world_size, rank=global_rank, shuffle=True + ) + print("Sampler_train = %s" % str(sampler_train)) + else: + sampler_train = torch.utils.data.RandomSampler(dataset) + data_loader_train = torch.utils.data.DataLoader( + dataset, sampler=sampler_train, + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=True, + ) + + ## model + print('Loading model: {:s}'.format(args.model)) + model = eval(args.model) + print('Loading criterion: MaskedMSE(norm_pix_loss={:s})'.format(str(bool(args.norm_pix_loss)))) + criterion = MaskedMSE(norm_pix_loss=bool(args.norm_pix_loss)) + + model.to(device) + model_without_ddp = model + print("Model = %s" % str(model_without_ddp)) + + eff_batch_size = args.batch_size * args.accum_iter * misc.get_world_size() + if args.lr is None: # only base_lr is specified + args.lr = args.blr * eff_batch_size / 256 + print("base lr: %.2e" % (args.lr * 256 / eff_batch_size)) + print("actual lr: %.2e" % args.lr) + print("accumulate grad iterations: %d" % args.accum_iter) + print("effective batch size: %d" % eff_batch_size) + + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], find_unused_parameters=True, static_graph=True) + model_without_ddp = model.module + + param_groups = misc.get_parameter_groups(model_without_ddp, args.weight_decay) # following timm: set wd as 0 for bias and norm layers + optimizer = torch.optim.AdamW(param_groups, lr=args.lr, betas=(0.9, 0.95)) + print(optimizer) + loss_scaler = NativeScaler() + + misc.load_model(args=args, model_without_ddp=model_without_ddp, optimizer=optimizer, loss_scaler=loss_scaler) + + if global_rank == 0 and args.output_dir is not None: + log_writer = SummaryWriter(log_dir=args.output_dir) + else: + log_writer = None + + print(f"Start training until {args.max_epoch} epochs") + start_time = time.time() + for epoch in range(args.start_epoch, args.max_epoch): + if world_size>1: + data_loader_train.sampler.set_epoch(epoch) + + train_stats = train_one_epoch( + model, criterion, data_loader_train, + optimizer, device, epoch, loss_scaler, + log_writer=log_writer, + args=args + ) + + if args.output_dir and epoch % args.save_freq == 0 : + misc.save_model( + args=args, model_without_ddp=model_without_ddp, optimizer=optimizer, + loss_scaler=loss_scaler, epoch=epoch, fname='last') + + if args.output_dir and (epoch % args.keep_freq == 0 or epoch + 1 == args.max_epoch) and (epoch>0 or args.max_epoch==1): + misc.save_model( + args=args, model_without_ddp=model_without_ddp, optimizer=optimizer, + loss_scaler=loss_scaler, epoch=epoch) + + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + 'epoch': epoch,} + + if args.output_dir and misc.is_main_process(): + if log_writer is not None: + log_writer.flush() + with open(os.path.join(args.output_dir, "log.txt"), mode="a", encoding="utf-8") as f: + f.write(json.dumps(log_stats) + "\n") + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + + + +def train_one_epoch(model: torch.nn.Module, criterion: torch.nn.Module, + data_loader: Iterable, optimizer: torch.optim.Optimizer, + device: torch.device, epoch: int, loss_scaler, + log_writer=None, + args=None): + model.train(True) + metric_logger = misc.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', misc.SmoothedValue(window_size=1, fmt='{value:.6f}')) + header = 'Epoch: [{}]'.format(epoch) + accum_iter = args.accum_iter + + optimizer.zero_grad() + + if log_writer is not None: + print('log_dir: {}'.format(log_writer.log_dir)) + + for data_iter_step, (image1, image2) in enumerate(metric_logger.log_every(data_loader, args.print_freq, header)): + + # we use a per iteration lr scheduler + if data_iter_step % accum_iter == 0: + misc.adjust_learning_rate(optimizer, data_iter_step / len(data_loader) + epoch, args) + + image1 = image1.to(device, non_blocking=True) + image2 = image2.to(device, non_blocking=True) + with torch.cuda.amp.autocast(enabled=bool(args.amp)): + out, mask, target = model(image1, image2) + loss = criterion(out, mask, target) + + loss_value = loss.item() + + if not math.isfinite(loss_value): + print("Loss is {}, stopping training".format(loss_value)) + sys.exit(1) + + loss /= accum_iter + loss_scaler(loss, optimizer, parameters=model.parameters(), + update_grad=(data_iter_step + 1) % accum_iter == 0) + if (data_iter_step + 1) % accum_iter == 0: + optimizer.zero_grad() + + torch.cuda.synchronize() + + metric_logger.update(loss=loss_value) + + lr = optimizer.param_groups[0]["lr"] + metric_logger.update(lr=lr) + + loss_value_reduce = misc.all_reduce_mean(loss_value) + if log_writer is not None and ((data_iter_step + 1) % (accum_iter*args.print_freq)) == 0: + # x-axis is based on epoch_1000x in the tensorboard, calibrating differences curves when batch size changes + epoch_1000x = int((data_iter_step / len(data_loader) + epoch) * 1000) + log_writer.add_scalar('train_loss', loss_value_reduce, epoch_1000x) + log_writer.add_scalar('lr', lr, epoch_1000x) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + return {k: meter.global_avg for k, meter in metric_logger.meters.items()} + + + +if __name__ == '__main__': + args = get_args_parser() + args = args.parse_args() + main(args) diff --git a/submodules/mast3r/dust3r/croco/stereoflow/README.MD b/submodules/mast3r/dust3r/croco/stereoflow/README.MD new file mode 100644 index 0000000000000000000000000000000000000000..81595380fadd274b523e0cf77921b1b65cbedb34 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/stereoflow/README.MD @@ -0,0 +1,318 @@ +## CroCo-Stereo and CroCo-Flow + +This README explains how to use CroCo-Stereo and CroCo-Flow as well as how they were trained. +All commands should be launched from the root directory. + +### Simple inference example + +We provide a simple inference exemple for CroCo-Stereo and CroCo-Flow in the Totebook `croco-stereo-flow-demo.ipynb`. +Before running it, please download the trained models with: +``` +bash stereoflow/download_model.sh crocostereo.pth +bash stereoflow/download_model.sh crocoflow.pth +``` + +### Prepare data for training or evaluation + +Put the datasets used for training/evaluation in `./data/stereoflow` (or update the paths at the top of `stereoflow/datasets_stereo.py` and `stereoflow/datasets_flow.py`). +Please find below on the file structure should look for each dataset: +
+FlyingChairs + +``` +./data/stereoflow/FlyingChairs/ +└───chairs_split.txt +└───data/ + └─── ... +``` +
+ +
+MPI-Sintel + +``` +./data/stereoflow/MPI-Sintel/ +└───training/ +│ └───clean/ +│ └───final/ +│ └───flow/ +└───test/ + └───clean/ + └───final/ +``` +
+ +
+SceneFlow (including FlyingThings) + +``` +./data/stereoflow/SceneFlow/ +└───Driving/ +│ └───disparity/ +│ └───frames_cleanpass/ +│ └───frames_finalpass/ +└───FlyingThings/ +│ └───disparity/ +│ └───frames_cleanpass/ +│ └───frames_finalpass/ +│ └───optical_flow/ +└───Monkaa/ + └───disparity/ + └───frames_cleanpass/ + └───frames_finalpass/ +``` +
+ +
+TartanAir + +``` +./data/stereoflow/TartanAir/ +└───abandonedfactory/ +│ └───.../ +└───abandonedfactory_night/ +│ └───.../ +└───.../ +``` +
+ +
+Booster + +``` +./data/stereoflow/booster_gt/ +└───train/ + └───balanced/ + └───Bathroom/ + └───Bedroom/ + └───... +``` +
+ +
+CREStereo + +``` +./data/stereoflow/crenet_stereo_trainset/ +└───stereo_trainset/ + └───crestereo/ + └───hole/ + └───reflective/ + └───shapenet/ + └───tree/ +``` +
+ +
+ETH3D Two-view Low-res + +``` +./data/stereoflow/eth3d_lowres/ +└───test/ +│ └───lakeside_1l/ +│ └───... +└───train/ +│ └───delivery_area_1l/ +│ └───... +└───train_gt/ + └───delivery_area_1l/ + └───... +``` +
+ +
+KITTI 2012 + +``` +./data/stereoflow/kitti-stereo-2012/ +└───testing/ +│ └───colored_0/ +│ └───colored_1/ +└───training/ + └───colored_0/ + └───colored_1/ + └───disp_occ/ + └───flow_occ/ +``` +
+ +
+KITTI 2015 + +``` +./data/stereoflow/kitti-stereo-2015/ +└───testing/ +│ └───image_2/ +│ └───image_3/ +└───training/ + └───image_2/ + └───image_3/ + └───disp_occ_0/ + └───flow_occ/ +``` +
+ +
+Middlebury + +``` +./data/stereoflow/middlebury +└───2005/ +│ └───train/ +│ └───Art/ +│ └───... +└───2006/ +│ └───Aloe/ +│ └───Baby1/ +│ └───... +└───2014/ +│ └───Adirondack-imperfect/ +│ └───Adirondack-perfect/ +│ └───... +└───2021/ +│ └───data/ +│ └───artroom1/ +│ └───artroom2/ +│ └───... +└───MiddEval3_F/ + └───test/ + │ └───Australia/ + │ └───... + └───train/ + └───Adirondack/ + └───... +``` +
+ +
+Spring + +``` +./data/stereoflow/spring/ +└───test/ +│ └───0003/ +│ └───... +└───train/ + └───0001/ + └───... +``` +
+ + +### CroCo-Stereo + +##### Main model + +The main training of CroCo-Stereo was performed on a series of datasets, and it was used as it for Middlebury v3 benchmark. + +``` +# Download the model +bash stereoflow/download_model.sh crocostereo.pth +# Middlebury v3 submission +python stereoflow/test.py --model stereoflow_models/crocostereo.pth --dataset "MdEval3('all_full')" --save submission --tile_overlap 0.9 +# Training command that was used, using checkpoint-last.pth +python -u stereoflow/train.py stereo --criterion "LaplacianLossBounded2()" --dataset "CREStereo('train')+SceneFlow('train_allpass')+30*ETH3DLowRes('train')+50*Md05('train')+50*Md06('train')+50*Md14('train')+50*Md21('train')+50*MdEval3('train_full')+Booster('train_balanced')" --val_dataset "SceneFlow('test1of100_finalpass')+SceneFlow('test1of100_cleanpass')+ETH3DLowRes('subval')+Md05('subval')+Md06('subval')+Md14('subval')+Md21('subval')+MdEval3('subval_full')+Booster('subval_balanced')" --lr 3e-5 --batch_size 6 --epochs 32 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --output_dir xps/crocostereo/main/ +# or it can be launched on multiple gpus (while maintaining the effective batch size), e.g. on 3 gpus: +torchrun --nproc_per_node 3 stereoflow/train.py stereo --criterion "LaplacianLossBounded2()" --dataset "CREStereo('train')+SceneFlow('train_allpass')+30*ETH3DLowRes('train')+50*Md05('train')+50*Md06('train')+50*Md14('train')+50*Md21('train')+50*MdEval3('train_full')+Booster('train_balanced')" --val_dataset "SceneFlow('test1of100_finalpass')+SceneFlow('test1of100_cleanpass')+ETH3DLowRes('subval')+Md05('subval')+Md06('subval')+Md14('subval')+Md21('subval')+MdEval3('subval_full')+Booster('subval_balanced')" --lr 3e-5 --batch_size 2 --epochs 32 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --output_dir xps/crocostereo/main/ +``` + +For evaluation of validation set, we also provide the model trained on the `subtrain` subset of the training sets. + +``` +# Download the model +bash stereoflow/download_model.sh crocostereo_subtrain.pth +# Evaluation on validation sets +python stereoflow/test.py --model stereoflow_models/crocostereo_subtrain.pth --dataset "MdEval3('subval_full')+ETH3DLowRes('subval')+SceneFlow('test_finalpass')+SceneFlow('test_cleanpass')" --save metrics --tile_overlap 0.9 +# Training command that was used (same as above but on subtrain, using checkpoint-best.pth), can also be launched on multiple gpus +python -u stereoflow/train.py stereo --criterion "LaplacianLossBounded2()" --dataset "CREStereo('train')+SceneFlow('train_allpass')+30*ETH3DLowRes('subtrain')+50*Md05('subtrain')+50*Md06('subtrain')+50*Md14('subtrain')+50*Md21('subtrain')+50*MdEval3('subtrain_full')+Booster('subtrain_balanced')" --val_dataset "SceneFlow('test1of100_finalpass')+SceneFlow('test1of100_cleanpass')+ETH3DLowRes('subval')+Md05('subval')+Md06('subval')+Md14('subval')+Md21('subval')+MdEval3('subval_full')+Booster('subval_balanced')" --lr 3e-5 --batch_size 6 --epochs 32 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --output_dir xps/crocostereo/main_subtrain/ +``` + +##### Other models + +
+ Model for ETH3D + The model used for the submission on ETH3D is trained with the same command but using an unbounded Laplacian loss. + + # Download the model + bash stereoflow/download_model.sh crocostereo_eth3d.pth + # ETH3D submission + python stereoflow/test.py --model stereoflow_models/crocostereo_eth3d.pth --dataset "ETH3DLowRes('all')" --save submission --tile_overlap 0.9 + # Training command that was used + python -u stereoflow/train.py stereo --criterion "LaplacianLoss()" --tile_conf_mode conf_expbeta3 --dataset "CREStereo('train')+SceneFlow('train_allpass')+30*ETH3DLowRes('train')+50*Md05('train')+50*Md06('train')+50*Md14('train')+50*Md21('train')+50*MdEval3('train_full')+Booster('train_balanced')" --val_dataset "SceneFlow('test1of100_finalpass')+SceneFlow('test1of100_cleanpass')+ETH3DLowRes('subval')+Md05('subval')+Md06('subval')+Md14('subval')+Md21('subval')+MdEval3('subval_full')+Booster('subval_balanced')" --lr 3e-5 --batch_size 6 --epochs 32 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --output_dir xps/crocostereo/main_eth3d/ + +
+ +
+ Main model finetuned on Kitti + + # Download the model + bash stereoflow/download_model.sh crocostereo_finetune_kitti.pth + # Kitti submission + python stereoflow/test.py --model stereoflow_models/crocostereo_finetune_kitti.pth --dataset "Kitti15('test')" --save submission --tile_overlap 0.9 + # Training that was used + python -u stereoflow/train.py stereo --crop 352 1216 --criterion "LaplacianLossBounded2()" --dataset "Kitti12('train')+Kitti15('train')" --lr 3e-5 --batch_size 1 --accum_iter 6 --epochs 20 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --start_from stereoflow_models/crocostereo.pth --output_dir xps/crocostereo/finetune_kitti/ --save_every 5 +
+ +
+ Main model finetuned on Spring + + # Download the model + bash stereoflow/download_model.sh crocostereo_finetune_spring.pth + # Spring submission + python stereoflow/test.py --model stereoflow_models/crocostereo_finetune_spring.pth --dataset "Spring('test')" --save submission --tile_overlap 0.9 + # Training command that was used + python -u stereoflow/train.py stereo --criterion "LaplacianLossBounded2()" --dataset "Spring('train')" --lr 3e-5 --batch_size 6 --epochs 8 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --start_from stereoflow_models/crocostereo.pth --output_dir xps/crocostereo/finetune_spring/ +
+ +
+ Smaller models + To train CroCo-Stereo with smaller CroCo pretrained models, simply replace the --pretrained argument. To download the smaller CroCo-Stereo models based on CroCo v2 pretraining with ViT-Base encoder and Small encoder, use bash stereoflow/download_model.sh crocostereo_subtrain_vitb_smalldecoder.pth, and for the model with a ViT-Base encoder and a Base decoder, use bash stereoflow/download_model.sh crocostereo_subtrain_vitb_basedecoder.pth. +
+ + +### CroCo-Flow + +##### Main model + +The main training of CroCo-Flow was performed on the FlyingThings, FlyingChairs, MPI-Sintel and TartanAir datasets. +It was used for our submission to the MPI-Sintel benchmark. + +``` +# Download the model +bash stereoflow/download_model.sh crocoflow.pth +# Evaluation +python stereoflow/test.py --model stereoflow_models/crocoflow.pth --dataset "MPISintel('subval_cleanpass')+MPISintel('subval_finalpass')" --save metrics --tile_overlap 0.9 +# Sintel submission +python stereoflow/test.py --model stereoflow_models/crocoflow.pth --dataset "MPISintel('test_allpass')" --save submission --tile_overlap 0.9 +# Training command that was used, with checkpoint-best.pth +python -u stereoflow/train.py flow --criterion "LaplacianLossBounded()" --dataset "40*MPISintel('subtrain_cleanpass')+40*MPISintel('subtrain_finalpass')+4*FlyingThings('train_allpass')+4*FlyingChairs('train')+TartanAir('train')" --val_dataset "MPISintel('subval_cleanpass')+MPISintel('subval_finalpass')" --lr 2e-5 --batch_size 8 --epochs 240 --img_per_epoch 30000 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --output_dir xps/crocoflow/main/ +``` + +##### Other models + +
+ Main model finetuned on Kitti + + # Download the model + bash stereoflow/download_model.sh crocoflow_finetune_kitti.pth + # Kitti submission + python stereoflow/test.py --model stereoflow_models/crocoflow_finetune_kitti.pth --dataset "Kitti15('test')" --save submission --tile_overlap 0.99 + # Training that was used, with checkpoint-last.pth + python -u stereoflow/train.py flow --crop 352 1216 --criterion "LaplacianLossBounded()" --dataset "Kitti15('train')+Kitti12('train')" --lr 2e-5 --batch_size 1 --accum_iter 8 --epochs 150 --save_every 5 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --start_from stereoflow_models/crocoflow.pth --output_dir xps/crocoflow/finetune_kitti/ +
+ +
+ Main model finetuned on Spring + + # Download the model + bash stereoflow/download_model.sh crocoflow_finetune_spring.pth + # Spring submission + python stereoflow/test.py --model stereoflow_models/crocoflow_finetune_spring.pth --dataset "Spring('test')" --save submission --tile_overlap 0.9 + # Training command that was used, with checkpoint-last.pth + python -u stereoflow/train.py flow --criterion "LaplacianLossBounded()" --dataset "Spring('train')" --lr 2e-5 --batch_size 8 --epochs 12 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --start_from stereoflow_models/crocoflow.pth --output_dir xps/crocoflow/finetune_spring/ +
+ +
+ Smaller models + To train CroCo-Flow with smaller CroCo pretrained models, simply replace the --pretrained argument. To download the smaller CroCo-Flow models based on CroCo v2 pretraining with ViT-Base encoder and Small encoder, use bash stereoflow/download_model.sh crocoflow_vitb_smalldecoder.pth, and for the model with a ViT-Base encoder and a Base decoder, use bash stereoflow/download_model.sh crocoflow_vitb_basedecoder.pth. +
diff --git a/submodules/mast3r/dust3r/croco/stereoflow/augmentor.py b/submodules/mast3r/dust3r/croco/stereoflow/augmentor.py new file mode 100644 index 0000000000000000000000000000000000000000..69e6117151988d94cbc4b385e0d88e982133bf10 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/stereoflow/augmentor.py @@ -0,0 +1,290 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# Data augmentation for training stereo and flow +# -------------------------------------------------------- + +# References +# https://github.com/autonomousvision/unimatch/blob/master/dataloader/stereo/transforms.py +# https://github.com/autonomousvision/unimatch/blob/master/dataloader/flow/transforms.py + + +import numpy as np +import random +from PIL import Image + +import cv2 +cv2.setNumThreads(0) +cv2.ocl.setUseOpenCL(False) + +import torch +from torchvision.transforms import ColorJitter +import torchvision.transforms.functional as FF + +class StereoAugmentor(object): + + def __init__(self, crop_size, scale_prob=0.5, scale_xonly=True, lhth=800., lminscale=0.0, lmaxscale=1.0, hminscale=-0.2, hmaxscale=0.4, scale_interp_nearest=True, rightjitterprob=0.5, v_flip_prob=0.5, color_aug_asym=True, color_choice_prob=0.5): + self.crop_size = crop_size + self.scale_prob = scale_prob + self.scale_xonly = scale_xonly + self.lhth = lhth + self.lminscale = lminscale + self.lmaxscale = lmaxscale + self.hminscale = hminscale + self.hmaxscale = hmaxscale + self.scale_interp_nearest = scale_interp_nearest + self.rightjitterprob = rightjitterprob + self.v_flip_prob = v_flip_prob + self.color_aug_asym = color_aug_asym + self.color_choice_prob = color_choice_prob + + def _random_scale(self, img1, img2, disp): + ch,cw = self.crop_size + h,w = img1.shape[:2] + if self.scale_prob>0. and np.random.rand()1.: + scale_x = clip_scale + scale_y = scale_x if not self.scale_xonly else 1.0 + img1 = cv2.resize(img1, None, fx=scale_x, fy=scale_y, interpolation=cv2.INTER_LINEAR) + img2 = cv2.resize(img2, None, fx=scale_x, fy=scale_y, interpolation=cv2.INTER_LINEAR) + disp = cv2.resize(disp, None, fx=scale_x, fy=scale_y, interpolation=cv2.INTER_LINEAR if not self.scale_interp_nearest else cv2.INTER_NEAREST) * scale_x + return img1, img2, disp + + def _random_crop(self, img1, img2, disp): + h,w = img1.shape[:2] + ch,cw = self.crop_size + assert ch<=h and cw<=w, (img1.shape, h,w,ch,cw) + offset_x = np.random.randint(w - cw + 1) + offset_y = np.random.randint(h - ch + 1) + img1 = img1[offset_y:offset_y+ch,offset_x:offset_x+cw] + img2 = img2[offset_y:offset_y+ch,offset_x:offset_x+cw] + disp = disp[offset_y:offset_y+ch,offset_x:offset_x+cw] + return img1, img2, disp + + def _random_vflip(self, img1, img2, disp): + # vertical flip + if self.v_flip_prob>0 and np.random.rand() < self.v_flip_prob: + img1 = np.copy(np.flipud(img1)) + img2 = np.copy(np.flipud(img2)) + disp = np.copy(np.flipud(disp)) + return img1, img2, disp + + def _random_rotate_shift_right(self, img2): + if self.rightjitterprob>0. and np.random.rand() 0) & (xx < wd1) & (yy > 0) & (yy < ht1) + xx = xx[v] + yy = yy[v] + flow1 = flow1[v] + + flow = np.inf * np.ones([ht1, wd1, 2], dtype=np.float32) # invalid value every where, before we fill it with the correct ones + flow[yy, xx] = flow1 + return flow + + def spatial_transform(self, img1, img2, flow, dname): + + if np.random.rand() < self.spatial_aug_prob: + # randomly sample scale + ht, wd = img1.shape[:2] + clip_min_scale = np.maximum( + (self.crop_size[0] + 8) / float(ht), + (self.crop_size[1] + 8) / float(wd)) + min_scale, max_scale = self.min_scale, self.max_scale + scale = 2 ** np.random.uniform(self.min_scale, self.max_scale) + scale_x = scale + scale_y = scale + if np.random.rand() < self.stretch_prob: + scale_x *= 2 ** np.random.uniform(-self.max_stretch, self.max_stretch) + scale_y *= 2 ** np.random.uniform(-self.max_stretch, self.max_stretch) + scale_x = np.clip(scale_x, clip_min_scale, None) + scale_y = np.clip(scale_y, clip_min_scale, None) + # rescale the images + img1 = cv2.resize(img1, None, fx=scale_x, fy=scale_y, interpolation=cv2.INTER_LINEAR) + img2 = cv2.resize(img2, None, fx=scale_x, fy=scale_y, interpolation=cv2.INTER_LINEAR) + flow = self._resize_flow(flow, scale_x, scale_y, factor=2.0 if dname=='Spring' else 1.0) + elif dname=="Spring": + flow = self._resize_flow(flow, 1.0, 1.0, factor=2.0) + + if self.h_flip_prob>0. and np.random.rand() < self.h_flip_prob: # h-flip + img1 = img1[:, ::-1] + img2 = img2[:, ::-1] + flow = flow[:, ::-1] * [-1.0, 1.0] + + if self.v_flip_prob>0. and np.random.rand() < self.v_flip_prob: # v-flip + img1 = img1[::-1, :] + img2 = img2[::-1, :] + flow = flow[::-1, :] * [1.0, -1.0] + + # In case no cropping + if img1.shape[0] - self.crop_size[0] > 0: + y0 = np.random.randint(0, img1.shape[0] - self.crop_size[0]) + else: + y0 = 0 + if img1.shape[1] - self.crop_size[1] > 0: + x0 = np.random.randint(0, img1.shape[1] - self.crop_size[1]) + else: + x0 = 0 + + img1 = img1[y0:y0 + self.crop_size[0], x0:x0 + self.crop_size[1]] + img2 = img2[y0:y0 + self.crop_size[0], x0:x0 + self.crop_size[1]] + flow = flow[y0:y0 + self.crop_size[0], x0:x0 + self.crop_size[1]] + + return img1, img2, flow + + def __call__(self, img1, img2, flow, dname): + img1, img2, flow = self.spatial_transform(img1, img2, flow, dname) + img1, img2 = self.color_transform(img1, img2) + img1 = np.ascontiguousarray(img1) + img2 = np.ascontiguousarray(img2) + flow = np.ascontiguousarray(flow) + return img1, img2, flow \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/stereoflow/criterion.py b/submodules/mast3r/dust3r/croco/stereoflow/criterion.py new file mode 100644 index 0000000000000000000000000000000000000000..57792ebeeee34827b317a4d32b7445837bb33f17 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/stereoflow/criterion.py @@ -0,0 +1,251 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# Losses, metrics per batch, metrics per dataset +# -------------------------------------------------------- + +import torch +from torch import nn +import torch.nn.functional as F + +def _get_gtnorm(gt): + if gt.size(1)==1: # stereo + return gt + # flow + return torch.sqrt(torch.sum(gt**2, dim=1, keepdims=True)) # Bx1xHxW + +############ losses without confidence + +class L1Loss(nn.Module): + + def __init__(self, max_gtnorm=None): + super().__init__() + self.max_gtnorm = max_gtnorm + self.with_conf = False + + def _error(self, gt, predictions): + return torch.abs(gt-predictions) + + def forward(self, predictions, gt, inspect=False): + mask = torch.isfinite(gt) + if self.max_gtnorm is not None: + mask *= _get_gtnorm(gt).expand(-1,gt.size(1),-1,-1) which is a constant + + +class LaplacianLossBounded(nn.Module): # used for CroCo-Flow ; in the equation of the paper, we have a=1/b + def __init__(self, max_gtnorm=10000., a=0.25, b=4.): + super().__init__() + self.max_gtnorm = max_gtnorm + self.with_conf = True + self.a, self.b = a, b + + def forward(self, predictions, gt, conf): + mask = torch.isfinite(gt) + mask = mask[:,0,:,:] + if self.max_gtnorm is not None: mask *= _get_gtnorm(gt)[:,0,:,:] which is a constant + +class LaplacianLossBounded2(nn.Module): # used for CroCo-Stereo (except for ETH3D) ; in the equation of the paper, we have a=b + def __init__(self, max_gtnorm=None, a=3.0, b=3.0): + super().__init__() + self.max_gtnorm = max_gtnorm + self.with_conf = True + self.a, self.b = a, b + + def forward(self, predictions, gt, conf): + mask = torch.isfinite(gt) + mask = mask[:,0,:,:] + if self.max_gtnorm is not None: mask *= _get_gtnorm(gt)[:,0,:,:] which is a constant + +############## metrics per batch + +class StereoMetrics(nn.Module): + + def __init__(self, do_quantile=False): + super().__init__() + self.bad_ths = [0.5,1,2,3] + self.do_quantile = do_quantile + + def forward(self, predictions, gt): + B = predictions.size(0) + metrics = {} + gtcopy = gt.clone() + mask = torch.isfinite(gtcopy) + gtcopy[~mask] = 999999.0 # we make a copy and put a non-infinite value, such that it does not become nan once multiplied by the mask value 0 + Npx = mask.view(B,-1).sum(dim=1) + L1error = (torch.abs(gtcopy-predictions)*mask).view(B,-1) + L2error = (torch.square(gtcopy-predictions)*mask).view(B,-1) + # avgerr + metrics['avgerr'] = torch.mean(L1error.sum(dim=1)/Npx ) + # rmse + metrics['rmse'] = torch.sqrt(L2error.sum(dim=1)/Npx).mean(dim=0) + # err > t for t in [0.5,1,2,3] + for ths in self.bad_ths: + metrics['bad@{:.1f}'.format(ths)] = (((L1error>ths)* mask.view(B,-1)).sum(dim=1)/Npx).mean(dim=0) * 100 + return metrics + +class FlowMetrics(nn.Module): + def __init__(self): + super().__init__() + self.bad_ths = [1,3,5] + + def forward(self, predictions, gt): + B = predictions.size(0) + metrics = {} + mask = torch.isfinite(gt[:,0,:,:]) # both x and y would be infinite + Npx = mask.view(B,-1).sum(dim=1) + gtcopy = gt.clone() # to compute L1/L2 error, we need to have non-infinite value, the error computed at this locations will be ignored + gtcopy[:,0,:,:][~mask] = 999999.0 + gtcopy[:,1,:,:][~mask] = 999999.0 + L1error = (torch.abs(gtcopy-predictions).sum(dim=1)*mask).view(B,-1) + L2error = (torch.sqrt(torch.sum(torch.square(gtcopy-predictions),dim=1))*mask).view(B,-1) + metrics['L1err'] = torch.mean(L1error.sum(dim=1)/Npx ) + metrics['EPE'] = torch.mean(L2error.sum(dim=1)/Npx ) + for ths in self.bad_ths: + metrics['bad@{:.1f}'.format(ths)] = (((L2error>ths)* mask.view(B,-1)).sum(dim=1)/Npx).mean(dim=0) * 100 + return metrics + +############## metrics per dataset +## we update the average and maintain the number of pixels while adding data batch per batch +## at the beggining, call reset() +## after each batch, call add_batch(...) +## at the end: call get_results() + +class StereoDatasetMetrics(nn.Module): + + def __init__(self): + super().__init__() + self.bad_ths = [0.5,1,2,3] + + def reset(self): + self.agg_N = 0 # number of pixels so far + self.agg_L1err = torch.tensor(0.0) # L1 error so far + self.agg_Nbad = [0 for _ in self.bad_ths] # counter of bad pixels + self._metrics = None + + def add_batch(self, predictions, gt): + assert predictions.size(1)==1, predictions.size() + assert gt.size(1)==1, gt.size() + if gt.size(2)==predictions.size(2)*2 and gt.size(3)==predictions.size(3)*2: # special case for Spring ... + L1err = torch.minimum( torch.minimum( torch.minimum( + torch.sum(torch.abs(gt[:,:,0::2,0::2]-predictions),dim=1), + torch.sum(torch.abs(gt[:,:,1::2,0::2]-predictions),dim=1)), + torch.sum(torch.abs(gt[:,:,0::2,1::2]-predictions),dim=1)), + torch.sum(torch.abs(gt[:,:,1::2,1::2]-predictions),dim=1)) + valid = torch.isfinite(L1err) + else: + valid = torch.isfinite(gt[:,0,:,:]) # both x and y would be infinite + L1err = torch.sum(torch.abs(gt-predictions),dim=1) + N = valid.sum() + Nnew = self.agg_N + N + self.agg_L1err = float(self.agg_N)/Nnew * self.agg_L1err + L1err[valid].mean().cpu() * float(N)/Nnew + self.agg_N = Nnew + for i,th in enumerate(self.bad_ths): + self.agg_Nbad[i] += (L1err[valid]>th).sum().cpu() + + def _compute_metrics(self): + if self._metrics is not None: return + out = {} + out['L1err'] = self.agg_L1err.item() + for i,th in enumerate(self.bad_ths): + out['bad@{:.1f}'.format(th)] = (float(self.agg_Nbad[i]) / self.agg_N).item() * 100.0 + self._metrics = out + + def get_results(self): + self._compute_metrics() # to avoid recompute them multiple times + return self._metrics + +class FlowDatasetMetrics(nn.Module): + + def __init__(self): + super().__init__() + self.bad_ths = [0.5,1,3,5] + self.speed_ths = [(0,10),(10,40),(40,torch.inf)] + + def reset(self): + self.agg_N = 0 # number of pixels so far + self.agg_L1err = torch.tensor(0.0) # L1 error so far + self.agg_L2err = torch.tensor(0.0) # L2 (=EPE) error so far + self.agg_Nbad = [0 for _ in self.bad_ths] # counter of bad pixels + self.agg_EPEspeed = [torch.tensor(0.0) for _ in self.speed_ths] # EPE per speed bin so far + self.agg_Nspeed = [0 for _ in self.speed_ths] # N pixels per speed bin so far + self._metrics = None + self.pairname_results = {} + + def add_batch(self, predictions, gt): + assert predictions.size(1)==2, predictions.size() + assert gt.size(1)==2, gt.size() + if gt.size(2)==predictions.size(2)*2 and gt.size(3)==predictions.size(3)*2: # special case for Spring ... + L1err = torch.minimum( torch.minimum( torch.minimum( + torch.sum(torch.abs(gt[:,:,0::2,0::2]-predictions),dim=1), + torch.sum(torch.abs(gt[:,:,1::2,0::2]-predictions),dim=1)), + torch.sum(torch.abs(gt[:,:,0::2,1::2]-predictions),dim=1)), + torch.sum(torch.abs(gt[:,:,1::2,1::2]-predictions),dim=1)) + L2err = torch.minimum( torch.minimum( torch.minimum( + torch.sqrt(torch.sum(torch.square(gt[:,:,0::2,0::2]-predictions),dim=1)), + torch.sqrt(torch.sum(torch.square(gt[:,:,1::2,0::2]-predictions),dim=1))), + torch.sqrt(torch.sum(torch.square(gt[:,:,0::2,1::2]-predictions),dim=1))), + torch.sqrt(torch.sum(torch.square(gt[:,:,1::2,1::2]-predictions),dim=1))) + valid = torch.isfinite(L1err) + gtspeed = (torch.sqrt(torch.sum(torch.square(gt[:,:,0::2,0::2]),dim=1)) + torch.sqrt(torch.sum(torch.square(gt[:,:,0::2,1::2]),dim=1)) +\ + torch.sqrt(torch.sum(torch.square(gt[:,:,1::2,0::2]),dim=1)) + torch.sqrt(torch.sum(torch.square(gt[:,:,1::2,1::2]),dim=1)) ) / 4.0 # let's just average them + else: + valid = torch.isfinite(gt[:,0,:,:]) # both x and y would be infinite + L1err = torch.sum(torch.abs(gt-predictions),dim=1) + L2err = torch.sqrt(torch.sum(torch.square(gt-predictions),dim=1)) + gtspeed = torch.sqrt(torch.sum(torch.square(gt),dim=1)) + N = valid.sum() + Nnew = self.agg_N + N + self.agg_L1err = float(self.agg_N)/Nnew * self.agg_L1err + L1err[valid].mean().cpu() * float(N)/Nnew + self.agg_L2err = float(self.agg_N)/Nnew * self.agg_L2err + L2err[valid].mean().cpu() * float(N)/Nnew + self.agg_N = Nnew + for i,th in enumerate(self.bad_ths): + self.agg_Nbad[i] += (L2err[valid]>th).sum().cpu() + for i,(th1,th2) in enumerate(self.speed_ths): + vv = (gtspeed[valid]>=th1) * (gtspeed[valid] don't use batch_size>1 at test time) + self._prepare_data() + self._load_or_build_cache() + + def prepare_data(self): + """ + to be defined for each dataset + """ + raise NotImplementedError + + def __len__(self): + return len(self.pairnames) # each pairname is typically of the form (str, int1, int2) + + def __getitem__(self, index): + pairname = self.pairnames[index] + + # get filenames + img1name = self.pairname_to_img1name(pairname) + img2name = self.pairname_to_img2name(pairname) + flowname = self.pairname_to_flowname(pairname) if self.pairname_to_flowname is not None else None + + # load images and disparities + img1 = _read_img(img1name) + img2 = _read_img(img2name) + flow = self.load_flow(flowname) if flowname is not None else None + + # apply augmentations + if self.augmentor is not None: + img1, img2, flow = self.augmentor(img1, img2, flow, self.name) + + if self.totensor: + img1 = img_to_tensor(img1) + img2 = img_to_tensor(img2) + if flow is not None: + flow = flow_to_tensor(flow) + else: + flow = torch.tensor([]) # to allow dataloader batching with default collate_gn + pairname = str(pairname) # transform potential tuple to str to be able to batch it + + return img1, img2, flow, pairname + + def __rmul__(self, v): + self.rmul *= v + self.pairnames = v * self.pairnames + return self + + def __str__(self): + return f'{self.__class__.__name__}_{self.split}' + + def __repr__(self): + s = f'{self.__class__.__name__}(split={self.split}, augmentor={self.augmentor_str}, crop_size={str(self.crop_size)}, totensor={self.totensor})' + if self.rmul==1: + s+=f'\n\tnum pairs: {len(self.pairnames)}' + else: + s+=f'\n\tnum pairs: {len(self.pairnames)} ({len(self.pairnames)//self.rmul}x{self.rmul})' + return s + + def _set_root(self): + self.root = dataset_to_root[self.name] + assert os.path.isdir(self.root), f"could not find root directory for dataset {self.name}: {self.root}" + + def _load_or_build_cache(self): + cache_file = osp.join(cache_dir, self.name+'.pkl') + if osp.isfile(cache_file): + with open(cache_file, 'rb') as fid: + self.pairnames = pickle.load(fid)[self.split] + else: + tosave = self._build_cache() + os.makedirs(cache_dir, exist_ok=True) + with open(cache_file, 'wb') as fid: + pickle.dump(tosave, fid) + self.pairnames = tosave[self.split] + +class TartanAirDataset(FlowDataset): + + def _prepare_data(self): + self.name = "TartanAir" + self._set_root() + assert self.split in ['train'] + self.pairname_to_img1name = lambda pairname: osp.join(self.root, pairname[0], 'image_left/{:06d}_left.png'.format(pairname[1])) + self.pairname_to_img2name = lambda pairname: osp.join(self.root, pairname[0], 'image_left/{:06d}_left.png'.format(pairname[2])) + self.pairname_to_flowname = lambda pairname: osp.join(self.root, pairname[0], 'flow/{:06d}_{:06d}_flow.npy'.format(pairname[1],pairname[2])) + self.pairname_to_str = lambda pairname: os.path.join(pairname[0][pairname[0].find('/')+1:], '{:06d}_{:06d}'.format(pairname[1], pairname[2])) + self.load_flow = _read_numpy_flow + + def _build_cache(self): + seqs = sorted(os.listdir(self.root)) + pairs = [(osp.join(s,s,difficulty,Pxxx),int(a[:6]),int(a[:6])+1) for s in seqs for difficulty in ['Easy','Hard'] for Pxxx in sorted(os.listdir(osp.join(self.root,s,s,difficulty))) for a in sorted(os.listdir(osp.join(self.root,s,s,difficulty,Pxxx,'image_left/')))[:-1]] + assert len(pairs)==306268, "incorrect parsing of pairs in TartanAir" + tosave = {'train': pairs} + return tosave + +class FlyingChairsDataset(FlowDataset): + + def _prepare_data(self): + self.name = "FlyingChairs" + self._set_root() + assert self.split in ['train','val'] + self.pairname_to_img1name = lambda pairname: osp.join(self.root, 'data', pairname+'_img1.ppm') + self.pairname_to_img2name = lambda pairname: osp.join(self.root, 'data', pairname+'_img2.ppm') + self.pairname_to_flowname = lambda pairname: osp.join(self.root, 'data', pairname+'_flow.flo') + self.pairname_to_str = lambda pairname: pairname + self.load_flow = _read_flo_file + + def _build_cache(self): + split_file = osp.join(self.root, 'chairs_split.txt') + split_list = np.loadtxt(split_file, dtype=np.int32) + trainpairs = ['{:05d}'.format(i) for i in np.where(split_list==1)[0]+1] + valpairs = ['{:05d}'.format(i) for i in np.where(split_list==2)[0]+1] + assert len(trainpairs)==22232 and len(valpairs)==640, "incorrect parsing of pairs in MPI-Sintel" + tosave = {'train': trainpairs, 'val': valpairs} + return tosave + +class FlyingThingsDataset(FlowDataset): + + def _prepare_data(self): + self.name = "FlyingThings" + self._set_root() + assert self.split in [f'{set_}_{pass_}pass{camstr}' for set_ in ['train','test','test1024'] for camstr in ['','_rightcam'] for pass_ in ['clean','final','all']] + self.pairname_to_img1name = lambda pairname: osp.join(self.root, f'frames_{pairname[3]}pass', pairname[0].replace('into_future','').replace('into_past',''), '{:04d}.png'.format(pairname[1])) + self.pairname_to_img2name = lambda pairname: osp.join(self.root, f'frames_{pairname[3]}pass', pairname[0].replace('into_future','').replace('into_past',''), '{:04d}.png'.format(pairname[2])) + self.pairname_to_flowname = lambda pairname: osp.join(self.root, 'optical_flow', pairname[0], 'OpticalFlowInto{f:s}_{i:04d}_{c:s}.pfm'.format(f='Future' if 'future' in pairname[0] else 'Past', i=pairname[1], c='L' if 'left' in pairname[0] else 'R' )) + self.pairname_to_str = lambda pairname: os.path.join(pairname[3]+'pass', pairname[0], 'Into{f:s}_{i:04d}_{c:s}'.format(f='Future' if 'future' in pairname[0] else 'Past', i=pairname[1], c='L' if 'left' in pairname[0] else 'R' )) + self.load_flow = _read_pfm_flow + + def _build_cache(self): + tosave = {} + # train and test splits for the different passes + for set_ in ['train', 'test']: + sroot = osp.join(self.root, 'optical_flow', set_.upper()) + fname_to_i = lambda f: int(f[len('OpticalFlowIntoFuture_'):-len('_L.pfm')]) + pp = [(osp.join(set_.upper(), d, s, 'into_future/left'),fname_to_i(fname)) for d in sorted(os.listdir(sroot)) for s in sorted(os.listdir(osp.join(sroot,d))) for fname in sorted(os.listdir(osp.join(sroot,d, s, 'into_future/left')))[:-1]] + pairs = [(a,i,i+1) for a,i in pp] + pairs += [(a.replace('into_future','into_past'),i+1,i) for a,i in pp] + assert len(pairs)=={'train': 40302, 'test': 7866}[set_], "incorrect parsing of pairs Flying Things" + for cam in ['left','right']: + camstr = '' if cam=='left' else f'_{cam}cam' + for pass_ in ['final', 'clean']: + tosave[f'{set_}_{pass_}pass{camstr}'] = [(a.replace('left',cam),i,j,pass_) for a,i,j in pairs] + tosave[f'{set_}_allpass{camstr}'] = tosave[f'{set_}_cleanpass{camstr}'] + tosave[f'{set_}_finalpass{camstr}'] + # test1024: this is the same split as unimatch 'validation' split + # see https://github.com/autonomousvision/unimatch/blob/master/dataloader/flow/datasets.py#L229 + test1024_nsamples = 1024 + alltest_nsamples = len(tosave['test_cleanpass']) # 7866 + stride = alltest_nsamples // test1024_nsamples + remove = alltest_nsamples % test1024_nsamples + for cam in ['left','right']: + camstr = '' if cam=='left' else f'_{cam}cam' + for pass_ in ['final','clean']: + tosave[f'test1024_{pass_}pass{camstr}'] = sorted(tosave[f'test_{pass_}pass{camstr}'])[:-remove][::stride] # warning, it was not sorted before + assert len(tosave['test1024_cleanpass'])==1024, "incorrect parsing of pairs in Flying Things" + tosave[f'test1024_allpass{camstr}'] = tosave[f'test1024_cleanpass{camstr}'] + tosave[f'test1024_finalpass{camstr}'] + return tosave + + +class MPISintelDataset(FlowDataset): + + def _prepare_data(self): + self.name = "MPISintel" + self._set_root() + assert self.split in [s+'_'+p for s in ['train','test','subval','subtrain'] for p in ['cleanpass','finalpass','allpass']] + self.pairname_to_img1name = lambda pairname: osp.join(self.root, pairname[0], 'frame_{:04d}.png'.format(pairname[1])) + self.pairname_to_img2name = lambda pairname: osp.join(self.root, pairname[0], 'frame_{:04d}.png'.format(pairname[1]+1)) + self.pairname_to_flowname = lambda pairname: None if pairname[0].startswith('test/') else osp.join(self.root, pairname[0].replace('/clean/','/flow/').replace('/final/','/flow/'), 'frame_{:04d}.flo'.format(pairname[1])) + self.pairname_to_str = lambda pairname: osp.join(pairname[0], 'frame_{:04d}'.format(pairname[1])) + self.load_flow = _read_flo_file + + def _build_cache(self): + trainseqs = sorted(os.listdir(self.root+'training/clean')) + trainpairs = [ (osp.join('training/clean', s),i) for s in trainseqs for i in range(1, len(os.listdir(self.root+'training/clean/'+s)))] + subvalseqs = ['temple_2','temple_3'] + subtrainseqs = [s for s in trainseqs if s not in subvalseqs] + subvalpairs = [ (p,i) for p,i in trainpairs if any(s in p for s in subvalseqs)] + subtrainpairs = [ (p,i) for p,i in trainpairs if any(s in p for s in subtrainseqs)] + testseqs = sorted(os.listdir(self.root+'test/clean')) + testpairs = [ (osp.join('test/clean', s),i) for s in testseqs for i in range(1, len(os.listdir(self.root+'test/clean/'+s)))] + assert len(trainpairs)==1041 and len(testpairs)==552 and len(subvalpairs)==98 and len(subtrainpairs)==943, "incorrect parsing of pairs in MPI-Sintel" + tosave = {} + tosave['train_cleanpass'] = trainpairs + tosave['test_cleanpass'] = testpairs + tosave['subval_cleanpass'] = subvalpairs + tosave['subtrain_cleanpass'] = subtrainpairs + for t in ['train','test','subval','subtrain']: + tosave[t+'_finalpass'] = [(p.replace('/clean/','/final/'),i) for p,i in tosave[t+'_cleanpass']] + tosave[t+'_allpass'] = tosave[t+'_cleanpass'] + tosave[t+'_finalpass'] + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, _time): + assert prediction.shape[2]==2 + outfile = os.path.join(outdir, 'submission', self.pairname_to_str(pairname)+'.flo') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + writeFlowFile(prediction, outfile) + + def finalize_submission(self, outdir): + assert self.split == 'test_allpass' + bundle_exe = "/nfs/data/ffs-3d/datasets/StereoFlow/MPI-Sintel/bundler/linux-x64/bundler" # eg + if os.path.isfile(bundle_exe): + cmd = f'{bundle_exe} "{outdir}/submission/test/clean/" "{outdir}/submission/test/final" "{outdir}/submission/bundled.lzma"' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at: "{outdir}/submission/bundled.lzma"') + else: + print('Could not find bundler executable for submission.') + print('Please download it and run:') + print(f' "{outdir}/submission/test/clean/" "{outdir}/submission/test/final" "{outdir}/submission/bundled.lzma"') + +class SpringDataset(FlowDataset): + + def _prepare_data(self): + self.name = "Spring" + self._set_root() + assert self.split in ['train','test','subtrain','subval'] + self.pairname_to_img1name = lambda pairname: osp.join(self.root, pairname[0], pairname[1], 'frame_'+pairname[3], 'frame_{:s}_{:04d}.png'.format(pairname[3], pairname[4])) + self.pairname_to_img2name = lambda pairname: osp.join(self.root, pairname[0], pairname[1], 'frame_'+pairname[3], 'frame_{:s}_{:04d}.png'.format(pairname[3], pairname[4]+(1 if pairname[2]=='FW' else -1))) + self.pairname_to_flowname = lambda pairname: None if pairname[0]=='test' else osp.join(self.root, pairname[0], pairname[1], f'flow_{pairname[2]}_{pairname[3]}', f'flow_{pairname[2]}_{pairname[3]}_{pairname[4]:04d}.flo5') + self.pairname_to_str = lambda pairname: osp.join(pairname[0], pairname[1], f'flow_{pairname[2]}_{pairname[3]}', f'flow_{pairname[2]}_{pairname[3]}_{pairname[4]:04d}') + self.load_flow = _read_hdf5_flow + + def _build_cache(self): + # train + trainseqs = sorted(os.listdir( osp.join(self.root,'train'))) + trainpairs = [] + for leftright in ['left','right']: + for fwbw in ['FW','BW']: + trainpairs += [('train',s,fwbw,leftright,int(f[len(f'flow_{fwbw}_{leftright}_'):-len('.flo5')])) for s in trainseqs for f in sorted(os.listdir(osp.join(self.root,'train',s,f'flow_{fwbw}_{leftright}')))] + # test + testseqs = sorted(os.listdir( osp.join(self.root,'test'))) + testpairs = [] + for leftright in ['left','right']: + testpairs += [('test',s,'FW',leftright,int(f[len(f'frame_{leftright}_'):-len('.png')])) for s in testseqs for f in sorted(os.listdir(osp.join(self.root,'test',s,f'frame_{leftright}')))[:-1]] + testpairs += [('test',s,'BW',leftright,int(f[len(f'frame_{leftright}_'):-len('.png')])+1) for s in testseqs for f in sorted(os.listdir(osp.join(self.root,'test',s,f'frame_{leftright}')))[:-1]] + # subtrain / subval + subtrainpairs = [p for p in trainpairs if p[1]!='0041'] + subvalpairs = [p for p in trainpairs if p[1]=='0041'] + assert len(trainpairs)==19852 and len(testpairs)==3960 and len(subtrainpairs)==19472 and len(subvalpairs)==380, "incorrect parsing of pairs in Spring" + tosave = {'train': trainpairs, 'test': testpairs, 'subtrain': subtrainpairs, 'subval': subvalpairs} + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==3 + assert prediction.shape[2]==2 + assert prediction.dtype==np.float32 + outfile = osp.join(outdir, pairname[0], pairname[1], f'flow_{pairname[2]}_{pairname[3]}', f'flow_{pairname[2]}_{pairname[3]}_{pairname[4]:04d}.flo5') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + writeFlo5File(prediction, outfile) + + def finalize_submission(self, outdir): + assert self.split=='test' + exe = "{self.root}/flow_subsampling" + if os.path.isfile(exe): + cmd = f'cd "{outdir}/test"; {exe} .' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at {outdir}/test/flow_submission.hdf5') + else: + print('Could not find flow_subsampling executable for submission.') + print('Please download it and run:') + print(f'cd "{outdir}/test"; .') + + +class Kitti12Dataset(FlowDataset): + + def _prepare_data(self): + self.name = "Kitti12" + self._set_root() + assert self.split in ['train','test'] + self.pairname_to_img1name = lambda pairname: osp.join(self.root, pairname+'_10.png') + self.pairname_to_img2name = lambda pairname: osp.join(self.root, pairname+'_11.png') + self.pairname_to_flowname = None if self.split=='test' else lambda pairname: osp.join(self.root, pairname.replace('/colored_0/','/flow_occ/')+'_10.png') + self.pairname_to_str = lambda pairname: pairname.replace('/colored_0/','/') + self.load_flow = _read_kitti_flow + + def _build_cache(self): + trainseqs = ["training/colored_0/%06d"%(i) for i in range(194)] + testseqs = ["testing/colored_0/%06d"%(i) for i in range(195)] + assert len(trainseqs)==194 and len(testseqs)==195, "incorrect parsing of pairs in Kitti12" + tosave = {'train': trainseqs, 'test': testseqs} + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==3 + assert prediction.shape[2]==2 + outfile = os.path.join(outdir, pairname.split('/')[-1]+'_10.png') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + writeFlowKitti(outfile, prediction) + + def finalize_submission(self, outdir): + assert self.split=='test' + cmd = f'cd {outdir}/; zip -r "kitti12_flow_results.zip" .' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at {outdir}/kitti12_flow_results.zip') + + +class Kitti15Dataset(FlowDataset): + + def _prepare_data(self): + self.name = "Kitti15" + self._set_root() + assert self.split in ['train','subtrain','subval','test'] + self.pairname_to_img1name = lambda pairname: osp.join(self.root, pairname+'_10.png') + self.pairname_to_img2name = lambda pairname: osp.join(self.root, pairname+'_11.png') + self.pairname_to_flowname = None if self.split=='test' else lambda pairname: osp.join(self.root, pairname.replace('/image_2/','/flow_occ/')+'_10.png') + self.pairname_to_str = lambda pairname: pairname.replace('/image_2/','/') + self.load_flow = _read_kitti_flow + + def _build_cache(self): + trainseqs = ["training/image_2/%06d"%(i) for i in range(200)] + subtrainseqs = trainseqs[:-10] + subvalseqs = trainseqs[-10:] + testseqs = ["testing/image_2/%06d"%(i) for i in range(200)] + assert len(trainseqs)==200 and len(subtrainseqs)==190 and len(subvalseqs)==10 and len(testseqs)==200, "incorrect parsing of pairs in Kitti15" + tosave = {'train': trainseqs, 'subtrain': subtrainseqs, 'subval': subvalseqs, 'test': testseqs} + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==3 + assert prediction.shape[2]==2 + outfile = os.path.join(outdir, 'flow', pairname.split('/')[-1]+'_10.png') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + writeFlowKitti(outfile, prediction) + + def finalize_submission(self, outdir): + assert self.split=='test' + cmd = f'cd {outdir}/; zip -r "kitti15_flow_results.zip" flow' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at {outdir}/kitti15_flow_results.zip') + + +import cv2 +def _read_numpy_flow(filename): + return np.load(filename) + +def _read_pfm_flow(filename): + f, _ = _read_pfm(filename) + assert np.all(f[:,:,2]==0.0) + return np.ascontiguousarray(f[:,:,:2]) + +TAG_FLOAT = 202021.25 # tag to check the sanity of the file +TAG_STRING = 'PIEH' # string containing the tag +MIN_WIDTH = 1 +MAX_WIDTH = 99999 +MIN_HEIGHT = 1 +MAX_HEIGHT = 99999 +def readFlowFile(filename): + """ + readFlowFile() reads a flow file into a 2-band np.array. + if does not exist, an IOError is raised. + if does not finish by '.flo' or the tag, the width, the height or the file's size is illegal, an Expcetion is raised. + ---- PARAMETERS ---- + filename: string containg the name of the file to read a flow + ---- OUTPUTS ---- + a np.array of dimension (height x width x 2) containing the flow of type 'float32' + """ + + # check filename + if not filename.endswith(".flo"): + raise Exception("readFlowFile({:s}): filename must finish with '.flo'".format(filename)) + + # open the file and read it + with open(filename,'rb') as f: + # check tag + tag = struct.unpack('f',f.read(4))[0] + if tag != TAG_FLOAT: + raise Exception("flow_utils.readFlowFile({:s}): wrong tag".format(filename)) + # read dimension + w,h = struct.unpack('ii',f.read(8)) + if w < MIN_WIDTH or w > MAX_WIDTH: + raise Exception("flow_utils.readFlowFile({:s}: illegal width {:d}".format(filename,w)) + if h < MIN_HEIGHT or h > MAX_HEIGHT: + raise Exception("flow_utils.readFlowFile({:s}: illegal height {:d}".format(filename,h)) + flow = np.fromfile(f,'float32') + if not flow.shape == (h*w*2,): + raise Exception("flow_utils.readFlowFile({:s}: illegal size of the file".format(filename)) + flow.shape = (h,w,2) + return flow + +def writeFlowFile(flow,filename): + """ + writeFlowFile(flow,) write flow to the file . + if does not exist, an IOError is raised. + if does not finish with '.flo' or the flow has not 2 bands, an Exception is raised. + ---- PARAMETERS ---- + flow: np.array of dimension (height x width x 2) containing the flow to write + filename: string containg the name of the file to write a flow + """ + + # check filename + if not filename.endswith(".flo"): + raise Exception("flow_utils.writeFlowFile(,{:s}): filename must finish with '.flo'".format(filename)) + + if not flow.shape[2:] == (2,): + raise Exception("flow_utils.writeFlowFile(,{:s}): must have 2 bands".format(filename)) + + + # open the file and write it + with open(filename,'wb') as f: + # write TAG + f.write( TAG_STRING.encode('utf-8') ) + # write dimension + f.write( struct.pack('ii',flow.shape[1],flow.shape[0]) ) + # write the flow + + flow.astype(np.float32).tofile(f) + +_read_flo_file = readFlowFile + +def _read_kitti_flow(filename): + flow = cv2.imread(filename, cv2.IMREAD_ANYDEPTH | cv2.IMREAD_COLOR) + flow = flow[:, :, ::-1].astype(np.float32) + valid = flow[:, :, 2]>0 + flow = flow[:, :, :2] + flow = (flow - 2 ** 15) / 64.0 + flow[~valid,0] = np.inf + flow[~valid,1] = np.inf + return flow +_read_hd1k_flow = _read_kitti_flow + + +def writeFlowKitti(filename, uv): + uv = 64.0 * uv + 2 ** 15 + valid = np.ones([uv.shape[0], uv.shape[1], 1]) + uv = np.concatenate([uv, valid], axis=-1).astype(np.uint16) + cv2.imwrite(filename, uv[..., ::-1]) + +def writeFlo5File(flow, filename): + with h5py.File(filename, "w") as f: + f.create_dataset("flow", data=flow, compression="gzip", compression_opts=5) + +def _read_hdf5_flow(filename): + flow = np.asarray(h5py.File(filename)['flow']) + flow[np.isnan(flow)] = np.inf # make invalid values as +inf + return flow.astype(np.float32) + +# flow visualization +RY = 15 +YG = 6 +GC = 4 +CB = 11 +BM = 13 +MR = 6 +UNKNOWN_THRESH = 1e9 + +def colorTest(): + """ + flow_utils.colorTest(): display an example of image showing the color encoding scheme + """ + import matplotlib.pylab as plt + truerange = 1 + h,w = 151,151 + trange = truerange*1.04 + s2 = round(h/2) + x,y = np.meshgrid(range(w),range(h)) + u = x*trange/s2-trange + v = y*trange/s2-trange + img = _computeColor(np.concatenate((u[:,:,np.newaxis],v[:,:,np.newaxis]),2)/trange/np.sqrt(2)) + plt.imshow(img) + plt.axis('off') + plt.axhline(round(h/2),color='k') + plt.axvline(round(w/2),color='k') + +def flowToColor(flow, maxflow=None, maxmaxflow=None, saturate=False): + """ + flow_utils.flowToColor(flow): return a color code flow field, normalized based on the maximum l2-norm of the flow + flow_utils.flowToColor(flow,maxflow): return a color code flow field, normalized by maxflow + ---- PARAMETERS ---- + flow: flow to display of shape (height x width x 2) + maxflow (default:None): if given, normalize the flow by its value, otherwise by the flow norm + maxmaxflow (default:None): if given, normalize the flow by the max of its value and the flow norm + ---- OUTPUT ---- + an np.array of shape (height x width x 3) of type uint8 containing a color code of the flow + """ + h,w,n = flow.shape + # check size of flow + assert n == 2, "flow_utils.flowToColor(flow): flow must have 2 bands" + # fix unknown flow + unknown_idx = np.max(np.abs(flow),2)>UNKNOWN_THRESH + flow[unknown_idx] = 0.0 + # compute max flow if needed + if maxflow is None: + maxflow = flowMaxNorm(flow) + if maxmaxflow is not None: + maxflow = min(maxmaxflow, maxflow) + # normalize flow + eps = np.spacing(1) # minimum positive float value to avoid division by 0 + # compute the flow + img = _computeColor(flow/(maxflow+eps), saturate=saturate) + # put black pixels in unknown location + img[ np.tile( unknown_idx[:,:,np.newaxis],[1,1,3]) ] = 0.0 + return img + +def flowMaxNorm(flow): + """ + flow_utils.flowMaxNorm(flow): return the maximum of the l2-norm of the given flow + ---- PARAMETERS ---- + flow: the flow + + ---- OUTPUT ---- + a float containing the maximum of the l2-norm of the flow + """ + return np.max( np.sqrt( np.sum( np.square( flow ) , 2) ) ) + +def _computeColor(flow, saturate=True): + """ + flow_utils._computeColor(flow): compute color codes for the flow field flow + + ---- PARAMETERS ---- + flow: np.array of dimension (height x width x 2) containing the flow to display + ---- OUTPUTS ---- + an np.array of dimension (height x width x 3) containing the color conversion of the flow + """ + # set nan to 0 + nanidx = np.isnan(flow[:,:,0]) + flow[nanidx] = 0.0 + + # colorwheel + ncols = RY + YG + GC + CB + BM + MR + nchans = 3 + colorwheel = np.zeros((ncols,nchans),'uint8') + col = 0; + #RY + colorwheel[:RY,0] = 255 + colorwheel[:RY,1] = [(255*i) // RY for i in range(RY)] + col += RY + # YG + colorwheel[col:col+YG,0] = [255 - (255*i) // YG for i in range(YG)] + colorwheel[col:col+YG,1] = 255 + col += YG + # GC + colorwheel[col:col+GC,1] = 255 + colorwheel[col:col+GC,2] = [(255*i) // GC for i in range(GC)] + col += GC + # CB + colorwheel[col:col+CB,1] = [255 - (255*i) // CB for i in range(CB)] + colorwheel[col:col+CB,2] = 255 + col += CB + # BM + colorwheel[col:col+BM,0] = [(255*i) // BM for i in range(BM)] + colorwheel[col:col+BM,2] = 255 + col += BM + # MR + colorwheel[col:col+MR,0] = 255 + colorwheel[col:col+MR,2] = [255 - (255*i) // MR for i in range(MR)] + + # compute utility variables + rad = np.sqrt( np.sum( np.square(flow) , 2) ) # magnitude + a = np.arctan2( -flow[:,:,1] , -flow[:,:,0]) / np.pi # angle + fk = (a+1)/2 * (ncols-1) # map [-1,1] to [0,ncols-1] + k0 = np.floor(fk).astype('int') + k1 = k0+1 + k1[k1==ncols] = 0 + f = fk-k0 + + if not saturate: + rad = np.minimum(rad,1) + + # compute the image + img = np.zeros( (flow.shape[0],flow.shape[1],nchans), 'uint8' ) + for i in range(nchans): + tmp = colorwheel[:,i].astype('float') + col0 = tmp[k0]/255 + col1 = tmp[k1]/255 + col = (1-f)*col0 + f*col1 + idx = (rad <= 1) + col[idx] = 1-rad[idx]*(1-col[idx]) # increase saturation with radius + col[~idx] *= 0.75 # out of range + img[:,:,i] = (255*col*(1-nanidx.astype('float'))).astype('uint8') + + return img + +# flow dataset getter + +def get_train_dataset_flow(dataset_str, augmentor=True, crop_size=None): + dataset_str = dataset_str.replace('(','Dataset(') + if augmentor: + dataset_str = dataset_str.replace(')',', augmentor=True)') + if crop_size is not None: + dataset_str = dataset_str.replace(')',', crop_size={:s})'.format(str(crop_size))) + return eval(dataset_str) + +def get_test_datasets_flow(dataset_str): + dataset_str = dataset_str.replace('(','Dataset(') + return [eval(s) for s in dataset_str.split('+')] \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/stereoflow/datasets_stereo.py b/submodules/mast3r/dust3r/croco/stereoflow/datasets_stereo.py new file mode 100644 index 0000000000000000000000000000000000000000..dbdf841a6650afa71ae5782702902c79eba31a5c --- /dev/null +++ b/submodules/mast3r/dust3r/croco/stereoflow/datasets_stereo.py @@ -0,0 +1,674 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# Dataset structure for stereo +# -------------------------------------------------------- + +import sys, os +import os.path as osp +import pickle +import numpy as np +from PIL import Image +import json +import h5py +from glob import glob +import cv2 + +import torch +from torch.utils import data + +from .augmentor import StereoAugmentor + + + +dataset_to_root = { + 'CREStereo': './data/stereoflow//crenet_stereo_trainset/stereo_trainset/crestereo/', + 'SceneFlow': './data/stereoflow//SceneFlow/', + 'ETH3DLowRes': './data/stereoflow/eth3d_lowres/', + 'Booster': './data/stereoflow/booster_gt/', + 'Middlebury2021': './data/stereoflow/middlebury/2021/data/', + 'Middlebury2014': './data/stereoflow/middlebury/2014/', + 'Middlebury2006': './data/stereoflow/middlebury/2006/', + 'Middlebury2005': './data/stereoflow/middlebury/2005/train/', + 'MiddleburyEval3': './data/stereoflow/middlebury/MiddEval3/', + 'Spring': './data/stereoflow/spring/', + 'Kitti15': './data/stereoflow/kitti-stereo-2015/', + 'Kitti12': './data/stereoflow/kitti-stereo-2012/', +} +cache_dir = "./data/stereoflow/datasets_stereo_cache/" + + +in1k_mean = torch.tensor([0.485, 0.456, 0.406]).view(3,1,1) +in1k_std = torch.tensor([0.229, 0.224, 0.225]).view(3,1,1) +def img_to_tensor(img): + img = torch.from_numpy(img).permute(2, 0, 1).float() / 255. + img = (img-in1k_mean)/in1k_std + return img +def disp_to_tensor(disp): + return torch.from_numpy(disp)[None,:,:] + +class StereoDataset(data.Dataset): + + def __init__(self, split, augmentor=False, crop_size=None, totensor=True): + self.split = split + if not augmentor: assert crop_size is None + if crop_size: assert augmentor + self.crop_size = crop_size + self.augmentor_str = augmentor + self.augmentor = StereoAugmentor(crop_size) if augmentor else None + self.totensor = totensor + self.rmul = 1 # keep track of rmul + self.has_constant_resolution = True # whether the dataset has constant resolution or not (=> don't use batch_size>1 at test time) + self._prepare_data() + self._load_or_build_cache() + + def prepare_data(self): + """ + to be defined for each dataset + """ + raise NotImplementedError + + def __len__(self): + return len(self.pairnames) + + def __getitem__(self, index): + pairname = self.pairnames[index] + + # get filenames + Limgname = self.pairname_to_Limgname(pairname) + Rimgname = self.pairname_to_Rimgname(pairname) + Ldispname = self.pairname_to_Ldispname(pairname) if self.pairname_to_Ldispname is not None else None + + # load images and disparities + Limg = _read_img(Limgname) + Rimg = _read_img(Rimgname) + disp = self.load_disparity(Ldispname) if Ldispname is not None else None + + # sanity check + if disp is not None: assert np.all(disp>0) or self.name=="Spring", (self.name, pairname, Ldispname) + + # apply augmentations + if self.augmentor is not None: + Limg, Rimg, disp = self.augmentor(Limg, Rimg, disp, self.name) + + if self.totensor: + Limg = img_to_tensor(Limg) + Rimg = img_to_tensor(Rimg) + if disp is None: + disp = torch.tensor([]) # to allow dataloader batching with default collate_gn + else: + disp = disp_to_tensor(disp) + + return Limg, Rimg, disp, str(pairname) + + def __rmul__(self, v): + self.rmul *= v + self.pairnames = v * self.pairnames + return self + + def __str__(self): + return f'{self.__class__.__name__}_{self.split}' + + def __repr__(self): + s = f'{self.__class__.__name__}(split={self.split}, augmentor={self.augmentor_str}, crop_size={str(self.crop_size)}, totensor={self.totensor})' + if self.rmul==1: + s+=f'\n\tnum pairs: {len(self.pairnames)}' + else: + s+=f'\n\tnum pairs: {len(self.pairnames)} ({len(self.pairnames)//self.rmul}x{self.rmul})' + return s + + def _set_root(self): + self.root = dataset_to_root[self.name] + assert os.path.isdir(self.root), f"could not find root directory for dataset {self.name}: {self.root}" + + def _load_or_build_cache(self): + cache_file = osp.join(cache_dir, self.name+'.pkl') + if osp.isfile(cache_file): + with open(cache_file, 'rb') as fid: + self.pairnames = pickle.load(fid)[self.split] + else: + tosave = self._build_cache() + os.makedirs(cache_dir, exist_ok=True) + with open(cache_file, 'wb') as fid: + pickle.dump(tosave, fid) + self.pairnames = tosave[self.split] + +class CREStereoDataset(StereoDataset): + + def _prepare_data(self): + self.name = 'CREStereo' + self._set_root() + assert self.split in ['train'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname+'_left.jpg') + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname+'_right.jpg') + self.pairname_to_Ldispname = lambda pairname: osp.join(self.root, pairname+'_left.disp.png') + self.pairname_to_str = lambda pairname: pairname + self.load_disparity = _read_crestereo_disp + + + def _build_cache(self): + allpairs = [s+'/'+f[:-len('_left.jpg')] for s in sorted(os.listdir(self.root)) for f in sorted(os.listdir(self.root+'/'+s)) if f.endswith('_left.jpg')] + assert len(allpairs)==200000, "incorrect parsing of pairs in CreStereo" + tosave = {'train': allpairs} + return tosave + +class SceneFlowDataset(StereoDataset): + + def _prepare_data(self): + self.name = "SceneFlow" + self._set_root() + assert self.split in ['train_finalpass','train_cleanpass','train_allpass','test_finalpass','test_cleanpass','test_allpass','test1of100_cleanpass','test1of100_finalpass'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname) + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname).replace('/left/','/right/') + self.pairname_to_Ldispname = lambda pairname: osp.join(self.root, pairname).replace('/frames_finalpass/','/disparity/').replace('/frames_cleanpass/','/disparity/')[:-4]+'.pfm' + self.pairname_to_str = lambda pairname: pairname[:-4] + self.load_disparity = _read_sceneflow_disp + + def _build_cache(self): + trainpairs = [] + # driving + pairs = sorted(glob(self.root+'Driving/frames_finalpass/*/*/*/left/*.png')) + pairs = list(map(lambda x: x[len(self.root):], pairs)) + assert len(pairs) == 4400, "incorrect parsing of pairs in SceneFlow" + trainpairs += pairs + # monkaa + pairs = sorted(glob(self.root+'Monkaa/frames_finalpass/*/left/*.png')) + pairs = list(map(lambda x: x[len(self.root):], pairs)) + assert len(pairs) == 8664, "incorrect parsing of pairs in SceneFlow" + trainpairs += pairs + # flyingthings + pairs = sorted(glob(self.root+'FlyingThings/frames_finalpass/TRAIN/*/*/left/*.png')) + pairs = list(map(lambda x: x[len(self.root):], pairs)) + assert len(pairs) == 22390, "incorrect parsing of pairs in SceneFlow" + trainpairs += pairs + assert len(trainpairs) == 35454, "incorrect parsing of pairs in SceneFlow" + testpairs = sorted(glob(self.root+'FlyingThings/frames_finalpass/TEST/*/*/left/*.png')) + testpairs = list(map(lambda x: x[len(self.root):], testpairs)) + assert len(testpairs) == 4370, "incorrect parsing of pairs in SceneFlow" + test1of100pairs = testpairs[::100] + assert len(test1of100pairs) == 44, "incorrect parsing of pairs in SceneFlow" + # all + tosave = {'train_finalpass': trainpairs, + 'train_cleanpass': list(map(lambda x: x.replace('frames_finalpass','frames_cleanpass'), trainpairs)), + 'test_finalpass': testpairs, + 'test_cleanpass': list(map(lambda x: x.replace('frames_finalpass','frames_cleanpass'), testpairs)), + 'test1of100_finalpass': test1of100pairs, + 'test1of100_cleanpass': list(map(lambda x: x.replace('frames_finalpass','frames_cleanpass'), test1of100pairs)), + } + tosave['train_allpass'] = tosave['train_finalpass']+tosave['train_cleanpass'] + tosave['test_allpass'] = tosave['test_finalpass']+tosave['test_cleanpass'] + return tosave + +class Md21Dataset(StereoDataset): + + def _prepare_data(self): + self.name = "Middlebury2021" + self._set_root() + assert self.split in ['train','subtrain','subval'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname) + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname.replace('/im0','/im1')) + self.pairname_to_Ldispname = lambda pairname: osp.join(self.root, pairname.split('/')[0], 'disp0.pfm') + self.pairname_to_str = lambda pairname: pairname[:-4] + self.load_disparity = _read_middlebury_disp + + def _build_cache(self): + seqs = sorted(os.listdir(self.root)) + trainpairs = [] + for s in seqs: + #trainpairs += [s+'/im0.png'] # we should remove it, it is included as such in other lightings + trainpairs += [s+'/ambient/'+b+'/'+a for b in sorted(os.listdir(osp.join(self.root,s,'ambient'))) for a in sorted(os.listdir(osp.join(self.root,s,'ambient',b))) if a.startswith('im0')] + assert len(trainpairs)==355 + subtrainpairs = [p for p in trainpairs if any(p.startswith(s+'/') for s in seqs[:-2])] + subvalpairs = [p for p in trainpairs if any(p.startswith(s+'/') for s in seqs[-2:])] + assert len(subtrainpairs)==335 and len(subvalpairs)==20, "incorrect parsing of pairs in Middlebury 2021" + tosave = {'train': trainpairs, 'subtrain': subtrainpairs, 'subval': subvalpairs} + return tosave + +class Md14Dataset(StereoDataset): + + def _prepare_data(self): + self.name = "Middlebury2014" + self._set_root() + assert self.split in ['train','subtrain','subval'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, osp.dirname(pairname), 'im0.png') + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname) + self.pairname_to_Ldispname = lambda pairname: osp.join(self.root, osp.dirname(pairname), 'disp0.pfm') + self.pairname_to_str = lambda pairname: pairname[:-4] + self.load_disparity = _read_middlebury_disp + self.has_constant_resolution = False + + def _build_cache(self): + seqs = sorted(os.listdir(self.root)) + trainpairs = [] + for s in seqs: + trainpairs += [s+'/im1.png',s+'/im1E.png',s+'/im1L.png'] + assert len(trainpairs)==138 + valseqs = ['Umbrella-imperfect','Vintage-perfect'] + assert all(s in seqs for s in valseqs) + subtrainpairs = [p for p in trainpairs if not any(p.startswith(s+'/') for s in valseqs)] + subvalpairs = [p for p in trainpairs if any(p.startswith(s+'/') for s in valseqs)] + assert len(subtrainpairs)==132 and len(subvalpairs)==6, "incorrect parsing of pairs in Middlebury 2014" + tosave = {'train': trainpairs, 'subtrain': subtrainpairs, 'subval': subvalpairs} + return tosave + +class Md06Dataset(StereoDataset): + + def _prepare_data(self): + self.name = "Middlebury2006" + self._set_root() + assert self.split in ['train','subtrain','subval'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname) + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, osp.dirname(pairname), 'view5.png') + self.pairname_to_Ldispname = lambda pairname: osp.join(self.root, pairname.split('/')[0], 'disp1.png') + self.load_disparity = _read_middlebury20052006_disp + self.has_constant_resolution = False + + def _build_cache(self): + seqs = sorted(os.listdir(self.root)) + trainpairs = [] + for s in seqs: + for i in ['Illum1','Illum2','Illum3']: + for e in ['Exp0','Exp1','Exp2']: + trainpairs.append(osp.join(s,i,e,'view1.png')) + assert len(trainpairs)==189 + valseqs = ['Rocks1','Wood2'] + assert all(s in seqs for s in valseqs) + subtrainpairs = [p for p in trainpairs if not any(p.startswith(s+'/') for s in valseqs)] + subvalpairs = [p for p in trainpairs if any(p.startswith(s+'/') for s in valseqs)] + assert len(subtrainpairs)==171 and len(subvalpairs)==18, "incorrect parsing of pairs in Middlebury 2006" + tosave = {'train': trainpairs, 'subtrain': subtrainpairs, 'subval': subvalpairs} + return tosave + +class Md05Dataset(StereoDataset): + + def _prepare_data(self): + self.name = "Middlebury2005" + self._set_root() + assert self.split in ['train','subtrain','subval'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname) + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, osp.dirname(pairname), 'view5.png') + self.pairname_to_Ldispname = lambda pairname: osp.join(self.root, pairname.split('/')[0], 'disp1.png') + self.pairname_to_str = lambda pairname: pairname[:-4] + self.load_disparity = _read_middlebury20052006_disp + + def _build_cache(self): + seqs = sorted(os.listdir(self.root)) + trainpairs = [] + for s in seqs: + for i in ['Illum1','Illum2','Illum3']: + for e in ['Exp0','Exp1','Exp2']: + trainpairs.append(osp.join(s,i,e,'view1.png')) + assert len(trainpairs)==54, "incorrect parsing of pairs in Middlebury 2005" + valseqs = ['Reindeer'] + assert all(s in seqs for s in valseqs) + subtrainpairs = [p for p in trainpairs if not any(p.startswith(s+'/') for s in valseqs)] + subvalpairs = [p for p in trainpairs if any(p.startswith(s+'/') for s in valseqs)] + assert len(subtrainpairs)==45 and len(subvalpairs)==9, "incorrect parsing of pairs in Middlebury 2005" + tosave = {'train': trainpairs, 'subtrain': subtrainpairs, 'subval': subvalpairs} + return tosave + +class MdEval3Dataset(StereoDataset): + + def _prepare_data(self): + self.name = "MiddleburyEval3" + self._set_root() + assert self.split in [s+'_'+r for s in ['train','subtrain','subval','test','all'] for r in ['full','half','quarter']] + if self.split.endswith('_full'): + self.root = self.root.replace('/MiddEval3','/MiddEval3_F') + elif self.split.endswith('_half'): + self.root = self.root.replace('/MiddEval3','/MiddEval3_H') + else: + assert self.split.endswith('_quarter') + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname, 'im0.png') + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname, 'im1.png') + self.pairname_to_Ldispname = lambda pairname: None if pairname.startswith('test') else osp.join(self.root, pairname, 'disp0GT.pfm') + self.pairname_to_str = lambda pairname: pairname + self.load_disparity = _read_middlebury_disp + # for submission only + self.submission_methodname = "CroCo-Stereo" + self.submission_sresolution = 'F' if self.split.endswith('_full') else ('H' if self.split.endswith('_half') else 'Q') + + def _build_cache(self): + trainpairs = ['train/'+s for s in sorted(os.listdir(self.root+'train/'))] + testpairs = ['test/'+s for s in sorted(os.listdir(self.root+'test/'))] + subvalpairs = trainpairs[-1:] + subtrainpairs = trainpairs[:-1] + allpairs = trainpairs+testpairs + assert len(trainpairs)==15 and len(testpairs)==15 and len(subvalpairs)==1 and len(subtrainpairs)==14 and len(allpairs)==30, "incorrect parsing of pairs in Middlebury Eval v3" + tosave = {} + for r in ['full','half','quarter']: + tosave.update(**{'train_'+r: trainpairs, 'subtrain_'+r: subtrainpairs, 'subval_'+r: subvalpairs, 'test_'+r: testpairs, 'all_'+r: allpairs}) + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==2 + assert prediction.dtype==np.float32 + outfile = os.path.join(outdir, pairname.split('/')[0].replace('train','training')+self.submission_sresolution, pairname.split('/')[1], 'disp0'+self.submission_methodname+'.pfm') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + writePFM(outfile, prediction) + timefile = os.path.join( os.path.dirname(outfile), "time"+self.submission_methodname+'.txt') + with open(timefile, 'w') as fid: + fid.write(str(time)) + + def finalize_submission(self, outdir): + cmd = f'cd {outdir}/; zip -r "{self.submission_methodname}.zip" .' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at {outdir}/{self.submission_methodname}.zip') + +class ETH3DLowResDataset(StereoDataset): + + def _prepare_data(self): + self.name = "ETH3DLowRes" + self._set_root() + assert self.split in ['train','test','subtrain','subval','all'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname, 'im0.png') + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname, 'im1.png') + self.pairname_to_Ldispname = None if self.split=='test' else lambda pairname: None if pairname.startswith('test/') else osp.join(self.root, pairname.replace('train/','train_gt/'), 'disp0GT.pfm') + self.pairname_to_str = lambda pairname: pairname + self.load_disparity = _read_eth3d_disp + self.has_constant_resolution = False + + def _build_cache(self): + trainpairs = ['train/' + s for s in sorted(os.listdir(self.root+'train/'))] + testpairs = ['test/' + s for s in sorted(os.listdir(self.root+'test/'))] + assert len(trainpairs) == 27 and len(testpairs) == 20, "incorrect parsing of pairs in ETH3D Low Res" + subvalpairs = ['train/delivery_area_3s','train/electro_3l','train/playground_3l'] + assert all(p in trainpairs for p in subvalpairs) + subtrainpairs = [p for p in trainpairs if not p in subvalpairs] + assert len(subvalpairs)==3 and len(subtrainpairs)==24, "incorrect parsing of pairs in ETH3D Low Res" + tosave = {'train': trainpairs, 'test': testpairs, 'subtrain': subtrainpairs, 'subval': subvalpairs, 'all': trainpairs+testpairs} + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==2 + assert prediction.dtype==np.float32 + outfile = os.path.join(outdir, 'low_res_two_view', pairname.split('/')[1]+'.pfm') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + writePFM(outfile, prediction) + timefile = outfile[:-4]+'.txt' + with open(timefile, 'w') as fid: + fid.write('runtime '+str(time)) + + def finalize_submission(self, outdir): + cmd = f'cd {outdir}/; zip -r "eth3d_low_res_two_view_results.zip" low_res_two_view' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at {outdir}/eth3d_low_res_two_view_results.zip') + +class BoosterDataset(StereoDataset): + + def _prepare_data(self): + self.name = "Booster" + self._set_root() + assert self.split in ['train_balanced','test_balanced','subtrain_balanced','subval_balanced'] # we use only the balanced version + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname) + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname).replace('/camera_00/','/camera_02/') + self.pairname_to_Ldispname = lambda pairname: osp.join(self.root, osp.dirname(pairname), '../disp_00.npy') # same images with different colors, same gt per sequence + self.pairname_to_str = lambda pairname: pairname[:-4].replace('/camera_00/','/') + self.load_disparity = _read_booster_disp + + + def _build_cache(self): + trainseqs = sorted(os.listdir(self.root+'train/balanced')) + trainpairs = ['train/balanced/'+s+'/camera_00/'+imname for s in trainseqs for imname in sorted(os.listdir(self.root+'train/balanced/'+s+'/camera_00/'))] + testpairs = ['test/balanced/'+s+'/camera_00/'+imname for s in sorted(os.listdir(self.root+'test/balanced')) for imname in sorted(os.listdir(self.root+'test/balanced/'+s+'/camera_00/'))] + assert len(trainpairs) == 228 and len(testpairs) == 191 + subtrainpairs = [p for p in trainpairs if any(s in p for s in trainseqs[:-2])] + subvalpairs = [p for p in trainpairs if any(s in p for s in trainseqs[-2:])] + # warning: if we do validation split, we should split scenes!!! + tosave = {'train_balanced': trainpairs, 'test_balanced': testpairs, 'subtrain_balanced': subtrainpairs, 'subval_balanced': subvalpairs,} + return tosave + +class SpringDataset(StereoDataset): + + def _prepare_data(self): + self.name = "Spring" + self._set_root() + assert self.split in ['train', 'test', 'subtrain', 'subval'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname+'.png') + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname+'.png').replace('frame_right','').replace('frame_left','frame_right').replace('','frame_left') + self.pairname_to_Ldispname = lambda pairname: None if pairname.startswith('test') else osp.join(self.root, pairname+'.dsp5').replace('frame_left','disp1_left').replace('frame_right','disp1_right') + self.pairname_to_str = lambda pairname: pairname + self.load_disparity = _read_hdf5_disp + + def _build_cache(self): + trainseqs = sorted(os.listdir( osp.join(self.root,'train'))) + trainpairs = [osp.join('train',s,'frame_left',f[:-4]) for s in trainseqs for f in sorted(os.listdir(osp.join(self.root,'train',s,'frame_left')))] + testseqs = sorted(os.listdir( osp.join(self.root,'test'))) + testpairs = [osp.join('test',s,'frame_left',f[:-4]) for s in testseqs for f in sorted(os.listdir(osp.join(self.root,'test',s,'frame_left')))] + testpairs += [p.replace('frame_left','frame_right') for p in testpairs] + """maxnorm = {'0001': 32.88, '0002': 228.5, '0004': 298.2, '0005': 142.5, '0006': 113.6, '0007': 27.3, '0008': 554.5, '0009': 155.6, '0010': 126.1, '0011': 87.6, '0012': 303.2, '0013': 24.14, '0014': 82.56, '0015': 98.44, '0016': 156.9, '0017': 28.17, '0018': 21.03, '0020': 178.0, '0021': 58.06, '0022': 354.2, '0023': 8.79, '0024': 97.06, '0025': 55.16, '0026': 91.9, '0027': 156.6, '0030': 200.4, '0032': 58.66, '0033': 373.5, '0036': 149.4, '0037': 5.625, '0038': 37.0, '0039': 12.2, '0041': 453.5, '0043': 457.0, '0044': 379.5, '0045': 161.8, '0047': 105.44} # => let'use 0041""" + subtrainpairs = [p for p in trainpairs if p.split('/')[1]!='0041'] + subvalpairs = [p for p in trainpairs if p.split('/')[1]=='0041'] + assert len(trainpairs)==5000 and len(testpairs)==2000 and len(subtrainpairs)==4904 and len(subvalpairs)==96, "incorrect parsing of pairs in Spring" + tosave = {'train': trainpairs, 'test': testpairs, 'subtrain': subtrainpairs, 'subval': subvalpairs} + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==2 + assert prediction.dtype==np.float32 + outfile = os.path.join(outdir, pairname+'.dsp5').replace('frame_left','disp1_left').replace('frame_right','disp1_right') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + writeDsp5File(prediction, outfile) + + def finalize_submission(self, outdir): + assert self.split=='test' + exe = "{self.root}/disp1_subsampling" + if os.path.isfile(exe): + cmd = f'cd "{outdir}/test"; {exe} .' + print(cmd) + os.system(cmd) + else: + print('Could not find disp1_subsampling executable for submission.') + print('Please download it and run:') + print(f'cd "{outdir}/test"; .') + +class Kitti12Dataset(StereoDataset): + + def _prepare_data(self): + self.name = "Kitti12" + self._set_root() + assert self.split in ['train','test'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname+'_10.png') + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname.replace('/colored_0/','/colored_1/')+'_10.png') + self.pairname_to_Ldispname = None if self.split=='test' else lambda pairname: osp.join(self.root, pairname.replace('/colored_0/','/disp_occ/')+'_10.png') + self.pairname_to_str = lambda pairname: pairname.replace('/colored_0/','/') + self.load_disparity = _read_kitti_disp + + def _build_cache(self): + trainseqs = ["training/colored_0/%06d"%(i) for i in range(194)] + testseqs = ["testing/colored_0/%06d"%(i) for i in range(195)] + assert len(trainseqs)==194 and len(testseqs)==195, "incorrect parsing of pairs in Kitti12" + tosave = {'train': trainseqs, 'test': testseqs} + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==2 + assert prediction.dtype==np.float32 + outfile = os.path.join(outdir, pairname.split('/')[-1]+'_10.png') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + img = (prediction * 256).astype('uint16') + Image.fromarray(img).save(outfile) + + def finalize_submission(self, outdir): + assert self.split=='test' + cmd = f'cd {outdir}/; zip -r "kitti12_results.zip" .' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at {outdir}/kitti12_results.zip') + +class Kitti15Dataset(StereoDataset): + + def _prepare_data(self): + self.name = "Kitti15" + self._set_root() + assert self.split in ['train','subtrain','subval','test'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname+'_10.png') + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname.replace('/image_2/','/image_3/')+'_10.png') + self.pairname_to_Ldispname = None if self.split=='test' else lambda pairname: osp.join(self.root, pairname.replace('/image_2/','/disp_occ_0/')+'_10.png') + self.pairname_to_str = lambda pairname: pairname.replace('/image_2/','/') + self.load_disparity = _read_kitti_disp + + def _build_cache(self): + trainseqs = ["training/image_2/%06d"%(i) for i in range(200)] + subtrainseqs = trainseqs[:-5] + subvalseqs = trainseqs[-5:] + testseqs = ["testing/image_2/%06d"%(i) for i in range(200)] + assert len(trainseqs)==200 and len(subtrainseqs)==195 and len(subvalseqs)==5 and len(testseqs)==200, "incorrect parsing of pairs in Kitti15" + tosave = {'train': trainseqs, 'subtrain': subtrainseqs, 'subval': subvalseqs, 'test': testseqs} + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==2 + assert prediction.dtype==np.float32 + outfile = os.path.join(outdir, 'disp_0', pairname.split('/')[-1]+'_10.png') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + img = (prediction * 256).astype('uint16') + Image.fromarray(img).save(outfile) + + def finalize_submission(self, outdir): + assert self.split=='test' + cmd = f'cd {outdir}/; zip -r "kitti15_results.zip" disp_0' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at {outdir}/kitti15_results.zip') + + +### auxiliary functions + +def _read_img(filename): + # convert to RGB for scene flow finalpass data + img = np.asarray(Image.open(filename).convert('RGB')) + return img + +def _read_booster_disp(filename): + disp = np.load(filename) + disp[disp==0.0] = np.inf + return disp + +def _read_png_disp(filename, coef=1.0): + disp = np.asarray(Image.open(filename)) + disp = disp.astype(np.float32) / coef + disp[disp==0.0] = np.inf + return disp + +def _read_pfm_disp(filename): + disp = np.ascontiguousarray(_read_pfm(filename)[0]) + disp[disp<=0] = np.inf # eg /nfs/data/ffs-3d/datasets/middlebury/2014/Shopvac-imperfect/disp0.pfm + return disp + +def _read_npy_disp(filename): + return np.load(filename) + +def _read_crestereo_disp(filename): return _read_png_disp(filename, coef=32.0) +def _read_middlebury20052006_disp(filename): return _read_png_disp(filename, coef=1.0) +def _read_kitti_disp(filename): return _read_png_disp(filename, coef=256.0) +_read_sceneflow_disp = _read_pfm_disp +_read_eth3d_disp = _read_pfm_disp +_read_middlebury_disp = _read_pfm_disp +_read_carla_disp = _read_pfm_disp +_read_tartanair_disp = _read_npy_disp + +def _read_hdf5_disp(filename): + disp = np.asarray(h5py.File(filename)['disparity']) + disp[np.isnan(disp)] = np.inf # make invalid values as +inf + #disp[disp==0.0] = np.inf # make invalid values as +inf + return disp.astype(np.float32) + +import re +def _read_pfm(file): + file = open(file, 'rb') + + color = None + width = None + height = None + scale = None + endian = None + + header = file.readline().rstrip() + if header.decode("ascii") == 'PF': + color = True + elif header.decode("ascii") == 'Pf': + color = False + else: + raise Exception('Not a PFM file.') + + dim_match = re.match(r'^(\d+)\s(\d+)\s$', file.readline().decode("ascii")) + if dim_match: + width, height = list(map(int, dim_match.groups())) + else: + raise Exception('Malformed PFM header.') + + scale = float(file.readline().decode("ascii").rstrip()) + if scale < 0: # little-endian + endian = '<' + scale = -scale + else: + endian = '>' # big-endian + + data = np.fromfile(file, endian + 'f') + shape = (height, width, 3) if color else (height, width) + + data = np.reshape(data, shape) + data = np.flipud(data) + return data, scale + +def writePFM(file, image, scale=1): + file = open(file, 'wb') + + color = None + + if image.dtype.name != 'float32': + raise Exception('Image dtype must be float32.') + + image = np.flipud(image) + + if len(image.shape) == 3 and image.shape[2] == 3: # color image + color = True + elif len(image.shape) == 2 or len(image.shape) == 3 and image.shape[2] == 1: # greyscale + color = False + else: + raise Exception('Image must have H x W x 3, H x W x 1 or H x W dimensions.') + + file.write('PF\n' if color else 'Pf\n'.encode()) + file.write('%d %d\n'.encode() % (image.shape[1], image.shape[0])) + + endian = image.dtype.byteorder + + if endian == '<' or endian == '=' and sys.byteorder == 'little': + scale = -scale + + file.write('%f\n'.encode() % scale) + + image.tofile(file) + +def writeDsp5File(disp, filename): + with h5py.File(filename, "w") as f: + f.create_dataset("disparity", data=disp, compression="gzip", compression_opts=5) + + +# disp visualization + +def vis_disparity(disp, m=None, M=None): + if m is None: m = disp.min() + if M is None: M = disp.max() + disp_vis = (disp - m) / (M-m) * 255.0 + disp_vis = disp_vis.astype("uint8") + disp_vis = cv2.applyColorMap(disp_vis, cv2.COLORMAP_INFERNO) + return disp_vis + +# dataset getter + +def get_train_dataset_stereo(dataset_str, augmentor=True, crop_size=None): + dataset_str = dataset_str.replace('(','Dataset(') + if augmentor: + dataset_str = dataset_str.replace(')',', augmentor=True)') + if crop_size is not None: + dataset_str = dataset_str.replace(')',', crop_size={:s})'.format(str(crop_size))) + return eval(dataset_str) + +def get_test_datasets_stereo(dataset_str): + dataset_str = dataset_str.replace('(','Dataset(') + return [eval(s) for s in dataset_str.split('+')] \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/stereoflow/download_model.sh b/submodules/mast3r/dust3r/croco/stereoflow/download_model.sh new file mode 100644 index 0000000000000000000000000000000000000000..533119609108c5ec3c22ff79b10e9215c1ac5098 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/stereoflow/download_model.sh @@ -0,0 +1,12 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +model=$1 +outfile="stereoflow_models/${model}" +if [[ ! -f $outfile ]] +then + mkdir -p stereoflow_models/; + wget https://download.europe.naverlabs.com/ComputerVision/CroCo/StereoFlow_models/$1 -P stereoflow_models/; +else + echo "Model ${model} already downloaded in ${outfile}." +fi \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/stereoflow/engine.py b/submodules/mast3r/dust3r/croco/stereoflow/engine.py new file mode 100644 index 0000000000000000000000000000000000000000..c057346b99143bf6b9c4666a58215b2b91aca7a6 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/stereoflow/engine.py @@ -0,0 +1,280 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# Main function for training one epoch or testing +# -------------------------------------------------------- + +import math +import sys +from typing import Iterable +import numpy as np +import torch +import torchvision + +from utils import misc as misc + + +def split_prediction_conf(predictions, with_conf=False): + if not with_conf: + return predictions, None + conf = predictions[:,-1:,:,:] + predictions = predictions[:,:-1,:,:] + return predictions, conf + +def train_one_epoch(model: torch.nn.Module, criterion: torch.nn.Module, metrics: torch.nn.Module, + data_loader: Iterable, optimizer: torch.optim.Optimizer, + device: torch.device, epoch: int, loss_scaler, + log_writer=None, print_freq = 20, + args=None): + model.train(True) + metric_logger = misc.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', misc.SmoothedValue(window_size=1, fmt='{value:.6f}')) + header = 'Epoch: [{}]'.format(epoch) + + accum_iter = args.accum_iter + + optimizer.zero_grad() + + details = {} + + if log_writer is not None: + print('log_dir: {}'.format(log_writer.log_dir)) + + if args.img_per_epoch: + iter_per_epoch = args.img_per_epoch // args.batch_size + int(args.img_per_epoch % args.batch_size > 0) + assert len(data_loader) >= iter_per_epoch, 'Dataset is too small for so many iterations' + len_data_loader = iter_per_epoch + else: + len_data_loader, iter_per_epoch = len(data_loader), None + + for data_iter_step, (image1, image2, gt, pairname) in enumerate(metric_logger.log_every(data_loader, print_freq, header, max_iter=iter_per_epoch)): + + image1 = image1.to(device, non_blocking=True) + image2 = image2.to(device, non_blocking=True) + gt = gt.to(device, non_blocking=True) + + # we use a per iteration (instead of per epoch) lr scheduler + if data_iter_step % accum_iter == 0: + misc.adjust_learning_rate(optimizer, data_iter_step / len_data_loader + epoch, args) + + with torch.cuda.amp.autocast(enabled=bool(args.amp)): + prediction = model(image1, image2) + prediction, conf = split_prediction_conf(prediction, criterion.with_conf) + batch_metrics = metrics(prediction.detach(), gt) + loss = criterion(prediction, gt) if conf is None else criterion(prediction, gt, conf) + + loss_value = loss.item() + if not math.isfinite(loss_value): + print("Loss is {}, stopping training".format(loss_value)) + sys.exit(1) + + loss /= accum_iter + loss_scaler(loss, optimizer, parameters=model.parameters(), + update_grad=(data_iter_step + 1) % accum_iter == 0) + if (data_iter_step + 1) % accum_iter == 0: + optimizer.zero_grad() + + torch.cuda.synchronize() + + metric_logger.update(loss=loss_value) + for k,v in batch_metrics.items(): + metric_logger.update(**{k: v.item()}) + lr = optimizer.param_groups[0]["lr"] + metric_logger.update(lr=lr) + + #if args.dsitributed: loss_value_reduce = misc.all_reduce_mean(loss_value) + time_to_log = ((data_iter_step + 1) % (args.tboard_log_step * accum_iter) == 0 or data_iter_step == len_data_loader-1) + loss_value_reduce = misc.all_reduce_mean(loss_value) + if log_writer is not None and time_to_log: + epoch_1000x = int((data_iter_step / len_data_loader + epoch) * 1000) + # We use epoch_1000x as the x-axis in tensorboard. This calibrates different curves when batch size changes. + log_writer.add_scalar('train/loss', loss_value_reduce, epoch_1000x) + log_writer.add_scalar('lr', lr, epoch_1000x) + for k,v in batch_metrics.items(): + log_writer.add_scalar('train/'+k, v.item(), epoch_1000x) + + # gather the stats from all processes + #if args.distributed: metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + return {k: meter.global_avg for k, meter in metric_logger.meters.items()} + + +@torch.no_grad() +def validate_one_epoch(model: torch.nn.Module, + criterion: torch.nn.Module, + metrics: torch.nn.Module, + data_loaders: list[Iterable], + device: torch.device, + epoch: int, + log_writer=None, + args=None): + + model.eval() + metric_loggers = [] + header = 'Epoch: [{}]'.format(epoch) + print_freq = 20 + + conf_mode = args.tile_conf_mode + crop = args.crop + + if log_writer is not None: + print('log_dir: {}'.format(log_writer.log_dir)) + + results = {} + dnames = [] + image1, image2, gt, prediction = None, None, None, None + for didx, data_loader in enumerate(data_loaders): + dname = str(data_loader.dataset) + dnames.append(dname) + metric_loggers.append(misc.MetricLogger(delimiter=" ")) + for data_iter_step, (image1, image2, gt, pairname) in enumerate(metric_loggers[didx].log_every(data_loader, print_freq, header)): + image1 = image1.to(device, non_blocking=True) + image2 = image2.to(device, non_blocking=True) + gt = gt.to(device, non_blocking=True) + if dname.startswith('Spring'): + assert gt.size(2)==image1.size(2)*2 and gt.size(3)==image1.size(3)*2 + gt = (gt[:,:,0::2,0::2] + gt[:,:,0::2,1::2] + gt[:,:,1::2,0::2] + gt[:,:,1::2,1::2] ) / 4.0 # we approximate the gt based on the 2x upsampled ones + + with torch.inference_mode(): + prediction, tiled_loss, c = tiled_pred(model, criterion, image1, image2, gt, conf_mode=conf_mode, overlap=args.val_overlap, crop=crop, with_conf=criterion.with_conf) + batch_metrics = metrics(prediction.detach(), gt) + loss = criterion(prediction.detach(), gt) if not criterion.with_conf else criterion(prediction.detach(), gt, c) + loss_value = loss.item() + metric_loggers[didx].update(loss_tiled=tiled_loss.item()) + metric_loggers[didx].update(**{f'loss': loss_value}) + for k,v in batch_metrics.items(): + metric_loggers[didx].update(**{dname+'_' + k: v.item()}) + + results = {k: meter.global_avg for ml in metric_loggers for k, meter in ml.meters.items()} + if len(dnames)>1: + for k in batch_metrics.keys(): + results['AVG_'+k] = sum(results[dname+'_'+k] for dname in dnames) / len(dnames) + + if log_writer is not None : + epoch_1000x = int((1 + epoch) * 1000) + for k,v in results.items(): + log_writer.add_scalar('val/'+k, v, epoch_1000x) + + print("Averaged stats:", results) + return results + +import torch.nn.functional as F +def _resize_img(img, new_size): + return F.interpolate(img, size=new_size, mode='bicubic', align_corners=False) +def _resize_stereo_or_flow(data, new_size): + assert data.ndim==4 + assert data.size(1) in [1,2] + scale_x = new_size[1]/float(data.size(3)) + out = F.interpolate(data, size=new_size, mode='bicubic', align_corners=False) + out[:,0,:,:] *= scale_x + if out.size(1)==2: + scale_y = new_size[0]/float(data.size(2)) + out[:,1,:,:] *= scale_y + print(scale_x, new_size, data.shape) + return out + + +@torch.no_grad() +def tiled_pred(model, criterion, img1, img2, gt, + overlap=0.5, bad_crop_thr=0.05, + downscale=False, crop=512, ret='loss', + conf_mode='conf_expsigmoid_10_5', with_conf=False, + return_time=False): + + # for each image, we are going to run inference on many overlapping patches + # then, all predictions will be weighted-averaged + if gt is not None: + B, C, H, W = gt.shape + else: + B, _, H, W = img1.shape + C = model.head.num_channels-int(with_conf) + win_height, win_width = crop[0], crop[1] + + # upscale to be larger than the crop + do_change_scale = H= window and 0 <= overlap < 1, (total, window, overlap) + num_windows = 1 + int(np.ceil( (total - window) / ((1-overlap) * window) )) + offsets = np.linspace(0, total-window, num_windows).round().astype(int) + yield from (slice(x, x+window) for x in offsets) + +def _crop(img, sy, sx): + B, THREE, H, W = img.shape + if 0 <= sy.start and sy.stop <= H and 0 <= sx.start and sx.stop <= W: + return img[:,:,sy,sx] + l, r = max(0,-sx.start), max(0,sx.stop-W) + t, b = max(0,-sy.start), max(0,sy.stop-H) + img = torch.nn.functional.pad(img, (l,r,t,b), mode='constant') + return img[:, :, slice(sy.start+t,sy.stop+t), slice(sx.start+l,sx.stop+l)] \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/stereoflow/test.py b/submodules/mast3r/dust3r/croco/stereoflow/test.py new file mode 100644 index 0000000000000000000000000000000000000000..0248e56664c769752595af251e1eadcfa3a479d9 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/stereoflow/test.py @@ -0,0 +1,216 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# Main test function +# -------------------------------------------------------- + +import os +import argparse +import pickle +from PIL import Image +import numpy as np +from tqdm import tqdm + +import torch +from torch.utils.data import DataLoader + +import utils.misc as misc +from models.croco_downstream import CroCoDownstreamBinocular +from models.head_downstream import PixelwiseTaskWithDPT + +from stereoflow.criterion import * +from stereoflow.datasets_stereo import get_test_datasets_stereo +from stereoflow.datasets_flow import get_test_datasets_flow +from stereoflow.engine import tiled_pred + +from stereoflow.datasets_stereo import vis_disparity +from stereoflow.datasets_flow import flowToColor + +def get_args_parser(): + parser = argparse.ArgumentParser('Test CroCo models on stereo/flow', add_help=False) + # important argument + parser.add_argument('--model', required=True, type=str, help='Path to the model to evaluate') + parser.add_argument('--dataset', required=True, type=str, help="test dataset (there can be multiple dataset separated by a +)") + # tiling + parser.add_argument('--tile_conf_mode', type=str, default='', help='Weights for the tiling aggregation based on confidence (empty means use the formula from the loaded checkpoint') + parser.add_argument('--tile_overlap', type=float, default=0.7, help='overlap between tiles') + # save (it will automatically go to _/_) + parser.add_argument('--save', type=str, nargs='+', default=[], + help='what to save: \ + metrics (pickle file), \ + pred (raw prediction save as torch tensor), \ + visu (visualization in png of each prediction), \ + err10 (visualization in png of the error clamp at 10 for each prediction), \ + submission (submission file)') + # other (no impact) + parser.add_argument('--num_workers', default=4, type=int) + return parser + + +def _load_model_and_criterion(model_path, do_load_metrics, device): + print('loading model from', model_path) + assert os.path.isfile(model_path) + ckpt = torch.load(model_path, 'cpu') + + ckpt_args = ckpt['args'] + task = ckpt_args.task + tile_conf_mode = ckpt_args.tile_conf_mode + num_channels = {'stereo': 1, 'flow': 2}[task] + with_conf = eval(ckpt_args.criterion).with_conf + if with_conf: num_channels += 1 + print('head: PixelwiseTaskWithDPT()') + head = PixelwiseTaskWithDPT() + head.num_channels = num_channels + print('croco_args:', ckpt_args.croco_args) + model = CroCoDownstreamBinocular(head, **ckpt_args.croco_args) + msg = model.load_state_dict(ckpt['model'], strict=True) + model.eval() + model = model.to(device) + + if do_load_metrics: + if task=='stereo': + metrics = StereoDatasetMetrics().to(device) + else: + metrics = FlowDatasetMetrics().to(device) + else: + metrics = None + + return model, metrics, ckpt_args.crop, with_conf, task, tile_conf_mode + + +def _save_batch(pred, gt, pairnames, dataset, task, save, outdir, time, submission_dir=None): + + for i in range(len(pairnames)): + + pairname = eval(pairnames[i]) if pairnames[i].startswith('(') else pairnames[i] # unbatch pairname + fname = os.path.join(outdir, dataset.pairname_to_str(pairname)) + os.makedirs(os.path.dirname(fname), exist_ok=True) + + predi = pred[i,...] + if gt is not None: gti = gt[i,...] + + if 'pred' in save: + torch.save(predi.squeeze(0).cpu(), fname+'_pred.pth') + + if 'visu' in save: + if task=='stereo': + disparity = predi.permute((1,2,0)).squeeze(2).cpu().numpy() + m,M = None + if gt is not None: + mask = torch.isfinite(gti) + m = gt[mask].min() + M = gt[mask].max() + img_disparity = vis_disparity(disparity, m=m, M=M) + Image.fromarray(img_disparity).save(fname+'_pred.png') + else: + # normalize flowToColor according to the maxnorm of gt (or prediction if not available) + flowNorm = torch.sqrt(torch.sum( (gti if gt is not None else predi)**2, dim=0)).max().item() + imgflow = flowToColor(predi.permute((1,2,0)).cpu().numpy(), maxflow=flowNorm) + Image.fromarray(imgflow).save(fname+'_pred.png') + + if 'err10' in save: + assert gt is not None + L2err = torch.sqrt(torch.sum( (gti-predi)**2, dim=0)) + valid = torch.isfinite(gti[0,:,:]) + L2err[~valid] = 0.0 + L2err = torch.clamp(L2err, max=10.0) + red = (L2err*255.0/10.0).to(dtype=torch.uint8)[:,:,None] + zer = torch.zeros_like(red) + imgerr = torch.cat( (red,zer,zer), dim=2).cpu().numpy() + Image.fromarray(imgerr).save(fname+'_err10.png') + + if 'submission' in save: + assert submission_dir is not None + predi_np = predi.permute(1,2,0).squeeze(2).cpu().numpy() # transform into HxWx2 for flow or HxW for stereo + dataset.submission_save_pairname(pairname, predi_np, submission_dir, time) + +def main(args): + + # load the pretrained model and metrics + device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu') + model, metrics, cropsize, with_conf, task, tile_conf_mode = _load_model_and_criterion(args.model, 'metrics' in args.save, device) + if args.tile_conf_mode=='': args.tile_conf_mode = tile_conf_mode + + # load the datasets + datasets = (get_test_datasets_stereo if task=='stereo' else get_test_datasets_flow)(args.dataset) + dataloaders = [DataLoader(dataset, batch_size=1, shuffle=False, num_workers=args.num_workers, pin_memory=True, drop_last=False) for dataset in datasets] + + # run + for i,dataloader in enumerate(dataloaders): + dataset = datasets[i] + dstr = args.dataset.split('+')[i] + + outdir = args.model+'_'+misc.filename(dstr) + if 'metrics' in args.save and len(args.save)==1: + fname = os.path.join(outdir, f'conf_{args.tile_conf_mode}_overlap_{args.tile_overlap}.pkl') + if os.path.isfile(fname) and len(args.save)==1: + print(' metrics already compute in '+fname) + with open(fname, 'rb') as fid: + results = pickle.load(fid) + for k,v in results.items(): + print('{:s}: {:.3f}'.format(k, v)) + continue + + if 'submission' in args.save: + dirname = f'submission_conf_{args.tile_conf_mode}_overlap_{args.tile_overlap}' + submission_dir = os.path.join(outdir, dirname) + else: + submission_dir = None + + print('') + print('saving {:s} in {:s}'.format('+'.join(args.save), outdir)) + print(repr(dataset)) + + if metrics is not None: + metrics.reset() + + for data_iter_step, (image1, image2, gt, pairnames) in enumerate(tqdm(dataloader)): + + do_flip = (task=='stereo' and dstr.startswith('Spring') and any("right" in p for p in pairnames)) # we flip the images and will flip the prediction after as we assume img1 is on the left + + image1 = image1.to(device, non_blocking=True) + image2 = image2.to(device, non_blocking=True) + gt = gt.to(device, non_blocking=True) if gt.numel()>0 else None # special case for test time + if do_flip: + assert all("right" in p for p in pairnames) + image1 = image1.flip(dims=[3]) # this is already the right frame, let's flip it + image2 = image2.flip(dims=[3]) + gt = gt # that is ok + + with torch.inference_mode(): + pred, _, _, time = tiled_pred(model, None, image1, image2, None if dataset.name=='Spring' else gt, conf_mode=args.tile_conf_mode, overlap=args.tile_overlap, crop=cropsize, with_conf=with_conf, return_time=True) + + if do_flip: + pred = pred.flip(dims=[3]) + + if metrics is not None: + metrics.add_batch(pred, gt) + + if any(k in args.save for k in ['pred','visu','err10','submission']): + _save_batch(pred, gt, pairnames, dataset, task, args.save, outdir, time, submission_dir=submission_dir) + + + # print + if metrics is not None: + results = metrics.get_results() + for k,v in results.items(): + print('{:s}: {:.3f}'.format(k, v)) + + # save if needed + if 'metrics' in args.save: + os.makedirs(os.path.dirname(fname), exist_ok=True) + with open(fname, 'wb') as fid: + pickle.dump(results, fid) + print('metrics saved in', fname) + + # finalize submission if needed + if 'submission' in args.save: + dataset.finalize_submission(submission_dir) + + + +if __name__ == '__main__': + args = get_args_parser() + args = args.parse_args() + main(args) \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/stereoflow/train.py b/submodules/mast3r/dust3r/croco/stereoflow/train.py new file mode 100644 index 0000000000000000000000000000000000000000..91f2414ffbe5ecd547d31c0e2455478d402719d6 --- /dev/null +++ b/submodules/mast3r/dust3r/croco/stereoflow/train.py @@ -0,0 +1,253 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# Main training function +# -------------------------------------------------------- + +import argparse +import datetime +import json +import numpy as np +import os +import sys +import time + +import torch +import torch.distributed as dist +import torch.backends.cudnn as cudnn +from torch.utils.tensorboard import SummaryWriter +import torchvision.transforms as transforms +import torchvision.datasets as datasets +from torch.utils.data import DataLoader + +import utils +import utils.misc as misc +from utils.misc import NativeScalerWithGradNormCount as NativeScaler +from models.croco_downstream import CroCoDownstreamBinocular, croco_args_from_ckpt +from models.pos_embed import interpolate_pos_embed +from models.head_downstream import PixelwiseTaskWithDPT + +from stereoflow.datasets_stereo import get_train_dataset_stereo, get_test_datasets_stereo +from stereoflow.datasets_flow import get_train_dataset_flow, get_test_datasets_flow +from stereoflow.engine import train_one_epoch, validate_one_epoch +from stereoflow.criterion import * + + +def get_args_parser(): + # prepare subparsers + parser = argparse.ArgumentParser('Finetuning CroCo models on stereo or flow', add_help=False) + subparsers = parser.add_subparsers(title="Task (stereo or flow)", dest="task", required=True) + parser_stereo = subparsers.add_parser('stereo', help='Training stereo model') + parser_flow = subparsers.add_parser('flow', help='Training flow model') + def add_arg(name_or_flags, default=None, default_stereo=None, default_flow=None, **kwargs): + if default is not None: assert default_stereo is None and default_flow is None, "setting default makes default_stereo and default_flow disabled" + parser_stereo.add_argument(name_or_flags, default=default if default is not None else default_stereo, **kwargs) + parser_flow.add_argument(name_or_flags, default=default if default is not None else default_flow, **kwargs) + # output dir + add_arg('--output_dir', required=True, type=str, help='path where to save, if empty, automatically created') + # model + add_arg('--crop', type=int, nargs = '+', default_stereo=[352, 704], default_flow=[320, 384], help = "size of the random image crops used during training.") + add_arg('--pretrained', required=True, type=str, help="Load pretrained model (required as croco arguments come from there)") + # criterion + add_arg('--criterion', default_stereo='LaplacianLossBounded2()', default_flow='LaplacianLossBounded()', type=str, help='string to evaluate to get criterion') + add_arg('--bestmetric', default_stereo='avgerr', default_flow='EPE', type=str) + # dataset + add_arg('--dataset', type=str, required=True, help="training set") + # training + add_arg('--seed', default=0, type=int, help='seed') + add_arg('--batch_size', default_stereo=6, default_flow=8, type=int, help='Batch size per GPU (effective batch size is batch_size * accum_iter * # gpus') + add_arg('--epochs', default=32, type=int, help='number of training epochs') + add_arg('--img_per_epoch', type=int, default=None, help='Fix the number of images seen in an epoch (None means use all training pairs)') + add_arg('--accum_iter', default=1, type=int, help='Accumulate gradient iterations (for increasing the effective batch size under memory constraints)') + add_arg('--weight_decay', type=float, default=0.05, help='weight decay (default: 0.05)') + add_arg('--lr', type=float, default_stereo=3e-5, default_flow=2e-5, metavar='LR', help='learning rate (absolute lr)') + add_arg('--min_lr', type=float, default=0., metavar='LR', help='lower lr bound for cyclic schedulers that hit 0') + add_arg('--warmup_epochs', type=int, default=1, metavar='N', help='epochs to warmup LR') + add_arg('--optimizer', default='AdamW(param_groups, lr=args.lr, betas=(0.9, 0.95))', type=str, + help="Optimizer from torch.optim [ default: AdamW(param_groups, lr=args.lr, betas=(0.9, 0.95)) ]") + add_arg('--amp', default=0, type=int, choices=[0,1], help='enable automatic mixed precision training') + # validation + add_arg('--val_dataset', type=str, default='', help="Validation sets, multiple separated by + (empty string means that no validation is performed)") + add_arg('--tile_conf_mode', type=str, default_stereo='conf_expsigmoid_15_3', default_flow='conf_expsigmoid_10_5', help='Weights for tile aggregation') + add_arg('--val_overlap', default=0.7, type=float, help='Overlap value for the tiling') + # others + add_arg('--num_workers', default=8, type=int) + add_arg('--eval_every', type=int, default=1, help='Val loss evaluation frequency') + add_arg('--save_every', type=int, default=1, help='Save checkpoint frequency') + add_arg('--start_from', type=str, default=None, help='Start training using weights from an other model (eg for finetuning)') + add_arg('--tboard_log_step', type=int, default=100, help='Log to tboard every so many steps') + add_arg('--dist_url', default='env://', help='url used to set up distributed training') + + return parser + + +def main(args): + misc.init_distributed_mode(args) + global_rank = misc.get_rank() + num_tasks = misc.get_world_size() + + assert os.path.isfile(args.pretrained) + print("output_dir: "+args.output_dir) + os.makedirs(args.output_dir, exist_ok=True) + + # fix the seed for reproducibility + seed = args.seed + misc.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + cudnn.benchmark = True + + # Metrics / criterion + device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') + metrics = (StereoMetrics if args.task=='stereo' else FlowMetrics)().to(device) + criterion = eval(args.criterion).to(device) + print('Criterion: ', args.criterion) + + # Prepare model + assert os.path.isfile(args.pretrained) + ckpt = torch.load(args.pretrained, 'cpu') + croco_args = croco_args_from_ckpt(ckpt) + croco_args['img_size'] = (args.crop[0], args.crop[1]) + print('Croco args: '+str(croco_args)) + args.croco_args = croco_args # saved for test time + # prepare head + num_channels = {'stereo': 1, 'flow': 2}[args.task] + if criterion.with_conf: num_channels += 1 + print(f'Building head PixelwiseTaskWithDPT() with {num_channels} channel(s)') + head = PixelwiseTaskWithDPT() + head.num_channels = num_channels + # build model and load pretrained weights + model = CroCoDownstreamBinocular(head, **croco_args) + interpolate_pos_embed(model, ckpt['model']) + msg = model.load_state_dict(ckpt['model'], strict=False) + print(msg) + + total_params = sum(p.numel() for p in model.parameters()) + total_params_trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) + print(f"Total params: {total_params}") + print(f"Total params trainable: {total_params_trainable}") + model_without_ddp = model.to(device) + + eff_batch_size = args.batch_size * args.accum_iter * misc.get_world_size() + print("lr: %.2e" % args.lr) + print("accumulate grad iterations: %d" % args.accum_iter) + print("effective batch size: %d" % eff_batch_size) + + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], static_graph=True) + model_without_ddp = model.module + + # following timm: set wd as 0 for bias and norm layers + param_groups = misc.get_parameter_groups(model_without_ddp, args.weight_decay) + optimizer = eval(f"torch.optim.{args.optimizer}") + print(optimizer) + loss_scaler = NativeScaler() + + # automatic restart + last_ckpt_fname = os.path.join(args.output_dir, f'checkpoint-last.pth') + args.resume = last_ckpt_fname if os.path.isfile(last_ckpt_fname) else None + + if not args.resume and args.start_from: + print(f"Starting from an other model's weights: {args.start_from}") + best_so_far = None + args.start_epoch = 0 + ckpt = torch.load(args.start_from, 'cpu') + msg = model_without_ddp.load_state_dict(ckpt['model'], strict=False) + print(msg) + else: + best_so_far = misc.load_model(args=args, model_without_ddp=model_without_ddp, optimizer=optimizer, loss_scaler=loss_scaler) + + if best_so_far is None: best_so_far = np.inf + + # tensorboard + log_writer = None + if global_rank == 0 and args.output_dir is not None: + log_writer = SummaryWriter(log_dir=args.output_dir, purge_step=args.start_epoch*1000) + + # dataset and loader + print('Building Train Data loader for dataset: ', args.dataset) + train_dataset = (get_train_dataset_stereo if args.task=='stereo' else get_train_dataset_flow)(args.dataset, crop_size=args.crop) + def _print_repr_dataset(d): + if isinstance(d, torch.utils.data.dataset.ConcatDataset): + for dd in d.datasets: + _print_repr_dataset(dd) + else: + print(repr(d)) + _print_repr_dataset(train_dataset) + print(' total length:', len(train_dataset)) + if args.distributed: + sampler_train = torch.utils.data.DistributedSampler( + train_dataset, num_replicas=num_tasks, rank=global_rank, shuffle=True + ) + else: + sampler_train = torch.utils.data.RandomSampler(train_dataset) + data_loader_train = torch.utils.data.DataLoader( + train_dataset, sampler=sampler_train, + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=True, + ) + if args.val_dataset=='': + data_loaders_val = None + else: + print('Building Val Data loader for datasets: ', args.val_dataset) + val_datasets = (get_test_datasets_stereo if args.task=='stereo' else get_test_datasets_flow)(args.val_dataset) + for val_dataset in val_datasets: print(repr(val_dataset)) + data_loaders_val = [DataLoader(val_dataset, batch_size=1, shuffle=False, num_workers=args.num_workers, pin_memory=True, drop_last=False) for val_dataset in val_datasets] + bestmetric = ("AVG_" if len(data_loaders_val)>1 else str(data_loaders_val[0].dataset)+'_')+args.bestmetric + + print(f"Start training for {args.epochs} epochs") + start_time = time.time() + # Training Loop + for epoch in range(args.start_epoch, args.epochs): + + if args.distributed: data_loader_train.sampler.set_epoch(epoch) + + # Train + epoch_start = time.time() + train_stats = train_one_epoch(model, criterion, metrics, data_loader_train, optimizer, device, epoch, loss_scaler, log_writer=log_writer, args=args) + epoch_time = time.time() - epoch_start + + if args.distributed: dist.barrier() + + # Validation (current naive implementation runs the validation on every gpu ... not smart ...) + if data_loaders_val is not None and args.eval_every > 0 and (epoch+1) % args.eval_every == 0: + val_epoch_start = time.time() + val_stats = validate_one_epoch(model, criterion, metrics, data_loaders_val, device, epoch, log_writer=log_writer, args=args) + val_epoch_time = time.time() - val_epoch_start + + val_best = val_stats[bestmetric] + + # Save best of all + if val_best <= best_so_far: + best_so_far = val_best + misc.save_model(args=args, model_without_ddp=model_without_ddp, optimizer=optimizer, loss_scaler=loss_scaler, epoch=epoch, best_so_far=best_so_far, fname='best') + + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + 'epoch': epoch, + **{f'val_{k}': v for k, v in val_stats.items()}} + else: + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + 'epoch': epoch,} + + if args.distributed: dist.barrier() + + # Save stuff + if args.output_dir and ((epoch+1) % args.save_every == 0 or epoch + 1 == args.epochs): + misc.save_model(args=args, model_without_ddp=model_without_ddp, optimizer=optimizer, loss_scaler=loss_scaler, epoch=epoch, best_so_far=best_so_far, fname='last') + + if args.output_dir: + if log_writer is not None: + log_writer.flush() + with open(os.path.join(args.output_dir, "log.txt"), mode="a", encoding="utf-8") as f: + f.write(json.dumps(log_stats) + "\n") + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + +if __name__ == '__main__': + args = get_args_parser() + args = args.parse_args() + main(args) \ No newline at end of file diff --git a/submodules/mast3r/dust3r/croco/utils/misc.py b/submodules/mast3r/dust3r/croco/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..132e102a662c987dce5282633cb8730b0e0d5c2d --- /dev/null +++ b/submodules/mast3r/dust3r/croco/utils/misc.py @@ -0,0 +1,463 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilitary functions for CroCo +# -------------------------------------------------------- +# References: +# MAE: https://github.com/facebookresearch/mae +# DeiT: https://github.com/facebookresearch/deit +# BEiT: https://github.com/microsoft/unilm/tree/master/beit +# -------------------------------------------------------- + +import builtins +import datetime +import os +import time +import math +import json +from collections import defaultdict, deque +from pathlib import Path +import numpy as np + +import torch +import torch.distributed as dist +from torch import inf + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device='cuda') + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value) + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if v is None: + continue + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format( + type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append( + "{}: {}".format(name, str(meter)) + ) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, header=None, max_iter=None): + i = 0 + if not header: + header = '' + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt='{avg:.4f}') + data_time = SmoothedValue(fmt='{avg:.4f}') + len_iterable = min(len(iterable), max_iter) if max_iter else len(iterable) + space_fmt = ':' + str(len(str(len_iterable))) + 'd' + log_msg = [ + header, + '[{0' + space_fmt + '}/{1}]', + 'eta: {eta}', + '{meters}', + 'time: {time}', + 'data: {data}' + ] + if torch.cuda.is_available(): + log_msg.append('max mem: {memory:.0f}') + log_msg = self.delimiter.join(log_msg) + MB = 1024.0 * 1024.0 + for it,obj in enumerate(iterable): + data_time.update(time.time() - end) + yield obj + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len_iterable - 1: + eta_seconds = iter_time.global_avg * (len_iterable - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print(log_msg.format( + i, len_iterable, eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB)) + else: + print(log_msg.format( + i, len_iterable, eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time))) + i += 1 + end = time.time() + if max_iter and it >= max_iter: + break + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('{} Total time: {} ({:.4f} s / it)'.format( + header, total_time_str, total_time / len_iterable)) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + builtin_print = builtins.print + + def print(*args, **kwargs): + force = kwargs.pop('force', False) + force = force or (get_world_size() > 8) + if is_master or force: + now = datetime.datetime.now().time() + builtin_print('[{}] '.format(now), end='') # print with time stamp + builtin_print(*args, **kwargs) + + builtins.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + nodist = args.nodist if hasattr(args,'nodist') else False + if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ and not nodist: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ['WORLD_SIZE']) + args.gpu = int(os.environ['LOCAL_RANK']) + else: + print('Not using distributed mode') + setup_for_distributed(is_master=True) # hack + args.distributed = False + return + + args.distributed = True + + torch.cuda.set_device(args.gpu) + args.dist_backend = 'nccl' + print('| distributed init (rank {}): {}, gpu {}'.format( + args.rank, args.dist_url, args.gpu), flush=True) + torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) + + +class NativeScalerWithGradNormCount: + state_dict_key = "amp_scaler" + + def __init__(self, enabled=True): + self._scaler = torch.cuda.amp.GradScaler(enabled=enabled) + + def __call__(self, loss, optimizer, clip_grad=None, parameters=None, create_graph=False, update_grad=True): + self._scaler.scale(loss).backward(create_graph=create_graph) + if update_grad: + if clip_grad is not None: + assert parameters is not None + self._scaler.unscale_(optimizer) # unscale the gradients of optimizer's assigned params in-place + norm = torch.nn.utils.clip_grad_norm_(parameters, clip_grad) + else: + self._scaler.unscale_(optimizer) + norm = get_grad_norm_(parameters) + self._scaler.step(optimizer) + self._scaler.update() + else: + norm = None + return norm + + def state_dict(self): + return self._scaler.state_dict() + + def load_state_dict(self, state_dict): + self._scaler.load_state_dict(state_dict) + + +def get_grad_norm_(parameters, norm_type: float = 2.0) -> torch.Tensor: + if isinstance(parameters, torch.Tensor): + parameters = [parameters] + parameters = [p for p in parameters if p.grad is not None] + norm_type = float(norm_type) + if len(parameters) == 0: + return torch.tensor(0.) + device = parameters[0].grad.device + if norm_type == inf: + total_norm = max(p.grad.detach().abs().max().to(device) for p in parameters) + else: + total_norm = torch.norm(torch.stack([torch.norm(p.grad.detach(), norm_type).to(device) for p in parameters]), norm_type) + return total_norm + + + + +def save_model(args, epoch, model_without_ddp, optimizer, loss_scaler, fname=None, best_so_far=None): + output_dir = Path(args.output_dir) + if fname is None: fname = str(epoch) + checkpoint_path = output_dir / ('checkpoint-%s.pth' % fname) + to_save = { + 'model': model_without_ddp.state_dict(), + 'optimizer': optimizer.state_dict(), + 'scaler': loss_scaler.state_dict(), + 'args': args, + 'epoch': epoch, + } + if best_so_far is not None: to_save['best_so_far'] = best_so_far + print(f'>> Saving model to {checkpoint_path} ...') + save_on_master(to_save, checkpoint_path) + + +def load_model(args, model_without_ddp, optimizer, loss_scaler): + args.start_epoch = 0 + best_so_far = None + if args.resume is not None: + if args.resume.startswith('https'): + checkpoint = torch.hub.load_state_dict_from_url( + args.resume, map_location='cpu', check_hash=True) + else: + checkpoint = torch.load(args.resume, map_location='cpu') + print("Resume checkpoint %s" % args.resume) + model_without_ddp.load_state_dict(checkpoint['model'], strict=False) + args.start_epoch = checkpoint['epoch'] + 1 + optimizer.load_state_dict(checkpoint['optimizer']) + if 'scaler' in checkpoint: + loss_scaler.load_state_dict(checkpoint['scaler']) + if 'best_so_far' in checkpoint: + best_so_far = checkpoint['best_so_far'] + print(" & best_so_far={:g}".format(best_so_far)) + else: + print("") + print("With optim & sched! start_epoch={:d}".format(args.start_epoch), end='') + return best_so_far + +def all_reduce_mean(x): + world_size = get_world_size() + if world_size > 1: + x_reduce = torch.tensor(x).cuda() + dist.all_reduce(x_reduce) + x_reduce /= world_size + return x_reduce.item() + else: + return x + +def _replace(text, src, tgt, rm=''): + """ Advanced string replacement. + Given a text: + - replace all elements in src by the corresponding element in tgt + - remove all elements in rm + """ + if len(tgt) == 1: + tgt = tgt * len(src) + assert len(src) == len(tgt), f"'{src}' and '{tgt}' should have the same len" + for s,t in zip(src, tgt): + text = text.replace(s,t) + for c in rm: + text = text.replace(c,'') + return text + +def filename( obj ): + """ transform a python obj or cmd into a proper filename. + - \1 gets replaced by slash '/' + - \2 gets replaced by comma ',' + """ + if not isinstance(obj, str): + obj = repr(obj) + obj = str(obj).replace('()','') + obj = _replace(obj, '_,(*/\1\2','-__x%/,', rm=' )\'"') + assert all(len(s) < 256 for s in obj.split(os.sep)), 'filename too long (>256 characters):\n'+obj + return obj + +def _get_num_layer_for_vit(var_name, enc_depth, dec_depth): + if var_name in ("cls_token", "mask_token", "pos_embed", "global_tokens"): + return 0 + elif var_name.startswith("patch_embed"): + return 0 + elif var_name.startswith("enc_blocks"): + layer_id = int(var_name.split('.')[1]) + return layer_id + 1 + elif var_name.startswith('decoder_embed') or var_name.startswith('enc_norm'): # part of the last black + return enc_depth + elif var_name.startswith('dec_blocks'): + layer_id = int(var_name.split('.')[1]) + return enc_depth + layer_id + 1 + elif var_name.startswith('dec_norm'): # part of the last block + return enc_depth + dec_depth + elif any(var_name.startswith(k) for k in ['head','prediction_head']): + return enc_depth + dec_depth + 1 + else: + raise NotImplementedError(var_name) + +def get_parameter_groups(model, weight_decay, layer_decay=1.0, skip_list=(), no_lr_scale_list=[]): + parameter_group_names = {} + parameter_group_vars = {} + enc_depth, dec_depth = None, None + # prepare layer decay values + assert layer_decay==1.0 or 0.> wrote {fpath}') + + print(f'Loaded {len(list_subscenes)} sub-scenes') + + # separate scenes + list_scenes = defaultdict(list) + for scene in list_subscenes: + scene, id = os.path.split(scene) + list_scenes[scene].append(id) + + list_scenes = list(list_scenes.items()) + print(f'from {len(list_scenes)} scenes in total') + + np.random.shuffle(list_scenes) + train_scenes = list_scenes[len(list_scenes)//10:] + val_scenes = list_scenes[:len(list_scenes)//10] + + def write_scene_list(scenes, n, fpath): + sub_scenes = [os.path.join(scene, id) for scene, ids in scenes for id in ids] + np.random.shuffle(sub_scenes) + + if len(sub_scenes) < n: + return + + with open(fpath, 'w') as f: + f.write('\n'.join(sub_scenes[:n])) + print(f'>> wrote {fpath}') + + for n in n_scenes: + write_scene_list(train_scenes, n, os.path.join(habitat_root, f'Habitat_{n}_scenes_train.txt')) + write_scene_list(val_scenes, n//10, os.path.join(habitat_root, f'Habitat_{n//10}_scenes_val.txt')) + + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--root", required=True) + parser.add_argument("--n_scenes", nargs='+', default=[1_000, 1_000_000], type=int) + + args = parser.parse_args() + find_all_scenes(args.root, args.n_scenes) diff --git a/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/__init__.py b/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a32692113d830ddc4af4e6ed608f222fbe062e6e --- /dev/null +++ b/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). diff --git a/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/habitat_sim_envmaps_renderer.py b/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/habitat_sim_envmaps_renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..4a31f1174a234b900ecaa76705fa271baf8a5669 --- /dev/null +++ b/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/habitat_sim_envmaps_renderer.py @@ -0,0 +1,170 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Render environment maps from 3D meshes using the Habitat Sim simulator. +# -------------------------------------------------------- +import numpy as np +import habitat_sim +import math +from habitat_renderer import projections + +# OpenCV to habitat camera convention transformation +R_OPENCV2HABITAT = np.stack((habitat_sim.geo.RIGHT, -habitat_sim.geo.UP, habitat_sim.geo.FRONT), axis=0) + +CUBEMAP_FACE_LABELS = ["left", "front", "right", "back", "up", "down"] +# Expressed while considering Habitat coordinates systems +CUBEMAP_FACE_ORIENTATIONS_ROTVEC = [ + [0, math.pi / 2, 0], # Left + [0, 0, 0], # Front + [0, - math.pi / 2, 0], # Right + [0, math.pi, 0], # Back + [math.pi / 2, 0, 0], # Up + [-math.pi / 2, 0, 0],] # Down + +class NoNaviguableSpaceError(RuntimeError): + def __init__(self, *args): + super().__init__(*args) + +class HabitatEnvironmentMapRenderer: + def __init__(self, + scene, + navmesh, + scene_dataset_config_file, + render_equirectangular=False, + equirectangular_resolution=(512, 1024), + render_cubemap=False, + cubemap_resolution=(512, 512), + render_depth=False, + gpu_id=0): + self.scene = scene + self.navmesh = navmesh + self.scene_dataset_config_file = scene_dataset_config_file + self.gpu_id = gpu_id + + self.render_equirectangular = render_equirectangular + self.equirectangular_resolution = equirectangular_resolution + self.equirectangular_projection = projections.EquirectangularProjection(*equirectangular_resolution) + # 3D unit ray associated to each pixel of the equirectangular map + equirectangular_rays = projections.get_projection_rays(self.equirectangular_projection) + # Not needed, but just in case. + equirectangular_rays /= np.linalg.norm(equirectangular_rays, axis=-1, keepdims=True) + # Depth map created by Habitat are produced by warping a cubemap, + # so the values do not correspond to distance to the center and need some scaling. + self.equirectangular_depth_scale_factors = 1.0 / np.max(np.abs(equirectangular_rays), axis=-1) + + self.render_cubemap = render_cubemap + self.cubemap_resolution = cubemap_resolution + + self.render_depth = render_depth + + self.seed = None + self._lazy_initialization() + + def _lazy_initialization(self): + # Lazy random seeding and instantiation of the simulator to deal with multiprocessing properly + if self.seed == None: + # Re-seed numpy generator + np.random.seed() + self.seed = np.random.randint(2**32-1) + sim_cfg = habitat_sim.SimulatorConfiguration() + sim_cfg.scene_id = self.scene + if self.scene_dataset_config_file is not None and self.scene_dataset_config_file != "": + sim_cfg.scene_dataset_config_file = self.scene_dataset_config_file + sim_cfg.random_seed = self.seed + sim_cfg.load_semantic_mesh = False + sim_cfg.gpu_device_id = self.gpu_id + + sensor_specifications = [] + + # Add cubemaps + if self.render_cubemap: + for face_id, orientation in enumerate(CUBEMAP_FACE_ORIENTATIONS_ROTVEC): + rgb_sensor_spec = habitat_sim.CameraSensorSpec() + rgb_sensor_spec.uuid = f"color_cubemap_{CUBEMAP_FACE_LABELS[face_id]}" + rgb_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR + rgb_sensor_spec.resolution = self.cubemap_resolution + rgb_sensor_spec.hfov = 90 + rgb_sensor_spec.position = [0.0, 0.0, 0.0] + rgb_sensor_spec.orientation = orientation + sensor_specifications.append(rgb_sensor_spec) + + if self.render_depth: + depth_sensor_spec = habitat_sim.CameraSensorSpec() + depth_sensor_spec.uuid = f"depth_cubemap_{CUBEMAP_FACE_LABELS[face_id]}" + depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH + depth_sensor_spec.resolution = self.cubemap_resolution + depth_sensor_spec.hfov = 90 + depth_sensor_spec.position = [0.0, 0.0, 0.0] + depth_sensor_spec.orientation = orientation + sensor_specifications.append(depth_sensor_spec) + + # Add equirectangular map + if self.render_equirectangular: + rgb_sensor_spec = habitat_sim.bindings.EquirectangularSensorSpec() + rgb_sensor_spec.uuid = "color_equirectangular" + rgb_sensor_spec.resolution = self.equirectangular_resolution + rgb_sensor_spec.position = [0.0, 0.0, 0.0] + sensor_specifications.append(rgb_sensor_spec) + + if self.render_depth: + depth_sensor_spec = habitat_sim.bindings.EquirectangularSensorSpec() + depth_sensor_spec.uuid = "depth_equirectangular" + depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH + depth_sensor_spec.resolution = self.equirectangular_resolution + depth_sensor_spec.position = [0.0, 0.0, 0.0] + depth_sensor_spec.orientation + sensor_specifications.append(depth_sensor_spec) + + agent_cfg = habitat_sim.agent.AgentConfiguration(sensor_specifications=sensor_specifications) + + cfg = habitat_sim.Configuration(sim_cfg, [agent_cfg]) + self.sim = habitat_sim.Simulator(cfg) + if self.navmesh is not None and self.navmesh != "": + # Use pre-computed navmesh (the one generated automatically does some weird stuffs like going on top of the roof) + # See https://youtu.be/kunFMRJAu2U?t=1522 regarding navmeshes + self.sim.pathfinder.load_nav_mesh(self.navmesh) + + # Check that the navmesh is not empty + if not self.sim.pathfinder.is_loaded: + # Try to compute a navmesh + navmesh_settings = habitat_sim.NavMeshSettings() + navmesh_settings.set_defaults() + self.sim.recompute_navmesh(self.sim.pathfinder, navmesh_settings, True) + + # Check that the navmesh is not empty + if not self.sim.pathfinder.is_loaded: + raise NoNaviguableSpaceError(f"No naviguable location (scene: {self.scene} -- navmesh: {self.navmesh})") + + self.agent = self.sim.initialize_agent(agent_id=0) + + def close(self): + if hasattr(self, 'sim'): + self.sim.close() + + def __del__(self): + self.close() + + def render_viewpoint(self, viewpoint_position): + agent_state = habitat_sim.AgentState() + agent_state.position = viewpoint_position + # agent_state.rotation = viewpoint_orientation + self.agent.set_state(agent_state) + viewpoint_observations = self.sim.get_sensor_observations(agent_ids=0) + + try: + # Depth map values have been obtained using cubemap rendering internally, + # so they do not really correspond to distance to the viewpoint in practice + # and they need some scaling + viewpoint_observations["depth_equirectangular"] *= self.equirectangular_depth_scale_factors + except KeyError: + pass + + data = dict(observations=viewpoint_observations, position=viewpoint_position) + return data + + def up_direction(self): + return np.asarray(habitat_sim.geo.UP).tolist() + + def R_cam_to_world(self): + return R_OPENCV2HABITAT.tolist() diff --git a/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/multiview_crop_generator.py b/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/multiview_crop_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..b86238b44a5cdd7a2e30b9d64773c2388f9711c3 --- /dev/null +++ b/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/multiview_crop_generator.py @@ -0,0 +1,93 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Generate pairs of crops from a dataset of environment maps. +# -------------------------------------------------------- +import os +import numpy as np +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" # noqa +import cv2 +import collections +from habitat_renderer import projections, projections_conversions +from habitat_renderer.habitat_sim_envmaps_renderer import HabitatEnvironmentMapRenderer + +ViewpointData = collections.namedtuple("ViewpointData", ["colormap", "distancemap", "pointmap", "position"]) + +class HabitatMultiviewCrops: + def __init__(self, + scene, + navmesh, + scene_dataset_config_file, + equirectangular_resolution=(400, 800), + crop_resolution=(240, 320), + pixel_jittering_iterations=5, + jittering_noise_level=1.0): + self.crop_resolution = crop_resolution + + self.pixel_jittering_iterations = pixel_jittering_iterations + self.jittering_noise_level = jittering_noise_level + + # Instanciate the low resolution habitat sim renderer + self.lowres_envmap_renderer = HabitatEnvironmentMapRenderer(scene=scene, + navmesh=navmesh, + scene_dataset_config_file=scene_dataset_config_file, + equirectangular_resolution=equirectangular_resolution, + render_depth=True, + render_equirectangular=True) + self.R_cam_to_world = np.asarray(self.lowres_envmap_renderer.R_cam_to_world()) + self.up_direction = np.asarray(self.lowres_envmap_renderer.up_direction()) + + # Projection applied by each environment map + self.envmap_height, self.envmap_width = self.lowres_envmap_renderer.equirectangular_resolution + base_projection = projections.EquirectangularProjection(self.envmap_height, self.envmap_width) + self.envmap_projection = projections.RotatedProjection(base_projection, self.R_cam_to_world.T) + # 3D Rays map associated to each envmap + self.envmap_rays = projections.get_projection_rays(self.envmap_projection) + + def compute_pointmap(self, distancemap, position): + # Point cloud associated to each ray + return self.envmap_rays * distancemap[:, :, None] + position + + def render_viewpoint_data(self, position): + data = self.lowres_envmap_renderer.render_viewpoint(np.asarray(position)) + colormap = data['observations']['color_equirectangular'][..., :3] # Ignore the alpha channel + distancemap = data['observations']['depth_equirectangular'] + pointmap = self.compute_pointmap(distancemap, position) + return ViewpointData(colormap=colormap, distancemap=distancemap, pointmap=pointmap, position=position) + + def extract_cropped_camera(self, projection, color_image, distancemap, pointmap, voxelmap=None): + remapper = projections_conversions.RemapProjection(input_projection=self.envmap_projection, output_projection=projection, + pixel_jittering_iterations=self.pixel_jittering_iterations, jittering_noise_level=self.jittering_noise_level) + cropped_color_image = remapper.convert( + color_image, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_WRAP, single_map=False) + cropped_distancemap = remapper.convert( + distancemap, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_WRAP, single_map=True) + cropped_pointmap = remapper.convert(pointmap, interpolation=cv2.INTER_NEAREST, + borderMode=cv2.BORDER_WRAP, single_map=True) + cropped_voxelmap = (None if voxelmap is None else + remapper.convert(voxelmap, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_WRAP, single_map=True)) + # Convert the distance map into a depth map + cropped_depthmap = np.asarray( + cropped_distancemap / np.linalg.norm(remapper.output_rays, axis=-1), dtype=cropped_distancemap.dtype) + + return cropped_color_image, cropped_depthmap, cropped_pointmap, cropped_voxelmap + +def perspective_projection_to_dict(persp_projection, position): + """ + Serialization-like function.""" + camera_params = dict(camera_intrinsics=projections.colmap_to_opencv_intrinsics(persp_projection.base_projection.K).tolist(), + size=(persp_projection.base_projection.width, persp_projection.base_projection.height), + R_cam2world=persp_projection.R_to_base_projection.T.tolist(), + t_cam2world=position) + return camera_params + + +def dict_to_perspective_projection(camera_params): + K = projections.opencv_to_colmap_intrinsics(np.asarray(camera_params["camera_intrinsics"])) + size = camera_params["size"] + R_cam2world = np.asarray(camera_params["R_cam2world"]) + projection = projections.PerspectiveProjection(K, height=size[1], width=size[0]) + projection = projections.RotatedProjection(projection, R_to_base_projection=R_cam2world.T) + position = camera_params["t_cam2world"] + return projection, position \ No newline at end of file diff --git a/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/projections.py b/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/projections.py new file mode 100644 index 0000000000000000000000000000000000000000..4db1f79d23e23a8ba144b4357c4d4daf10cf8fab --- /dev/null +++ b/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/projections.py @@ -0,0 +1,151 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Various 3D/2D projection utils, useful to sample virtual cameras. +# -------------------------------------------------------- +import numpy as np + +class EquirectangularProjection: + """ + Convention for the central pixel of the equirectangular map similar to OpenCV perspective model: + +X from left to right + +Y from top to bottom + +Z going outside the camera + EXCEPT that the top left corner of the image is assumed to have (0,0) coordinates (OpenCV assumes (-0.5,-0.5)) + """ + + def __init__(self, height, width): + self.height = height + self.width = width + self.u_scaling = (2 * np.pi) / self.width + self.v_scaling = np.pi / self.height + + def unproject(self, u, v): + """ + Args: + u, v: 2D coordinates + Returns: + unnormalized 3D rays. + """ + longitude = self.u_scaling * u - np.pi + minus_latitude = self.v_scaling * v - np.pi/2 + + cos_latitude = np.cos(minus_latitude) + x, z = np.sin(longitude) * cos_latitude, np.cos(longitude) * cos_latitude + y = np.sin(minus_latitude) + + rays = np.stack([x, y, z], axis=-1) + return rays + + def project(self, rays): + """ + Args: + rays: Bx3 array of 3D rays. + Returns: + u, v: tuple of 2D coordinates. + """ + rays = rays / np.linalg.norm(rays, axis=-1, keepdims=True) + x, y, z = [rays[..., i] for i in range(3)] + + longitude = np.arctan2(x, z) + minus_latitude = np.arcsin(y) + + u = (longitude + np.pi) * (1.0 / self.u_scaling) + v = (minus_latitude + np.pi/2) * (1.0 / self.v_scaling) + return u, v + + +class PerspectiveProjection: + """ + OpenCV convention: + World space: + +X from left to right + +Y from top to bottom + +Z going outside the camera + Pixel space: + +u from left to right + +v from top to bottom + EXCEPT that the top left corner of the image is assumed to have (0,0) coordinates (OpenCV assumes (-0.5,-0.5)). + """ + + def __init__(self, K, height, width): + self.height = height + self.width = width + self.K = K + self.Kinv = np.linalg.inv(K) + + def project(self, rays): + uv_homogeneous = np.einsum("ik, ...k -> ...i", self.K, rays) + uv = uv_homogeneous[..., :2] / uv_homogeneous[..., 2, None] + return uv[..., 0], uv[..., 1] + + def unproject(self, u, v): + uv_homogeneous = np.stack((u, v, np.ones_like(u)), axis=-1) + rays = np.einsum("ik, ...k -> ...i", self.Kinv, uv_homogeneous) + return rays + + +class RotatedProjection: + def __init__(self, base_projection, R_to_base_projection): + self.base_projection = base_projection + self.R_to_base_projection = R_to_base_projection + + @property + def width(self): + return self.base_projection.width + + @property + def height(self): + return self.base_projection.height + + def project(self, rays): + if self.R_to_base_projection is not None: + rays = np.einsum("ik, ...k -> ...i", self.R_to_base_projection, rays) + return self.base_projection.project(rays) + + def unproject(self, u, v): + rays = self.base_projection.unproject(u, v) + if self.R_to_base_projection is not None: + rays = np.einsum("ik, ...k -> ...i", self.R_to_base_projection.T, rays) + return rays + +def get_projection_rays(projection, noise_level=0): + """ + Return a 2D map of 3D rays corresponding to the projection. + If noise_level > 0, add some jittering noise to these rays. + """ + grid_u, grid_v = np.meshgrid(0.5 + np.arange(projection.width), 0.5 + np.arange(projection.height)) + if noise_level > 0: + grid_u += np.clip(0, noise_level * np.random.uniform(-0.5, 0.5, size=grid_u.shape), projection.width) + grid_v += np.clip(0, noise_level * np.random.uniform(-0.5, 0.5, size=grid_v.shape), projection.height) + return projection.unproject(grid_u, grid_v) + +def compute_camera_intrinsics(height, width, hfov): + f = width/2 / np.tan(hfov/2 * np.pi/180) + cu, cv = width/2, height/2 + return f, cu, cv + +def colmap_to_opencv_intrinsics(K): + """ + Modify camera intrinsics to follow a different convention. + Coordinates of the center of the top-left pixels are by default: + - (0.5, 0.5) in Colmap + - (0,0) in OpenCV + """ + K = K.copy() + K[0, 2] -= 0.5 + K[1, 2] -= 0.5 + return K + +def opencv_to_colmap_intrinsics(K): + """ + Modify camera intrinsics to follow a different convention. + Coordinates of the center of the top-left pixels are by default: + - (0.5, 0.5) in Colmap + - (0,0) in OpenCV + """ + K = K.copy() + K[0, 2] += 0.5 + K[1, 2] += 0.5 + return K \ No newline at end of file diff --git a/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/projections_conversions.py b/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/projections_conversions.py new file mode 100644 index 0000000000000000000000000000000000000000..4bcfed4066bbac62fa4254ea6417bf429b098b75 --- /dev/null +++ b/submodules/mast3r/dust3r/datasets_preprocess/habitat/habitat_renderer/projections_conversions.py @@ -0,0 +1,45 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Remap data from one projection to an other +# -------------------------------------------------------- +import numpy as np +import cv2 +from habitat_renderer import projections + +class RemapProjection: + def __init__(self, input_projection, output_projection, pixel_jittering_iterations=0, jittering_noise_level=0): + """ + Some naive random jittering can be introduced in the remapping to mitigate aliasing artecfacts. + """ + assert jittering_noise_level >= 0 + assert pixel_jittering_iterations >= 0 + + maps = [] + # Initial map + self.output_rays = projections.get_projection_rays(output_projection) + map_u, map_v = input_projection.project(self.output_rays) + map_u, map_v = np.asarray(map_u, dtype=np.float32), np.asarray(map_v, dtype=np.float32) + maps.append((map_u, map_v)) + + for _ in range(pixel_jittering_iterations): + # Define multiple mappings using some coordinates jittering to mitigate aliasing effects + crop_rays = projections.get_projection_rays(output_projection, jittering_noise_level) + map_u, map_v = input_projection.project(crop_rays) + map_u, map_v = np.asarray(map_u, dtype=np.float32), np.asarray(map_v, dtype=np.float32) + maps.append((map_u, map_v)) + self.maps = maps + + def convert(self, img, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_WRAP, single_map=False): + remapped = [] + for map_u, map_v in self.maps: + res = cv2.remap(img, map_u, map_v, interpolation=interpolation, borderMode=borderMode) + remapped.append(res) + if single_map: + break + if len(remapped) == 1: + res = remapped[0] + else: + res = np.asarray(np.mean(remapped, axis=0), dtype=img.dtype) + return res diff --git a/submodules/mast3r/dust3r/datasets_preprocess/habitat/preprocess_habitat.py b/submodules/mast3r/dust3r/datasets_preprocess/habitat/preprocess_habitat.py new file mode 100644 index 0000000000000000000000000000000000000000..cacbe2467a8e9629c2472b0e05fc0cf8326367e2 --- /dev/null +++ b/submodules/mast3r/dust3r/datasets_preprocess/habitat/preprocess_habitat.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# main executable for preprocessing habitat +# export METADATA_DIR="/path/to/habitat/5views_v1_512x512_metadata" +# export SCENES_DIR="/path/to/habitat/data/scene_datasets/" +# export OUTPUT_DIR="data/habitat_processed" +# export PYTHONPATH=$(pwd) +# python preprocess_habitat.py --scenes_dir=$SCENES_DIR --metadata_dir=$METADATA_DIR --output_dir=$OUTPUT_DIR | parallel -j 16 +# -------------------------------------------------------- +import os +import glob +import json +import os + +import PIL.Image +import json +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" # noqa +import cv2 +from habitat_renderer import multiview_crop_generator +from tqdm import tqdm + + +def preprocess_metadata(metadata_filename, + scenes_dir, + output_dir, + crop_resolution=[512, 512], + equirectangular_resolution=None, + fix_existing_dataset=False): + # Load data + with open(metadata_filename, "r") as f: + metadata = json.load(f) + + if metadata["scene_dataset_config_file"] == "": + scene = os.path.join(scenes_dir, metadata["scene"]) + scene_dataset_config_file = "" + else: + scene = metadata["scene"] + scene_dataset_config_file = os.path.join(scenes_dir, metadata["scene_dataset_config_file"]) + navmesh = None + + # Use 4 times the crop size as resolution for rendering the environment map. + max_res = max(crop_resolution) + + if equirectangular_resolution == None: + # Use 4 times the crop size as resolution for rendering the environment map. + max_res = max(crop_resolution) + equirectangular_resolution = (4*max_res, 8*max_res) + + print("equirectangular_resolution:", equirectangular_resolution) + + if os.path.exists(output_dir) and not fix_existing_dataset: + raise FileExistsError(output_dir) + + # Lazy initialization + highres_dataset = None + + for batch_label, batch in tqdm(metadata["view_batches"].items()): + for view_label, view_params in batch.items(): + + assert view_params["size"] == crop_resolution + label = f"{batch_label}_{view_label}" + + output_camera_params_filename = os.path.join(output_dir, f"{label}_camera_params.json") + if fix_existing_dataset and os.path.isfile(output_camera_params_filename): + # Skip generation if we are fixing a dataset and the corresponding output file already exists + continue + + # Lazy initialization + if highres_dataset is None: + highres_dataset = multiview_crop_generator.HabitatMultiviewCrops(scene=scene, + navmesh=navmesh, + scene_dataset_config_file=scene_dataset_config_file, + equirectangular_resolution=equirectangular_resolution, + crop_resolution=crop_resolution,) + os.makedirs(output_dir, exist_ok=bool(fix_existing_dataset)) + + # Generate a higher resolution crop + original_projection, position = multiview_crop_generator.dict_to_perspective_projection(view_params) + # Render an envmap at the given position + viewpoint_data = highres_dataset.render_viewpoint_data(position) + + projection = original_projection + colormap, depthmap, pointmap, _ = highres_dataset.extract_cropped_camera( + projection, viewpoint_data.colormap, viewpoint_data.distancemap, viewpoint_data.pointmap) + + camera_params = multiview_crop_generator.perspective_projection_to_dict(projection, position) + + # Color image + PIL.Image.fromarray(colormap).save(os.path.join(output_dir, f"{label}.jpeg")) + # Depth image + cv2.imwrite(os.path.join(output_dir, f"{label}_depth.exr"), + depthmap, [cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_HALF]) + # Camera parameters + with open(output_camera_params_filename, "w") as f: + json.dump(camera_params, f) + + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--metadata_dir", required=True) + parser.add_argument("--scenes_dir", required=True) + parser.add_argument("--output_dir", required=True) + parser.add_argument("--metadata_filename", default="") + + args = parser.parse_args() + + if args.metadata_filename == "": + # Walk through the metadata dir to generate commandlines + for filename in glob.iglob(os.path.join(args.metadata_dir, "**/metadata.json"), recursive=True): + output_dir = os.path.join(args.output_dir, os.path.relpath(os.path.dirname(filename), args.metadata_dir)) + if not os.path.exists(output_dir): + commandline = f"python {__file__} --metadata_filename={filename} --metadata_dir={args.metadata_dir} --scenes_dir={args.scenes_dir} --output_dir={output_dir}" + print(commandline) + else: + preprocess_metadata(metadata_filename=args.metadata_filename, + scenes_dir=args.scenes_dir, + output_dir=args.output_dir) diff --git a/submodules/mast3r/dust3r/datasets_preprocess/path_to_root.py b/submodules/mast3r/dust3r/datasets_preprocess/path_to_root.py new file mode 100644 index 0000000000000000000000000000000000000000..6e076a17a408d0a9e043fbda2d73f1592e7cb71a --- /dev/null +++ b/submodules/mast3r/dust3r/datasets_preprocess/path_to_root.py @@ -0,0 +1,13 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# DUSt3R repo root import +# -------------------------------------------------------- + +import sys +import os.path as path +HERE_PATH = path.normpath(path.dirname(__file__)) +DUST3R_REPO_PATH = path.normpath(path.join(HERE_PATH, '../')) +# workaround for sibling import +sys.path.insert(0, DUST3R_REPO_PATH) diff --git a/submodules/mast3r/dust3r/datasets_preprocess/preprocess_arkitscenes.py b/submodules/mast3r/dust3r/datasets_preprocess/preprocess_arkitscenes.py new file mode 100644 index 0000000000000000000000000000000000000000..5dbc103a82d646293e1d81f5132683e2b08cd879 --- /dev/null +++ b/submodules/mast3r/dust3r/datasets_preprocess/preprocess_arkitscenes.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Script to pre-process the arkitscenes dataset. +# Usage: +# python3 datasets_preprocess/preprocess_arkitscenes.py --arkitscenes_dir /path/to/arkitscenes --precomputed_pairs /path/to/arkitscenes_pairs +# -------------------------------------------------------- +import os +import json +import os.path as osp +import decimal +import argparse +import math +from bisect import bisect_left +from PIL import Image +import numpy as np +import quaternion +from scipy import interpolate +import cv2 + + +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument('--arkitscenes_dir', required=True) + parser.add_argument('--precomputed_pairs', required=True) + parser.add_argument('--output_dir', default='data/arkitscenes_processed') + return parser + + +def value_to_decimal(value, decimal_places): + decimal.getcontext().rounding = decimal.ROUND_HALF_UP # define rounding method + return decimal.Decimal(str(float(value))).quantize(decimal.Decimal('1e-{}'.format(decimal_places))) + + +def closest(value, sorted_list): + index = bisect_left(sorted_list, value) + if index == 0: + return sorted_list[0] + elif index == len(sorted_list): + return sorted_list[-1] + else: + value_before = sorted_list[index - 1] + value_after = sorted_list[index] + if value_after - value < value - value_before: + return value_after + else: + return value_before + + +def get_up_vectors(pose_device_to_world): + return np.matmul(pose_device_to_world, np.array([[0.0], [-1.0], [0.0], [0.0]])) + + +def get_right_vectors(pose_device_to_world): + return np.matmul(pose_device_to_world, np.array([[1.0], [0.0], [0.0], [0.0]])) + + +def read_traj(traj_path): + quaternions = [] + poses = [] + timestamps = [] + poses_p_to_w = [] + with open(traj_path) as f: + traj_lines = f.readlines() + for line in traj_lines: + tokens = line.split() + assert len(tokens) == 7 + traj_timestamp = float(tokens[0]) + + timestamps_decimal_value = value_to_decimal(traj_timestamp, 3) + timestamps.append(float(timestamps_decimal_value)) # for spline interpolation + + angle_axis = [float(tokens[1]), float(tokens[2]), float(tokens[3])] + r_w_to_p, _ = cv2.Rodrigues(np.asarray(angle_axis)) + t_w_to_p = np.asarray([float(tokens[4]), float(tokens[5]), float(tokens[6])]) + + pose_w_to_p = np.eye(4) + pose_w_to_p[:3, :3] = r_w_to_p + pose_w_to_p[:3, 3] = t_w_to_p + + pose_p_to_w = np.linalg.inv(pose_w_to_p) + + r_p_to_w_as_quat = quaternion.from_rotation_matrix(pose_p_to_w[:3, :3]) + t_p_to_w = pose_p_to_w[:3, 3] + poses_p_to_w.append(pose_p_to_w) + poses.append(t_p_to_w) + quaternions.append(r_p_to_w_as_quat) + return timestamps, poses, quaternions, poses_p_to_w + + +def main(rootdir, pairsdir, outdir): + os.makedirs(outdir, exist_ok=True) + + subdirs = ['Test', 'Training'] + for subdir in subdirs: + if not osp.isdir(osp.join(rootdir, subdir)): + continue + # STEP 1: list all scenes + outsubdir = osp.join(outdir, subdir) + os.makedirs(outsubdir, exist_ok=True) + listfile = osp.join(pairsdir, subdir, 'scene_list.json') + with open(listfile, 'r') as f: + scene_dirs = json.load(f) + + valid_scenes = [] + for scene_subdir in scene_dirs: + out_scene_subdir = osp.join(outsubdir, scene_subdir) + os.makedirs(out_scene_subdir, exist_ok=True) + + scene_dir = osp.join(rootdir, subdir, scene_subdir) + depth_dir = osp.join(scene_dir, 'lowres_depth') + rgb_dir = osp.join(scene_dir, 'vga_wide') + intrinsics_dir = osp.join(scene_dir, 'vga_wide_intrinsics') + traj_path = osp.join(scene_dir, 'lowres_wide.traj') + + # STEP 2: read selected_pairs.npz + selected_pairs_path = osp.join(pairsdir, subdir, scene_subdir, 'selected_pairs.npz') + selected_npz = np.load(selected_pairs_path) + selection, pairs = selected_npz['selection'], selected_npz['pairs'] + selected_sky_direction_scene = str(selected_npz['sky_direction_scene'][0]) + if len(selection) == 0 or len(pairs) == 0: + # not a valid scene + continue + valid_scenes.append(scene_subdir) + + # STEP 3: parse the scene and export the list of valid (K, pose, rgb, depth) and convert images + scene_metadata_path = osp.join(out_scene_subdir, 'scene_metadata.npz') + if osp.isfile(scene_metadata_path): + continue + else: + print(f'parsing {scene_subdir}') + # loads traj + timestamps, poses, quaternions, poses_cam_to_world = read_traj(traj_path) + + poses = np.array(poses) + quaternions = np.array(quaternions, dtype=np.quaternion) + quaternions = quaternion.unflip_rotors(quaternions) + timestamps = np.array(timestamps) + + selected_images = [(basename, basename.split(".png")[0].split("_")[1]) for basename in selection] + timestamps_selected = [float(frame_id) for _, frame_id in selected_images] + + sky_direction_scene, trajectories, intrinsics, images = convert_scene_metadata(scene_subdir, + intrinsics_dir, + timestamps, + quaternions, + poses, + poses_cam_to_world, + selected_images, + timestamps_selected) + assert selected_sky_direction_scene == sky_direction_scene + + os.makedirs(os.path.join(out_scene_subdir, 'vga_wide'), exist_ok=True) + os.makedirs(os.path.join(out_scene_subdir, 'lowres_depth'), exist_ok=True) + assert isinstance(sky_direction_scene, str) + for basename in images: + img_out = os.path.join(out_scene_subdir, 'vga_wide', basename.replace('.png', '.jpg')) + depth_out = os.path.join(out_scene_subdir, 'lowres_depth', basename) + if osp.isfile(img_out) and osp.isfile(depth_out): + continue + + vga_wide_path = osp.join(rgb_dir, basename) + depth_path = osp.join(depth_dir, basename) + + img = Image.open(vga_wide_path) + depth = cv2.imread(depth_path, cv2.IMREAD_UNCHANGED) + + # rotate the image + if sky_direction_scene == 'RIGHT': + try: + img = img.transpose(Image.Transpose.ROTATE_90) + except Exception: + img = img.transpose(Image.ROTATE_90) + depth = cv2.rotate(depth, cv2.ROTATE_90_COUNTERCLOCKWISE) + elif sky_direction_scene == 'LEFT': + try: + img = img.transpose(Image.Transpose.ROTATE_270) + except Exception: + img = img.transpose(Image.ROTATE_270) + depth = cv2.rotate(depth, cv2.ROTATE_90_CLOCKWISE) + elif sky_direction_scene == 'DOWN': + try: + img = img.transpose(Image.Transpose.ROTATE_180) + except Exception: + img = img.transpose(Image.ROTATE_180) + depth = cv2.rotate(depth, cv2.ROTATE_180) + + W, H = img.size + if not osp.isfile(img_out): + img.save(img_out) + + depth = cv2.resize(depth, (W, H), interpolation=cv2.INTER_NEAREST_EXACT) + if not osp.isfile(depth_out): # avoid destroying the base dataset when you mess up the paths + cv2.imwrite(depth_out, depth) + + # save at the end + np.savez(scene_metadata_path, + trajectories=trajectories, + intrinsics=intrinsics, + images=images, + pairs=pairs) + + outlistfile = osp.join(outsubdir, 'scene_list.json') + with open(outlistfile, 'w') as f: + json.dump(valid_scenes, f) + + # STEP 5: concat all scene_metadata.npz into a single file + scene_data = {} + for scene_subdir in valid_scenes: + scene_metadata_path = osp.join(outsubdir, scene_subdir, 'scene_metadata.npz') + with np.load(scene_metadata_path) as data: + trajectories = data['trajectories'] + intrinsics = data['intrinsics'] + images = data['images'] + pairs = data['pairs'] + scene_data[scene_subdir] = {'trajectories': trajectories, + 'intrinsics': intrinsics, + 'images': images, + 'pairs': pairs} + offset = 0 + counts = [] + scenes = [] + sceneids = [] + images = [] + intrinsics = [] + trajectories = [] + pairs = [] + for scene_idx, (scene_subdir, data) in enumerate(scene_data.items()): + num_imgs = data['images'].shape[0] + img_pairs = data['pairs'] + + scenes.append(scene_subdir) + sceneids.extend([scene_idx] * num_imgs) + + images.append(data['images']) + + K = np.expand_dims(np.eye(3), 0).repeat(num_imgs, 0) + K[:, 0, 0] = [fx for _, _, fx, _, _, _ in data['intrinsics']] + K[:, 1, 1] = [fy for _, _, _, fy, _, _ in data['intrinsics']] + K[:, 0, 2] = [hw for _, _, _, _, hw, _ in data['intrinsics']] + K[:, 1, 2] = [hh for _, _, _, _, _, hh in data['intrinsics']] + + intrinsics.append(K) + trajectories.append(data['trajectories']) + + # offset pairs + img_pairs[:, 0:2] += offset + pairs.append(img_pairs) + counts.append(offset) + + offset += num_imgs + + images = np.concatenate(images, axis=0) + intrinsics = np.concatenate(intrinsics, axis=0) + trajectories = np.concatenate(trajectories, axis=0) + pairs = np.concatenate(pairs, axis=0) + np.savez(osp.join(outsubdir, 'all_metadata.npz'), + counts=counts, + scenes=scenes, + sceneids=sceneids, + images=images, + intrinsics=intrinsics, + trajectories=trajectories, + pairs=pairs) + + +def convert_scene_metadata(scene_subdir, intrinsics_dir, + timestamps, quaternions, poses, poses_cam_to_world, + selected_images, timestamps_selected): + # find scene orientation + sky_direction_scene, rotated_to_cam = find_scene_orientation(poses_cam_to_world) + + # find/compute pose for selected timestamps + # most images have a valid timestamp / exact pose associated + timestamps_selected = np.array(timestamps_selected) + spline = interpolate.interp1d(timestamps, poses, kind='linear', axis=0) + interpolated_rotations = quaternion.squad(quaternions, timestamps, timestamps_selected) + interpolated_positions = spline(timestamps_selected) + + trajectories = [] + intrinsics = [] + images = [] + for i, (basename, frame_id) in enumerate(selected_images): + intrinsic_fn = osp.join(intrinsics_dir, f"{scene_subdir}_{frame_id}.pincam") + if not osp.exists(intrinsic_fn): + intrinsic_fn = osp.join(intrinsics_dir, f"{scene_subdir}_{float(frame_id) - 0.001:.3f}.pincam") + if not osp.exists(intrinsic_fn): + intrinsic_fn = osp.join(intrinsics_dir, f"{scene_subdir}_{float(frame_id) + 0.001:.3f}.pincam") + assert osp.exists(intrinsic_fn) + w, h, fx, fy, hw, hh = np.loadtxt(intrinsic_fn) # PINHOLE + + pose = np.eye(4) + pose[:3, :3] = quaternion.as_rotation_matrix(interpolated_rotations[i]) + pose[:3, 3] = interpolated_positions[i] + + images.append(basename) + if sky_direction_scene == 'RIGHT' or sky_direction_scene == 'LEFT': + intrinsics.append([h, w, fy, fx, hh, hw]) # swapped intrinsics + else: + intrinsics.append([w, h, fx, fy, hw, hh]) + trajectories.append(pose @ rotated_to_cam) # pose_cam_to_world @ rotated_to_cam = rotated(cam) to world + + return sky_direction_scene, trajectories, intrinsics, images + + +def find_scene_orientation(poses_cam_to_world): + if len(poses_cam_to_world) > 0: + up_vector = sum(get_up_vectors(p) for p in poses_cam_to_world) / len(poses_cam_to_world) + right_vector = sum(get_right_vectors(p) for p in poses_cam_to_world) / len(poses_cam_to_world) + up_world = np.array([[0.0], [0.0], [1.0], [0.0]]) + else: + up_vector = np.array([[0.0], [-1.0], [0.0], [0.0]]) + right_vector = np.array([[1.0], [0.0], [0.0], [0.0]]) + up_world = np.array([[0.0], [0.0], [1.0], [0.0]]) + + # value between 0, 180 + device_up_to_world_up_angle = np.arccos(np.clip(np.dot(np.transpose(up_world), + up_vector), -1.0, 1.0)).item() * 180.0 / np.pi + device_right_to_world_up_angle = np.arccos(np.clip(np.dot(np.transpose(up_world), + right_vector), -1.0, 1.0)).item() * 180.0 / np.pi + + up_closest_to_90 = abs(device_up_to_world_up_angle - 90.0) < abs(device_right_to_world_up_angle - 90.0) + if up_closest_to_90: + assert abs(device_up_to_world_up_angle - 90.0) < 45.0 + # LEFT + if device_right_to_world_up_angle > 90.0: + sky_direction_scene = 'LEFT' + cam_to_rotated_q = quaternion.from_rotation_vector([0.0, 0.0, math.pi / 2.0]) + else: + # note that in metadata.csv RIGHT does not exist, but again it's not accurate... + # well, turns out there are scenes oriented like this + # for example Training/41124801 + sky_direction_scene = 'RIGHT' + cam_to_rotated_q = quaternion.from_rotation_vector([0.0, 0.0, -math.pi / 2.0]) + else: + # right is close to 90 + assert abs(device_right_to_world_up_angle - 90.0) < 45.0 + if device_up_to_world_up_angle > 90.0: + sky_direction_scene = 'DOWN' + cam_to_rotated_q = quaternion.from_rotation_vector([0.0, 0.0, math.pi]) + else: + sky_direction_scene = 'UP' + cam_to_rotated_q = quaternion.quaternion(1, 0, 0, 0) + cam_to_rotated = np.eye(4) + cam_to_rotated[:3, :3] = quaternion.as_rotation_matrix(cam_to_rotated_q) + rotated_to_cam = np.linalg.inv(cam_to_rotated) + return sky_direction_scene, rotated_to_cam + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + main(args.arkitscenes_dir, args.precomputed_pairs, args.output_dir) diff --git a/submodules/mast3r/dust3r/datasets_preprocess/preprocess_blendedMVS.py b/submodules/mast3r/dust3r/datasets_preprocess/preprocess_blendedMVS.py new file mode 100644 index 0000000000000000000000000000000000000000..d22793793c1219ebb1b3ba8eff51226c2b13f657 --- /dev/null +++ b/submodules/mast3r/dust3r/datasets_preprocess/preprocess_blendedMVS.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Preprocessing code for the BlendedMVS dataset +# dataset at https://github.com/YoYo000/BlendedMVS +# 1) Download BlendedMVS.zip +# 2) Download BlendedMVS+.zip +# 3) Download BlendedMVS++.zip +# 4) Unzip everything in the same /path/to/tmp/blendedMVS/ directory +# 5) python datasets_preprocess/preprocess_blendedMVS.py --blendedmvs_dir /path/to/tmp/blendedMVS/ +# -------------------------------------------------------- +import os +import os.path as osp +import re +from tqdm import tqdm +import numpy as np +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 + +import path_to_root # noqa +from dust3r.utils.parallel import parallel_threads +from dust3r.datasets.utils import cropping # noqa + + +def get_parser(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--blendedmvs_dir', required=True) + parser.add_argument('--precomputed_pairs', required=True) + parser.add_argument('--output_dir', default='data/blendedmvs_processed') + return parser + + +def main(db_root, pairs_path, output_dir): + print('>> Listing all sequences') + sequences = [f for f in os.listdir(db_root) if len(f) == 24] + # should find 502 scenes + assert sequences, f'did not found any sequences at {db_root}' + print(f' (found {len(sequences)} sequences)') + + for i, seq in enumerate(tqdm(sequences)): + out_dir = osp.join(output_dir, seq) + os.makedirs(out_dir, exist_ok=True) + + # generate the crops + root = osp.join(db_root, seq) + cam_dir = osp.join(root, 'cams') + func_args = [(root, f[:-8], out_dir) for f in os.listdir(cam_dir) if not f.startswith('pair')] + parallel_threads(load_crop_and_save, func_args, star_args=True, leave=False) + + # verify that all pairs are there + pairs = np.load(pairs_path) + for seqh, seql, img1, img2, score in tqdm(pairs): + for view_index in [img1, img2]: + impath = osp.join(output_dir, f"{seqh:08x}{seql:016x}", f"{view_index:08n}.jpg") + assert osp.isfile(impath), f'missing image at {impath=}' + + print(f'>> Done, saved everything in {output_dir}/') + + +def load_crop_and_save(root, img, out_dir): + if osp.isfile(osp.join(out_dir, img + '.npz')): + return # already done + + # load everything + intrinsics_in, R_camin2world, t_camin2world = _load_pose(osp.join(root, 'cams', img + '_cam.txt')) + color_image_in = cv2.cvtColor(cv2.imread(osp.join(root, 'blended_images', img + + '.jpg'), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB) + depthmap_in = load_pfm_file(osp.join(root, 'rendered_depth_maps', img + '.pfm')) + + # do the crop + H, W = color_image_in.shape[:2] + assert H * 4 == W * 3 + image, depthmap, intrinsics_out, R_in2out = _crop_image(intrinsics_in, color_image_in, depthmap_in, (512, 384)) + + # write everything + image.save(osp.join(out_dir, img + '.jpg'), quality=80) + cv2.imwrite(osp.join(out_dir, img + '.exr'), depthmap) + + # New camera parameters + R_camout2world = R_camin2world @ R_in2out.T + t_camout2world = t_camin2world + np.savez(osp.join(out_dir, img + '.npz'), intrinsics=intrinsics_out, + R_cam2world=R_camout2world, t_cam2world=t_camout2world) + + +def _crop_image(intrinsics_in, color_image_in, depthmap_in, resolution_out=(800, 800)): + image, depthmap, intrinsics_out = cropping.rescale_image_depthmap( + color_image_in, depthmap_in, intrinsics_in, resolution_out) + R_in2out = np.eye(3) + return image, depthmap, intrinsics_out, R_in2out + + +def _load_pose(path, ret_44=False): + f = open(path) + RT = np.loadtxt(f, skiprows=1, max_rows=4, dtype=np.float32) + assert RT.shape == (4, 4) + RT = np.linalg.inv(RT) # world2cam to cam2world + + K = np.loadtxt(f, skiprows=2, max_rows=3, dtype=np.float32) + assert K.shape == (3, 3) + + if ret_44: + return K, RT + return K, RT[:3, :3], RT[:3, 3] # , depth_uint8_to_f32 + + +def load_pfm_file(file_path): + with open(file_path, 'rb') as file: + header = file.readline().decode('UTF-8').strip() + + if header == 'PF': + is_color = True + elif header == 'Pf': + is_color = False + else: + raise ValueError('The provided file is not a valid PFM file.') + + dimensions = re.match(r'^(\d+)\s(\d+)\s$', file.readline().decode('UTF-8')) + if dimensions: + img_width, img_height = map(int, dimensions.groups()) + else: + raise ValueError('Invalid PFM header format.') + + endian_scale = float(file.readline().decode('UTF-8').strip()) + if endian_scale < 0: + dtype = '= img_size * 3/4, and max dimension will be >= img_size")) + return parser + + +def convert_ndc_to_pinhole(focal_length, principal_point, image_size): + focal_length = np.array(focal_length) + principal_point = np.array(principal_point) + image_size_wh = np.array([image_size[1], image_size[0]]) + half_image_size = image_size_wh / 2 + rescale = half_image_size.min() + principal_point_px = half_image_size - principal_point * rescale + focal_length_px = focal_length * rescale + fx, fy = focal_length_px[0], focal_length_px[1] + cx, cy = principal_point_px[0], principal_point_px[1] + K = np.array([[fx, 0.0, cx], [0.0, fy, cy], [0.0, 0.0, 1.0]], dtype=np.float32) + return K + + +def opencv_from_cameras_projection(R, T, focal, p0, image_size): + R = torch.from_numpy(R)[None, :, :] + T = torch.from_numpy(T)[None, :] + focal = torch.from_numpy(focal)[None, :] + p0 = torch.from_numpy(p0)[None, :] + image_size = torch.from_numpy(image_size)[None, :] + + R_pytorch3d = R.clone() + T_pytorch3d = T.clone() + focal_pytorch3d = focal + p0_pytorch3d = p0 + T_pytorch3d[:, :2] *= -1 + R_pytorch3d[:, :, :2] *= -1 + tvec = T_pytorch3d + R = R_pytorch3d.permute(0, 2, 1) + + # Retype the image_size correctly and flip to width, height. + image_size_wh = image_size.to(R).flip(dims=(1,)) + + # NDC to screen conversion. + scale = image_size_wh.to(R).min(dim=1, keepdim=True)[0] / 2.0 + scale = scale.expand(-1, 2) + c0 = image_size_wh / 2.0 + + principal_point = -p0_pytorch3d * scale + c0 + focal_length = focal_pytorch3d * scale + + camera_matrix = torch.zeros_like(R) + camera_matrix[:, :2, 2] = principal_point + camera_matrix[:, 2, 2] = 1.0 + camera_matrix[:, 0, 0] = focal_length[:, 0] + camera_matrix[:, 1, 1] = focal_length[:, 1] + return R[0], tvec[0], camera_matrix[0] + + +def get_set_list(category_dir, split, is_single_sequence_subset=False): + listfiles = os.listdir(osp.join(category_dir, "set_lists")) + if is_single_sequence_subset: + # not all objects have manyview_dev + subset_list_files = [f for f in listfiles if "manyview_dev" in f] + else: + subset_list_files = [f for f in listfiles if f"fewview_train" in f] + + sequences_all = [] + for subset_list_file in subset_list_files: + with open(osp.join(category_dir, "set_lists", subset_list_file)) as f: + subset_lists_data = json.load(f) + sequences_all.extend(subset_lists_data[split]) + + return sequences_all + + +def prepare_sequences(category, co3d_dir, output_dir, img_size, split, min_quality, max_num_sequences_per_object, + seed, is_single_sequence_subset=False): + random.seed(seed) + category_dir = osp.join(co3d_dir, category) + category_output_dir = osp.join(output_dir, category) + sequences_all = get_set_list(category_dir, split, is_single_sequence_subset) + sequences_numbers = sorted(set(seq_name for seq_name, _, _ in sequences_all)) + + frame_file = osp.join(category_dir, "frame_annotations.jgz") + sequence_file = osp.join(category_dir, "sequence_annotations.jgz") + + with gzip.open(frame_file, "r") as fin: + frame_data = json.loads(fin.read()) + with gzip.open(sequence_file, "r") as fin: + sequence_data = json.loads(fin.read()) + + frame_data_processed = {} + for f_data in frame_data: + sequence_name = f_data["sequence_name"] + frame_data_processed.setdefault(sequence_name, {})[f_data["frame_number"]] = f_data + + good_quality_sequences = set() + for seq_data in sequence_data: + if seq_data["viewpoint_quality_score"] > min_quality: + good_quality_sequences.add(seq_data["sequence_name"]) + + sequences_numbers = [seq_name for seq_name in sequences_numbers if seq_name in good_quality_sequences] + if len(sequences_numbers) < max_num_sequences_per_object: + selected_sequences_numbers = sequences_numbers + else: + selected_sequences_numbers = random.sample(sequences_numbers, max_num_sequences_per_object) + + selected_sequences_numbers_dict = {seq_name: [] for seq_name in selected_sequences_numbers} + sequences_all = [(seq_name, frame_number, filepath) + for seq_name, frame_number, filepath in sequences_all + if seq_name in selected_sequences_numbers_dict] + + for seq_name, frame_number, filepath in tqdm(sequences_all): + frame_idx = int(filepath.split('/')[-1][5:-4]) + selected_sequences_numbers_dict[seq_name].append(frame_idx) + mask_path = filepath.replace("images", "masks").replace(".jpg", ".png") + frame_data = frame_data_processed[seq_name][frame_number] + focal_length = frame_data["viewpoint"]["focal_length"] + principal_point = frame_data["viewpoint"]["principal_point"] + image_size = frame_data["image"]["size"] + K = convert_ndc_to_pinhole(focal_length, principal_point, image_size) + R, tvec, camera_intrinsics = opencv_from_cameras_projection(np.array(frame_data["viewpoint"]["R"]), + np.array(frame_data["viewpoint"]["T"]), + np.array(focal_length), + np.array(principal_point), + np.array(image_size)) + + frame_data = frame_data_processed[seq_name][frame_number] + depth_path = os.path.join(co3d_dir, frame_data["depth"]["path"]) + assert frame_data["depth"]["scale_adjustment"] == 1.0 + image_path = os.path.join(co3d_dir, filepath) + mask_path_full = os.path.join(co3d_dir, mask_path) + + input_rgb_image = PIL.Image.open(image_path).convert('RGB') + input_mask = plt.imread(mask_path_full) + + with PIL.Image.open(depth_path) as depth_pil: + # the image is stored with 16-bit depth but PIL reads it as I (32 bit). + # we cast it to uint16, then reinterpret as float16, then cast to float32 + input_depthmap = ( + np.frombuffer(np.array(depth_pil, dtype=np.uint16), dtype=np.float16) + .astype(np.float32) + .reshape((depth_pil.size[1], depth_pil.size[0]))) + depth_mask = np.stack((input_depthmap, input_mask), axis=-1) + H, W = input_depthmap.shape + + camera_intrinsics = camera_intrinsics.numpy() + cx, cy = camera_intrinsics[:2, 2].round().astype(int) + min_margin_x = min(cx, W - cx) + min_margin_y = min(cy, H - cy) + + # the new window will be a rectangle of size (2*min_margin_x, 2*min_margin_y) centered on (cx,cy) + l, t = cx - min_margin_x, cy - min_margin_y + r, b = cx + min_margin_x, cy + min_margin_y + crop_bbox = (l, t, r, b) + input_rgb_image, depth_mask, input_camera_intrinsics = cropping.crop_image_depthmap( + input_rgb_image, depth_mask, camera_intrinsics, crop_bbox) + + # try to set the lower dimension to img_size * 3/4 -> img_size=512 => 384 + scale_final = ((img_size * 3 // 4) / min(H, W)) + 1e-8 + output_resolution = np.floor(np.array([W, H]) * scale_final).astype(int) + if max(output_resolution) < img_size: + # let's put the max dimension to img_size + scale_final = (img_size / max(H, W)) + 1e-8 + output_resolution = np.floor(np.array([W, H]) * scale_final).astype(int) + + input_rgb_image, depth_mask, input_camera_intrinsics = cropping.rescale_image_depthmap( + input_rgb_image, depth_mask, input_camera_intrinsics, output_resolution) + input_depthmap = depth_mask[:, :, 0] + input_mask = depth_mask[:, :, 1] + + # generate and adjust camera pose + camera_pose = np.eye(4, dtype=np.float32) + camera_pose[:3, :3] = R + camera_pose[:3, 3] = tvec + camera_pose = np.linalg.inv(camera_pose) + + # save crop images and depth, metadata + save_img_path = os.path.join(output_dir, filepath) + save_depth_path = os.path.join(output_dir, frame_data["depth"]["path"]) + save_mask_path = os.path.join(output_dir, mask_path) + os.makedirs(os.path.split(save_img_path)[0], exist_ok=True) + os.makedirs(os.path.split(save_depth_path)[0], exist_ok=True) + os.makedirs(os.path.split(save_mask_path)[0], exist_ok=True) + + input_rgb_image.save(save_img_path) + scaled_depth_map = (input_depthmap / np.max(input_depthmap) * 65535).astype(np.uint16) + cv2.imwrite(save_depth_path, scaled_depth_map) + cv2.imwrite(save_mask_path, (input_mask * 255).astype(np.uint8)) + + save_meta_path = save_img_path.replace('jpg', 'npz') + np.savez(save_meta_path, camera_intrinsics=input_camera_intrinsics, + camera_pose=camera_pose, maximum_depth=np.max(input_depthmap)) + + return selected_sequences_numbers_dict + + +if __name__ == "__main__": + parser = get_parser() + args = parser.parse_args() + assert args.co3d_dir != args.output_dir + if args.category is None: + if args.single_sequence_subset: + categories = SINGLE_SEQUENCE_CATEGORIES + else: + categories = CATEGORIES + else: + categories = [args.category] + os.makedirs(args.output_dir, exist_ok=True) + + for split in ['train', 'test']: + selected_sequences_path = os.path.join(args.output_dir, f'selected_seqs_{split}.json') + if os.path.isfile(selected_sequences_path): + continue + + all_selected_sequences = {} + for category in categories: + category_output_dir = osp.join(args.output_dir, category) + os.makedirs(category_output_dir, exist_ok=True) + category_selected_sequences_path = os.path.join(category_output_dir, f'selected_seqs_{split}.json') + if os.path.isfile(category_selected_sequences_path): + with open(category_selected_sequences_path, 'r') as fid: + category_selected_sequences = json.load(fid) + else: + print(f"Processing {split} - category = {category}") + category_selected_sequences = prepare_sequences( + category=category, + co3d_dir=args.co3d_dir, + output_dir=args.output_dir, + img_size=args.img_size, + split=split, + min_quality=args.min_quality, + max_num_sequences_per_object=args.num_sequences_per_object, + seed=args.seed + CATEGORIES_IDX[category], + is_single_sequence_subset=args.single_sequence_subset + ) + with open(category_selected_sequences_path, 'w') as file: + json.dump(category_selected_sequences, file) + + all_selected_sequences[category] = category_selected_sequences + with open(selected_sequences_path, 'w') as file: + json.dump(all_selected_sequences, file) diff --git a/submodules/mast3r/dust3r/datasets_preprocess/preprocess_megadepth.py b/submodules/mast3r/dust3r/datasets_preprocess/preprocess_megadepth.py new file mode 100644 index 0000000000000000000000000000000000000000..b07c0c5dff0cfd828f9ce4fd204cf2eaa22487f1 --- /dev/null +++ b/submodules/mast3r/dust3r/datasets_preprocess/preprocess_megadepth.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Preprocessing code for the MegaDepth dataset +# dataset at https://www.cs.cornell.edu/projects/megadepth/ +# -------------------------------------------------------- +import os +import os.path as osp +import collections +from tqdm import tqdm +import numpy as np +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 +import h5py + +import path_to_root # noqa +from dust3r.utils.parallel import parallel_threads +from dust3r.datasets.utils import cropping # noqa + + +def get_parser(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--megadepth_dir', required=True) + parser.add_argument('--precomputed_pairs', required=True) + parser.add_argument('--output_dir', default='data/megadepth_processed') + return parser + + +def main(db_root, pairs_path, output_dir): + os.makedirs(output_dir, exist_ok=True) + + # load all pairs + data = np.load(pairs_path, allow_pickle=True) + scenes = data['scenes'] + images = data['images'] + pairs = data['pairs'] + + # enumerate all unique images + todo = collections.defaultdict(set) + for scene, im1, im2, score in pairs: + todo[scene].add(im1) + todo[scene].add(im2) + + # for each scene, load intrinsics and then parallel crops + for scene, im_idxs in tqdm(todo.items(), desc='Overall'): + scene, subscene = scenes[scene].split() + out_dir = osp.join(output_dir, scene, subscene) + os.makedirs(out_dir, exist_ok=True) + + # load all camera params + _, pose_w2cam, intrinsics = _load_kpts_and_poses(db_root, scene, subscene, intrinsics=True) + + in_dir = osp.join(db_root, scene, 'dense' + subscene) + args = [(in_dir, img, intrinsics[img], pose_w2cam[img], out_dir) + for img in [images[im_id] for im_id in im_idxs]] + parallel_threads(resize_one_image, args, star_args=True, front_num=0, leave=False, desc=f'{scene}/{subscene}') + + # save pairs + print('Done! prepared all pairs in', output_dir) + + +def resize_one_image(root, tag, K_pre_rectif, pose_w2cam, out_dir): + if osp.isfile(osp.join(out_dir, tag + '.npz')): + return + + # load image + img = cv2.cvtColor(cv2.imread(osp.join(root, 'imgs', tag), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB) + H, W = img.shape[:2] + + # load depth + with h5py.File(osp.join(root, 'depths', osp.splitext(tag)[0] + '.h5'), 'r') as hd5: + depthmap = np.asarray(hd5['depth']) + + # rectify = undistort the intrinsics + imsize_pre, K_pre, distortion = K_pre_rectif + imsize_post = img.shape[1::-1] + K_post = cv2.getOptimalNewCameraMatrix(K_pre, distortion, imsize_pre, alpha=0, + newImgSize=imsize_post, centerPrincipalPoint=True)[0] + + # downscale + img_out, depthmap_out, intrinsics_out, R_in2out = _downscale_image(K_post, img, depthmap, resolution_out=(800, 600)) + + # write everything + img_out.save(osp.join(out_dir, tag + '.jpg'), quality=90) + cv2.imwrite(osp.join(out_dir, tag + '.exr'), depthmap_out) + + camout2world = np.linalg.inv(pose_w2cam) + camout2world[:3, :3] = camout2world[:3, :3] @ R_in2out.T + np.savez(osp.join(out_dir, tag + '.npz'), intrinsics=intrinsics_out, cam2world=camout2world) + + +def _downscale_image(camera_intrinsics, image, depthmap, resolution_out=(512, 384)): + H, W = image.shape[:2] + resolution_out = sorted(resolution_out)[::+1 if W < H else -1] + + image, depthmap, intrinsics_out = cropping.rescale_image_depthmap( + image, depthmap, camera_intrinsics, resolution_out, force=False) + R_in2out = np.eye(3) + + return image, depthmap, intrinsics_out, R_in2out + + +def _load_kpts_and_poses(root, scene_id, subscene, z_only=False, intrinsics=False): + if intrinsics: + with open(os.path.join(root, scene_id, 'sparse', 'manhattan', subscene, 'cameras.txt'), 'r') as f: + raw = f.readlines()[3:] # skip the header + + camera_intrinsics = {} + for camera in raw: + camera = camera.split(' ') + width, height, focal, cx, cy, k0 = [float(elem) for elem in camera[2:]] + K = np.eye(3) + K[0, 0] = focal + K[1, 1] = focal + K[0, 2] = cx + K[1, 2] = cy + camera_intrinsics[int(camera[0])] = ((int(width), int(height)), K, (k0, 0, 0, 0)) + + with open(os.path.join(root, scene_id, 'sparse', 'manhattan', subscene, 'images.txt'), 'r') as f: + raw = f.read().splitlines()[4:] # skip the header + + extract_pose = colmap_raw_pose_to_principal_axis if z_only else colmap_raw_pose_to_RT + + poses = {} + points3D_idxs = {} + camera = [] + + for image, points in zip(raw[:: 2], raw[1:: 2]): + image = image.split(' ') + points = points.split(' ') + + image_id = image[-1] + camera.append(int(image[-2])) + + # find the principal axis + raw_pose = [float(elem) for elem in image[1: -2]] + poses[image_id] = extract_pose(raw_pose) + + current_points3D_idxs = {int(i) for i in points[2:: 3] if i != '-1'} + assert -1 not in current_points3D_idxs, bb() + points3D_idxs[image_id] = current_points3D_idxs + + if intrinsics: + image_intrinsics = {im_id: camera_intrinsics[cam] for im_id, cam in zip(poses, camera)} + return points3D_idxs, poses, image_intrinsics + else: + return points3D_idxs, poses + + +def colmap_raw_pose_to_principal_axis(image_pose): + qvec = image_pose[: 4] + qvec = qvec / np.linalg.norm(qvec) + w, x, y, z = qvec + z_axis = np.float32([ + 2 * x * z - 2 * y * w, + 2 * y * z + 2 * x * w, + 1 - 2 * x * x - 2 * y * y + ]) + return z_axis + + +def colmap_raw_pose_to_RT(image_pose): + qvec = image_pose[: 4] + qvec = qvec / np.linalg.norm(qvec) + w, x, y, z = qvec + R = np.array([ + [ + 1 - 2 * y * y - 2 * z * z, + 2 * x * y - 2 * z * w, + 2 * x * z + 2 * y * w + ], + [ + 2 * x * y + 2 * z * w, + 1 - 2 * x * x - 2 * z * z, + 2 * y * z - 2 * x * w + ], + [ + 2 * x * z - 2 * y * w, + 2 * y * z + 2 * x * w, + 1 - 2 * x * x - 2 * y * y + ] + ]) + # principal_axis.append(R[2, :]) + t = image_pose[4: 7] + # World-to-Camera pose + current_pose = np.eye(4) + current_pose[: 3, : 3] = R + current_pose[: 3, 3] = t + return current_pose + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + main(args.megadepth_dir, args.precomputed_pairs, args.output_dir) diff --git a/submodules/mast3r/dust3r/datasets_preprocess/preprocess_scannetpp.py b/submodules/mast3r/dust3r/datasets_preprocess/preprocess_scannetpp.py new file mode 100644 index 0000000000000000000000000000000000000000..34e26dc9474df16cf0736f71248d01b7853d4786 --- /dev/null +++ b/submodules/mast3r/dust3r/datasets_preprocess/preprocess_scannetpp.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Script to pre-process the scannet++ dataset. +# Usage: +# python3 datasets_preprocess/preprocess_scannetpp.py --scannetpp_dir /path/to/scannetpp --precomputed_pairs /path/to/scannetpp_pairs --pyopengl-platform egl +# -------------------------------------------------------- +import os +import argparse +import os.path as osp +import re +from tqdm import tqdm +import json +from scipy.spatial.transform import Rotation +import pyrender +import trimesh +import trimesh.exchange.ply +import numpy as np +import cv2 +import PIL.Image as Image + +from dust3r.datasets.utils.cropping import rescale_image_depthmap +import dust3r.utils.geometry as geometry + +inv = np.linalg.inv +norm = np.linalg.norm +REGEXPR_DSLR = re.compile(r'^DSC(?P\d+).JPG$') +REGEXPR_IPHONE = re.compile(r'frame_(?P\d+).jpg$') + +DEBUG_VIZ = None # 'iou' +if DEBUG_VIZ is not None: + import matplotlib.pyplot as plt # noqa + + +OPENGL_TO_OPENCV = np.float32([[1, 0, 0, 0], + [0, -1, 0, 0], + [0, 0, -1, 0], + [0, 0, 0, 1]]) + + +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument('--scannetpp_dir', required=True) + parser.add_argument('--precomputed_pairs', required=True) + parser.add_argument('--output_dir', default='data/scannetpp_processed') + parser.add_argument('--target_resolution', default=920, type=int, help="images resolution") + parser.add_argument('--pyopengl-platform', type=str, default='', help='PyOpenGL env variable') + return parser + + +def pose_from_qwxyz_txyz(elems): + qw, qx, qy, qz, tx, ty, tz = map(float, elems) + pose = np.eye(4) + pose[:3, :3] = Rotation.from_quat((qx, qy, qz, qw)).as_matrix() + pose[:3, 3] = (tx, ty, tz) + return np.linalg.inv(pose) # returns cam2world + + +def get_frame_number(name, cam_type='dslr'): + if cam_type == 'dslr': + regex_expr = REGEXPR_DSLR + elif cam_type == 'iphone': + regex_expr = REGEXPR_IPHONE + else: + raise NotImplementedError(f'wrong {cam_type=} for get_frame_number') + matches = re.match(regex_expr, name) + return matches['frameid'] + + +def load_sfm(sfm_dir, cam_type='dslr'): + # load cameras + with open(osp.join(sfm_dir, 'cameras.txt'), 'r') as f: + raw = f.read().splitlines()[3:] # skip header + + intrinsics = {} + for camera in tqdm(raw, position=1, leave=False): + camera = camera.split(' ') + intrinsics[int(camera[0])] = [camera[1]] + [float(cam) for cam in camera[2:]] + + # load images + with open(os.path.join(sfm_dir, 'images.txt'), 'r') as f: + raw = f.read().splitlines() + raw = [line for line in raw if not line.startswith('#')] # skip header + + img_idx = {} + img_infos = {} + for image, points in tqdm(zip(raw[0::2], raw[1::2]), total=len(raw) // 2, position=1, leave=False): + image = image.split(' ') + points = points.split(' ') + + idx = image[0] + img_name = image[-1] + assert img_name not in img_idx, 'duplicate db image: ' + img_name + img_idx[img_name] = idx # register image name + + current_points2D = {int(i): (float(x), float(y)) + for i, x, y in zip(points[2::3], points[0::3], points[1::3]) if i != '-1'} + img_infos[idx] = dict(intrinsics=intrinsics[int(image[-2])], + path=img_name, + frame_id=get_frame_number(img_name, cam_type), + cam_to_world=pose_from_qwxyz_txyz(image[1: -2]), + sparse_pts2d=current_points2D) + + # load 3D points + with open(os.path.join(sfm_dir, 'points3D.txt'), 'r') as f: + raw = f.read().splitlines() + raw = [line for line in raw if not line.startswith('#')] # skip header + + points3D = {} + observations = {idx: [] for idx in img_infos.keys()} + for point in tqdm(raw, position=1, leave=False): + point = point.split() + point_3d_idx = int(point[0]) + points3D[point_3d_idx] = tuple(map(float, point[1:4])) + if len(point) > 8: + for idx, point_2d_idx in zip(point[8::2], point[9::2]): + observations[idx].append((point_3d_idx, int(point_2d_idx))) + + return img_idx, img_infos, points3D, observations + + +def subsample_img_infos(img_infos, num_images, allowed_name_subset=None): + img_infos_val = [(idx, val) for idx, val in img_infos.items()] + if allowed_name_subset is not None: + img_infos_val = [(idx, val) for idx, val in img_infos_val if val['path'] in allowed_name_subset] + + if len(img_infos_val) > num_images: + img_infos_val = sorted(img_infos_val, key=lambda x: x[1]['frame_id']) + kept_idx = np.round(np.linspace(0, len(img_infos_val) - 1, num_images)).astype(int).tolist() + img_infos_val = [img_infos_val[idx] for idx in kept_idx] + return {idx: val for idx, val in img_infos_val} + + +def undistort_images(intrinsics, rgb, mask): + camera_type = intrinsics[0] + + width = int(intrinsics[1]) + height = int(intrinsics[2]) + fx = intrinsics[3] + fy = intrinsics[4] + cx = intrinsics[5] + cy = intrinsics[6] + distortion = np.array(intrinsics[7:]) + + K = np.zeros([3, 3]) + K[0, 0] = fx + K[0, 2] = cx + K[1, 1] = fy + K[1, 2] = cy + K[2, 2] = 1 + + K = geometry.colmap_to_opencv_intrinsics(K) + if camera_type == "OPENCV_FISHEYE": + assert len(distortion) == 4 + + new_K = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify( + K, + distortion, + (width, height), + np.eye(3), + balance=0.0, + ) + # Make the cx and cy to be the center of the image + new_K[0, 2] = width / 2.0 + new_K[1, 2] = height / 2.0 + + map1, map2 = cv2.fisheye.initUndistortRectifyMap(K, distortion, np.eye(3), new_K, (width, height), cv2.CV_32FC1) + else: + new_K, _ = cv2.getOptimalNewCameraMatrix(K, distortion, (width, height), 1, (width, height), True) + map1, map2 = cv2.initUndistortRectifyMap(K, distortion, np.eye(3), new_K, (width, height), cv2.CV_32FC1) + + undistorted_image = cv2.remap(rgb, map1, map2, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT_101) + undistorted_mask = cv2.remap(mask, map1, map2, interpolation=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, borderValue=255) + K = geometry.opencv_to_colmap_intrinsics(K) + return width, height, new_K, undistorted_image, undistorted_mask + + +def process_scenes(root, pairsdir, output_dir, target_resolution): + os.makedirs(output_dir, exist_ok=True) + + # default values from + # https://github.com/scannetpp/scannetpp/blob/main/common/configs/render.yml + znear = 0.05 + zfar = 20.0 + + listfile = osp.join(pairsdir, 'scene_list.json') + with open(listfile, 'r') as f: + scenes = json.load(f) + + # for each of these, we will select some dslr images and some iphone images + # we will undistort them and render their depth + renderer = pyrender.OffscreenRenderer(0, 0) + for scene in tqdm(scenes, position=0, leave=True): + data_dir = os.path.join(root, 'data', scene) + dir_dslr = os.path.join(data_dir, 'dslr') + dir_iphone = os.path.join(data_dir, 'iphone') + dir_scans = os.path.join(data_dir, 'scans') + + assert os.path.isdir(data_dir) and os.path.isdir(dir_dslr) \ + and os.path.isdir(dir_iphone) and os.path.isdir(dir_scans) + + output_dir_scene = os.path.join(output_dir, scene) + scene_metadata_path = osp.join(output_dir_scene, 'scene_metadata.npz') + if osp.isfile(scene_metadata_path): + continue + + pairs_dir_scene = os.path.join(pairsdir, scene) + pairs_dir_scene_selected_pairs = os.path.join(pairs_dir_scene, 'selected_pairs.npz') + assert osp.isfile(pairs_dir_scene_selected_pairs) + selected_npz = np.load(pairs_dir_scene_selected_pairs) + selection, pairs = selected_npz['selection'], selected_npz['pairs'] + + # set up the output paths + output_dir_scene_rgb = os.path.join(output_dir_scene, 'images') + output_dir_scene_depth = os.path.join(output_dir_scene, 'depth') + os.makedirs(output_dir_scene_rgb, exist_ok=True) + os.makedirs(output_dir_scene_depth, exist_ok=True) + + ply_path = os.path.join(dir_scans, 'mesh_aligned_0.05.ply') + + sfm_dir_dslr = os.path.join(dir_dslr, 'colmap') + rgb_dir_dslr = os.path.join(dir_dslr, 'resized_images') + mask_dir_dslr = os.path.join(dir_dslr, 'resized_anon_masks') + + sfm_dir_iphone = os.path.join(dir_iphone, 'colmap') + rgb_dir_iphone = os.path.join(dir_iphone, 'rgb') + mask_dir_iphone = os.path.join(dir_iphone, 'rgb_masks') + + # load the mesh + with open(ply_path, 'rb') as f: + mesh_kwargs = trimesh.exchange.ply.load_ply(f) + mesh_scene = trimesh.Trimesh(**mesh_kwargs) + + # read colmap reconstruction, we will only use the intrinsics and pose here + img_idx_dslr, img_infos_dslr, points3D_dslr, observations_dslr = load_sfm(sfm_dir_dslr, cam_type='dslr') + dslr_paths = { + "in_colmap": sfm_dir_dslr, + "in_rgb": rgb_dir_dslr, + "in_mask": mask_dir_dslr, + } + + img_idx_iphone, img_infos_iphone, points3D_iphone, observations_iphone = load_sfm( + sfm_dir_iphone, cam_type='iphone') + iphone_paths = { + "in_colmap": sfm_dir_iphone, + "in_rgb": rgb_dir_iphone, + "in_mask": mask_dir_iphone, + } + + mesh = pyrender.Mesh.from_trimesh(mesh_scene, smooth=False) + pyrender_scene = pyrender.Scene() + pyrender_scene.add(mesh) + + selection_dslr = [imgname + '.JPG' for imgname in selection if imgname.startswith('DSC')] + selection_iphone = [imgname + '.jpg' for imgname in selection if imgname.startswith('frame_')] + + # resize the image to a more manageable size and render depth + for selection_cam, img_idx, img_infos, paths_data in [(selection_dslr, img_idx_dslr, img_infos_dslr, dslr_paths), + (selection_iphone, img_idx_iphone, img_infos_iphone, iphone_paths)]: + rgb_dir = paths_data['in_rgb'] + mask_dir = paths_data['in_mask'] + for imgname in tqdm(selection_cam, position=1, leave=False): + imgidx = img_idx[imgname] + img_infos_idx = img_infos[imgidx] + rgb = np.array(Image.open(os.path.join(rgb_dir, img_infos_idx['path']))) + mask = np.array(Image.open(os.path.join(mask_dir, img_infos_idx['path'][:-3] + 'png'))) + + _, _, K, rgb, mask = undistort_images(img_infos_idx['intrinsics'], rgb, mask) + + # rescale_image_depthmap assumes opencv intrinsics + intrinsics = geometry.colmap_to_opencv_intrinsics(K) + image, mask, intrinsics = rescale_image_depthmap( + rgb, mask, intrinsics, (target_resolution, target_resolution * 3.0 / 4)) + + W, H = image.size + intrinsics = geometry.opencv_to_colmap_intrinsics(intrinsics) + + # update inpace img_infos_idx + img_infos_idx['intrinsics'] = intrinsics + rgb_outpath = os.path.join(output_dir_scene_rgb, img_infos_idx['path'][:-3] + 'jpg') + image.save(rgb_outpath) + + depth_outpath = os.path.join(output_dir_scene_depth, img_infos_idx['path'][:-3] + 'png') + # render depth image + renderer.viewport_width, renderer.viewport_height = W, H + fx, fy, cx, cy = intrinsics[0, 0], intrinsics[1, 1], intrinsics[0, 2], intrinsics[1, 2] + camera = pyrender.camera.IntrinsicsCamera(fx, fy, cx, cy, znear=znear, zfar=zfar) + camera_node = pyrender_scene.add(camera, pose=img_infos_idx['cam_to_world'] @ OPENGL_TO_OPENCV) + + depth = renderer.render(pyrender_scene, flags=pyrender.RenderFlags.DEPTH_ONLY) + pyrender_scene.remove_node(camera_node) # dont forget to remove camera + + depth = (depth * 1000).astype('uint16') + # invalidate depth from mask before saving + depth_mask = (mask < 255) + depth[depth_mask] = 0 + Image.fromarray(depth).save(depth_outpath) + + trajectories = [] + intrinsics = [] + for imgname in selection: + if imgname.startswith('DSC'): + imgidx = img_idx_dslr[imgname + '.JPG'] + img_infos_idx = img_infos_dslr[imgidx] + elif imgname.startswith('frame_'): + imgidx = img_idx_iphone[imgname + '.jpg'] + img_infos_idx = img_infos_iphone[imgidx] + else: + raise ValueError('invalid image name') + + intrinsics.append(img_infos_idx['intrinsics']) + trajectories.append(img_infos_idx['cam_to_world']) + + intrinsics = np.stack(intrinsics, axis=0) + trajectories = np.stack(trajectories, axis=0) + # save metadata for this scene + np.savez(scene_metadata_path, + trajectories=trajectories, + intrinsics=intrinsics, + images=selection, + pairs=pairs) + + del img_infos + del pyrender_scene + + # concat all scene_metadata.npz into a single file + scene_data = {} + for scene_subdir in scenes: + scene_metadata_path = osp.join(output_dir, scene_subdir, 'scene_metadata.npz') + with np.load(scene_metadata_path) as data: + trajectories = data['trajectories'] + intrinsics = data['intrinsics'] + images = data['images'] + pairs = data['pairs'] + scene_data[scene_subdir] = {'trajectories': trajectories, + 'intrinsics': intrinsics, + 'images': images, + 'pairs': pairs} + + offset = 0 + counts = [] + scenes = [] + sceneids = [] + images = [] + intrinsics = [] + trajectories = [] + pairs = [] + for scene_idx, (scene_subdir, data) in enumerate(scene_data.items()): + num_imgs = data['images'].shape[0] + img_pairs = data['pairs'] + + scenes.append(scene_subdir) + sceneids.extend([scene_idx] * num_imgs) + + images.append(data['images']) + + intrinsics.append(data['intrinsics']) + trajectories.append(data['trajectories']) + + # offset pairs + img_pairs[:, 0:2] += offset + pairs.append(img_pairs) + counts.append(offset) + + offset += num_imgs + + images = np.concatenate(images, axis=0) + intrinsics = np.concatenate(intrinsics, axis=0) + trajectories = np.concatenate(trajectories, axis=0) + pairs = np.concatenate(pairs, axis=0) + np.savez(osp.join(output_dir, 'all_metadata.npz'), + counts=counts, + scenes=scenes, + sceneids=sceneids, + images=images, + intrinsics=intrinsics, + trajectories=trajectories, + pairs=pairs) + print('all done') + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + if args.pyopengl_platform.strip(): + os.environ['PYOPENGL_PLATFORM'] = args.pyopengl_platform + process_scenes(args.scannetpp_dir, args.precomputed_pairs, args.output_dir, args.target_resolution) diff --git a/submodules/mast3r/dust3r/datasets_preprocess/preprocess_staticthings3d.py b/submodules/mast3r/dust3r/datasets_preprocess/preprocess_staticthings3d.py new file mode 100644 index 0000000000000000000000000000000000000000..ee3eec16321c14b12291699f1fee492b5a7d8b1c --- /dev/null +++ b/submodules/mast3r/dust3r/datasets_preprocess/preprocess_staticthings3d.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Preprocessing code for the StaticThings3D dataset +# dataset at https://github.com/lmb-freiburg/robustmvd/blob/master/rmvd/data/README.md#staticthings3d +# 1) Download StaticThings3D in /path/to/StaticThings3D/ +# with the script at https://github.com/lmb-freiburg/robustmvd/blob/master/rmvd/data/scripts/download_staticthings3d.sh +# --> depths.tar.bz2 frames_finalpass.tar.bz2 poses.tar.bz2 frames_cleanpass.tar.bz2 intrinsics.tar.bz2 +# 2) unzip everything in the same /path/to/StaticThings3D/ directory +# 5) python datasets_preprocess/preprocess_staticthings3d.py --StaticThings3D_dir /path/to/tmp/StaticThings3D/ +# -------------------------------------------------------- +import os +import os.path as osp +import re +from tqdm import tqdm +import numpy as np +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 + +import path_to_root # noqa +from dust3r.utils.parallel import parallel_threads +from dust3r.datasets.utils import cropping # noqa + + +def get_parser(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--StaticThings3D_dir', required=True) + parser.add_argument('--precomputed_pairs', required=True) + parser.add_argument('--output_dir', default='data/staticthings3d_processed') + return parser + + +def main(db_root, pairs_path, output_dir): + all_scenes = _list_all_scenes(db_root) + + # crop images + args = [(db_root, osp.join(split, subsplit, seq), camera, f'{n:04d}', output_dir) + for split, subsplit, seq in all_scenes for camera in ['left', 'right'] for n in range(6, 16)] + parallel_threads(load_crop_and_save, args, star_args=True, front_num=1) + + # verify that all images are there + CAM = {b'l': 'left', b'r': 'right'} + pairs = np.load(pairs_path) + for scene, seq, cam1, im1, cam2, im2 in tqdm(pairs): + seq_path = osp.join('TRAIN', scene.decode('ascii'), f'{seq:04d}') + for cam, idx in [(CAM[cam1], im1), (CAM[cam2], im2)]: + for ext in ['clean', 'final']: + impath = osp.join(output_dir, seq_path, cam, f"{idx:04n}_{ext}.jpg") + assert osp.isfile(impath), f'missing an image at {impath=}' + + print(f'>> Saved all data to {output_dir}!') + + +def load_crop_and_save(db_root, relpath_, camera, num, out_dir): + relpath = osp.join(relpath_, camera, num) + if osp.isfile(osp.join(out_dir, relpath + '.npz')): + return + os.makedirs(osp.join(out_dir, relpath_, camera), exist_ok=True) + + # load everything + intrinsics_in = readFloat(osp.join(db_root, 'intrinsics', relpath_, num + '.float3')) + cam2world = np.linalg.inv(readFloat(osp.join(db_root, 'poses', relpath + '.float3'))) + depthmap_in = readFloat(osp.join(db_root, 'depths', relpath + '.float3')) + img_clean = cv2.cvtColor(cv2.imread(osp.join(db_root, 'frames_cleanpass', + relpath + '.png'), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB) + img_final = cv2.cvtColor(cv2.imread(osp.join(db_root, 'frames_finalpass', + relpath + '.png'), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB) + + # do the crop + assert img_clean.shape[:2] == (540, 960) + assert img_final.shape[:2] == (540, 960) + (clean_out, final_out), depthmap, intrinsics_out, R_in2out = _crop_image( + intrinsics_in, (img_clean, img_final), depthmap_in, (512, 384)) + + # write everything + clean_out.save(osp.join(out_dir, relpath + '_clean.jpg'), quality=80) + final_out.save(osp.join(out_dir, relpath + '_final.jpg'), quality=80) + cv2.imwrite(osp.join(out_dir, relpath + '.exr'), depthmap) + + # New camera parameters + cam2world[:3, :3] = cam2world[:3, :3] @ R_in2out.T + np.savez(osp.join(out_dir, relpath + '.npz'), intrinsics=intrinsics_out, cam2world=cam2world) + + +def _crop_image(intrinsics_in, color_image_in, depthmap_in, resolution_out=(512, 512)): + image, depthmap, intrinsics_out = cropping.rescale_image_depthmap( + color_image_in, depthmap_in, intrinsics_in, resolution_out) + R_in2out = np.eye(3) + return image, depthmap, intrinsics_out, R_in2out + + +def _list_all_scenes(path): + print('>> Listing all scenes') + + res = [] + for split in ['TRAIN']: + for subsplit in 'ABC': + for seq in os.listdir(osp.join(path, 'intrinsics', split, subsplit)): + res.append((split, subsplit, seq)) + print(f' (found ({len(res)}) scenes)') + assert res, f'Did not find anything at {path=}' + return res + + +def readFloat(name): + with open(name, 'rb') as f: + if (f.readline().decode("utf-8")) != 'float\n': + raise Exception('float file %s did not contain keyword' % name) + + dim = int(f.readline()) + + dims = [] + count = 1 + for i in range(0, dim): + d = int(f.readline()) + dims.append(d) + count *= d + + dims = list(reversed(dims)) + data = np.fromfile(f, np.float32, count).reshape(dims) + return data # Hxw or CxHxW NxCxHxW + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + main(args.StaticThings3D_dir, args.precomputed_pairs, args.output_dir) diff --git a/submodules/mast3r/dust3r/datasets_preprocess/preprocess_waymo.py b/submodules/mast3r/dust3r/datasets_preprocess/preprocess_waymo.py new file mode 100644 index 0000000000000000000000000000000000000000..203f337330a7e06e61d2fb9dd99647063967922d --- /dev/null +++ b/submodules/mast3r/dust3r/datasets_preprocess/preprocess_waymo.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Preprocessing code for the WayMo Open dataset +# dataset at https://github.com/waymo-research/waymo-open-dataset +# 1) Accept the license +# 2) download all training/*.tfrecord files from Perception Dataset, version 1.4.2 +# 3) put all .tfrecord files in '/path/to/waymo_dir' +# 4) install the waymo_open_dataset package with +# `python3 -m pip install gcsfs waymo-open-dataset-tf-2-12-0==1.6.4` +# 5) execute this script as `python preprocess_waymo.py --waymo_dir /path/to/waymo_dir` +# -------------------------------------------------------- +import sys +import os +import os.path as osp +import shutil +import json +from tqdm import tqdm +import PIL.Image +import numpy as np +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 + +import tensorflow.compat.v1 as tf +tf.enable_eager_execution() + +import path_to_root # noqa +from dust3r.utils.geometry import geotrf, inv +from dust3r.utils.image import imread_cv2 +from dust3r.utils.parallel import parallel_processes as parallel_map +from dust3r.datasets.utils import cropping +from dust3r.viz import show_raw_pointcloud + + +def get_parser(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--waymo_dir', required=True) + parser.add_argument('--precomputed_pairs', required=True) + parser.add_argument('--output_dir', default='data/waymo_processed') + parser.add_argument('--workers', type=int, default=1) + return parser + + +def main(waymo_root, pairs_path, output_dir, workers=1): + extract_frames(waymo_root, output_dir, workers=workers) + make_crops(output_dir, workers=args.workers) + + # make sure all pairs are there + with np.load(pairs_path) as data: + scenes = data['scenes'] + frames = data['frames'] + pairs = data['pairs'] # (array of (scene_id, img1_id, img2_id) + + for scene_id, im1_id, im2_id in pairs: + for im_id in (im1_id, im2_id): + path = osp.join(output_dir, scenes[scene_id], frames[im_id] + '.jpg') + assert osp.isfile(path), f'Missing a file at {path=}\nDid you download all .tfrecord files?' + + shutil.rmtree(osp.join(output_dir, 'tmp')) + print('Done! all data generated at', output_dir) + + +def _list_sequences(db_root): + print('>> Looking for sequences in', db_root) + res = sorted(f for f in os.listdir(db_root) if f.endswith('.tfrecord')) + print(f' found {len(res)} sequences') + return res + + +def extract_frames(db_root, output_dir, workers=8): + sequences = _list_sequences(db_root) + output_dir = osp.join(output_dir, 'tmp') + print('>> outputing result to', output_dir) + args = [(db_root, output_dir, seq) for seq in sequences] + parallel_map(process_one_seq, args, star_args=True, workers=workers) + + +def process_one_seq(db_root, output_dir, seq): + out_dir = osp.join(output_dir, seq) + os.makedirs(out_dir, exist_ok=True) + calib_path = osp.join(out_dir, 'calib.json') + if osp.isfile(calib_path): + return + + try: + with tf.device('/CPU:0'): + calib, frames = extract_frames_one_seq(osp.join(db_root, seq)) + except RuntimeError: + print(f'/!\\ Error with sequence {seq} /!\\', file=sys.stderr) + return # nothing is saved + + for f, (frame_name, views) in enumerate(tqdm(frames, leave=False)): + for cam_idx, view in views.items(): + img = PIL.Image.fromarray(view.pop('img')) + img.save(osp.join(out_dir, f'{f:05d}_{cam_idx}.jpg')) + np.savez(osp.join(out_dir, f'{f:05d}_{cam_idx}.npz'), **view) + + with open(calib_path, 'w') as f: + json.dump(calib, f) + + +def extract_frames_one_seq(filename): + from waymo_open_dataset import dataset_pb2 as open_dataset + from waymo_open_dataset.utils import frame_utils + + print('>> Opening', filename) + dataset = tf.data.TFRecordDataset(filename, compression_type='') + + calib = None + frames = [] + + for data in tqdm(dataset, leave=False): + frame = open_dataset.Frame() + frame.ParseFromString(bytearray(data.numpy())) + + content = frame_utils.parse_range_image_and_camera_projection(frame) + range_images, camera_projections, _, range_image_top_pose = content + + views = {} + frames.append((frame.context.name, views)) + + # once in a sequence, read camera calibration info + if calib is None: + calib = [] + for cam in frame.context.camera_calibrations: + calib.append((cam.name, + dict(width=cam.width, + height=cam.height, + intrinsics=list(cam.intrinsic), + extrinsics=list(cam.extrinsic.transform)))) + + # convert LIDAR to pointcloud + points, cp_points = frame_utils.convert_range_image_to_point_cloud( + frame, + range_images, + camera_projections, + range_image_top_pose) + + # 3d points in vehicle frame. + points_all = np.concatenate(points, axis=0) + cp_points_all = np.concatenate(cp_points, axis=0) + + # The distance between lidar points and vehicle frame origin. + cp_points_all_tensor = tf.constant(cp_points_all, dtype=tf.int32) + + for i, image in enumerate(frame.images): + # select relevant 3D points for this view + mask = tf.equal(cp_points_all_tensor[..., 0], image.name) + cp_points_msk_tensor = tf.cast(tf.gather_nd(cp_points_all_tensor, tf.where(mask)), dtype=tf.float32) + + pose = np.asarray(image.pose.transform).reshape(4, 4) + timestamp = image.pose_timestamp + + rgb = tf.image.decode_jpeg(image.image).numpy() + + pix = cp_points_msk_tensor[..., 1:3].numpy().round().astype(np.int16) + pts3d = points_all[mask.numpy()] + + views[image.name] = dict(img=rgb, pose=pose, pixels=pix, pts3d=pts3d, timestamp=timestamp) + + if not 'show full point cloud': + show_raw_pointcloud([v['pts3d'] for v in views.values()], [v['img'] for v in views.values()]) + + return calib, frames + + +def make_crops(output_dir, workers=16, **kw): + tmp_dir = osp.join(output_dir, 'tmp') + sequences = _list_sequences(tmp_dir) + args = [(tmp_dir, output_dir, seq) for seq in sequences] + parallel_map(crop_one_seq, args, star_args=True, workers=workers, front_num=0) + + +def crop_one_seq(input_dir, output_dir, seq, resolution=512): + seq_dir = osp.join(input_dir, seq) + out_dir = osp.join(output_dir, seq) + if osp.isfile(osp.join(out_dir, '00100_1.jpg')): + return + os.makedirs(out_dir, exist_ok=True) + + # load calibration file + try: + with open(osp.join(seq_dir, 'calib.json')) as f: + calib = json.load(f) + except IOError: + print(f'/!\\ Error: Missing calib.json in sequence {seq} /!\\', file=sys.stderr) + return + + axes_transformation = np.array([ + [0, -1, 0, 0], + [0, 0, -1, 0], + [1, 0, 0, 0], + [0, 0, 0, 1]]) + + cam_K = {} + cam_distortion = {} + cam_res = {} + cam_to_car = {} + for cam_idx, cam_info in calib: + cam_idx = str(cam_idx) + cam_res[cam_idx] = (W, H) = (cam_info['width'], cam_info['height']) + f1, f2, cx, cy, k1, k2, p1, p2, k3 = cam_info['intrinsics'] + cam_K[cam_idx] = np.asarray([(f1, 0, cx), (0, f2, cy), (0, 0, 1)]) + cam_distortion[cam_idx] = np.asarray([k1, k2, p1, p2, k3]) + cam_to_car[cam_idx] = np.asarray(cam_info['extrinsics']).reshape(4, 4) # cam-to-vehicle + + frames = sorted(f[:-3] for f in os.listdir(seq_dir) if f.endswith('.jpg')) + + # from dust3r.viz import SceneViz + # viz = SceneViz() + + for frame in tqdm(frames, leave=False): + cam_idx = frame[-2] # cam index + assert cam_idx in '12345', f'bad {cam_idx=} in {frame=}' + data = np.load(osp.join(seq_dir, frame + 'npz')) + car_to_world = data['pose'] + W, H = cam_res[cam_idx] + + # load depthmap + pos2d = data['pixels'].round().astype(np.uint16) + x, y = pos2d.T + pts3d = data['pts3d'] # already in the car frame + pts3d = geotrf(axes_transformation @ inv(cam_to_car[cam_idx]), pts3d) + # X=LEFT_RIGHT y=ALTITUDE z=DEPTH + + # load image + image = imread_cv2(osp.join(seq_dir, frame + 'jpg')) + + # downscale image + output_resolution = (resolution, 1) if W > H else (1, resolution) + image, _, intrinsics2 = cropping.rescale_image_depthmap(image, None, cam_K[cam_idx], output_resolution) + image.save(osp.join(out_dir, frame + 'jpg'), quality=80) + + # save as an EXR file? yes it's smaller (and easier to load) + W, H = image.size + depthmap = np.zeros((H, W), dtype=np.float32) + pos2d = geotrf(intrinsics2 @ inv(cam_K[cam_idx]), pos2d).round().astype(np.int16) + x, y = pos2d.T + depthmap[y.clip(min=0, max=H - 1), x.clip(min=0, max=W - 1)] = pts3d[:, 2] + cv2.imwrite(osp.join(out_dir, frame + 'exr'), depthmap) + + # save camera parametes + cam2world = car_to_world @ cam_to_car[cam_idx] @ inv(axes_transformation) + np.savez(osp.join(out_dir, frame + 'npz'), intrinsics=intrinsics2, + cam2world=cam2world, distortion=cam_distortion[cam_idx]) + + # viz.add_rgbd(np.asarray(image), depthmap, intrinsics2, cam2world) + # viz.show() + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + main(args.waymo_dir, args.precomputed_pairs, args.output_dir, workers=args.workers) diff --git a/submodules/mast3r/dust3r/datasets_preprocess/preprocess_wildrgbd.py b/submodules/mast3r/dust3r/datasets_preprocess/preprocess_wildrgbd.py new file mode 100644 index 0000000000000000000000000000000000000000..ff3f0f7abb7d9ef43bba6a7c6cd6f4e652a8f510 --- /dev/null +++ b/submodules/mast3r/dust3r/datasets_preprocess/preprocess_wildrgbd.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Script to pre-process the WildRGB-D dataset. +# Usage: +# python3 datasets_preprocess/preprocess_wildrgbd.py --wildrgbd_dir /path/to/wildrgbd +# -------------------------------------------------------- + +import argparse +import random +import json +import os +import os.path as osp + +import PIL.Image +import numpy as np +import cv2 + +from tqdm.auto import tqdm +import matplotlib.pyplot as plt + +import path_to_root # noqa +import dust3r.datasets.utils.cropping as cropping # noqa +from dust3r.utils.image import imread_cv2 + + +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("--output_dir", type=str, default="data/wildrgbd_processed") + parser.add_argument("--wildrgbd_dir", type=str, required=True) + parser.add_argument("--train_num_sequences_per_object", type=int, default=50) + parser.add_argument("--test_num_sequences_per_object", type=int, default=10) + parser.add_argument("--num_frames", type=int, default=100) + parser.add_argument("--seed", type=int, default=42) + + parser.add_argument("--img_size", type=int, default=512, + help=("lower dimension will be >= img_size * 3/4, and max dimension will be >= img_size")) + return parser + + +def get_set_list(category_dir, split): + listfiles = ["camera_eval_list.json", "nvs_list.json"] + + sequences_all = {s: {k: set() for k in listfiles} for s in ['train', 'val']} + for listfile in listfiles: + with open(osp.join(category_dir, listfile)) as f: + subset_lists_data = json.load(f) + for s in ['train', 'val']: + sequences_all[s][listfile].update(subset_lists_data[s]) + train_intersection = set.intersection(*list(sequences_all['train'].values())) + if split == "train": + return train_intersection + else: + all_seqs = set.union(*list(sequences_all['train'].values()), *list(sequences_all['val'].values())) + return all_seqs.difference(train_intersection) + + +def prepare_sequences(category, wildrgbd_dir, output_dir, img_size, split, max_num_sequences_per_object, + output_num_frames, seed): + random.seed(seed) + category_dir = osp.join(wildrgbd_dir, category) + category_output_dir = osp.join(output_dir, category) + sequences_all = get_set_list(category_dir, split) + sequences_all = sorted(sequences_all) + + sequences_all_tmp = [] + for seq_name in sequences_all: + scene_dir = osp.join(wildrgbd_dir, category_dir, seq_name) + if not os.path.isdir(scene_dir): + print(f'{scene_dir} does not exist, skipped') + continue + sequences_all_tmp.append(seq_name) + sequences_all = sequences_all_tmp + if len(sequences_all) <= max_num_sequences_per_object: + selected_sequences = sequences_all + else: + selected_sequences = random.sample(sequences_all, max_num_sequences_per_object) + + selected_sequences_numbers_dict = {} + for seq_name in tqdm(selected_sequences, leave=False): + scene_dir = osp.join(category_dir, seq_name) + scene_output_dir = osp.join(category_output_dir, seq_name) + with open(osp.join(scene_dir, 'metadata'), 'r') as f: + metadata = json.load(f) + + K = np.array(metadata["K"]).reshape(3, 3).T + fx, fy, cx, cy = K[0, 0], K[1, 1], K[0, 2], K[1, 2] + w, h = metadata["w"], metadata["h"] + + camera_intrinsics = np.array( + [[fx, 0, cx], + [0, fy, cy], + [0, 0, 1]] + ) + camera_to_world_path = os.path.join(scene_dir, 'cam_poses.txt') + camera_to_world_content = np.genfromtxt(camera_to_world_path) + camera_to_world = camera_to_world_content[:, 1:].reshape(-1, 4, 4) + + frame_idx = camera_to_world_content[:, 0] + num_frames = frame_idx.shape[0] + assert num_frames >= output_num_frames + assert np.all(frame_idx == np.arange(num_frames)) + + # selected_sequences_numbers_dict[seq_name] = num_frames + + selected_frames = np.round(np.linspace(0, num_frames - 1, output_num_frames)).astype(int).tolist() + selected_sequences_numbers_dict[seq_name] = selected_frames + + for frame_id in tqdm(selected_frames): + depth_path = os.path.join(scene_dir, 'depth', f'{frame_id:0>5d}.png') + masks_path = os.path.join(scene_dir, 'masks', f'{frame_id:0>5d}.png') + rgb_path = os.path.join(scene_dir, 'rgb', f'{frame_id:0>5d}.png') + + input_rgb_image = PIL.Image.open(rgb_path).convert('RGB') + input_mask = plt.imread(masks_path) + input_depthmap = imread_cv2(depth_path, cv2.IMREAD_UNCHANGED).astype(np.float64) + depth_mask = np.stack((input_depthmap, input_mask), axis=-1) + H, W = input_depthmap.shape + + min_margin_x = min(cx, W - cx) + min_margin_y = min(cy, H - cy) + + # the new window will be a rectangle of size (2*min_margin_x, 2*min_margin_y) centered on (cx,cy) + l, t = int(cx - min_margin_x), int(cy - min_margin_y) + r, b = int(cx + min_margin_x), int(cy + min_margin_y) + crop_bbox = (l, t, r, b) + input_rgb_image, depth_mask, input_camera_intrinsics = cropping.crop_image_depthmap( + input_rgb_image, depth_mask, camera_intrinsics, crop_bbox) + + # try to set the lower dimension to img_size * 3/4 -> img_size=512 => 384 + scale_final = ((img_size * 3 // 4) / min(H, W)) + 1e-8 + output_resolution = np.floor(np.array([W, H]) * scale_final).astype(int) + if max(output_resolution) < img_size: + # let's put the max dimension to img_size + scale_final = (img_size / max(H, W)) + 1e-8 + output_resolution = np.floor(np.array([W, H]) * scale_final).astype(int) + + input_rgb_image, depth_mask, input_camera_intrinsics = cropping.rescale_image_depthmap( + input_rgb_image, depth_mask, input_camera_intrinsics, output_resolution) + input_depthmap = depth_mask[:, :, 0] + input_mask = depth_mask[:, :, 1] + + camera_pose = camera_to_world[frame_id] + + # save crop images and depth, metadata + save_img_path = os.path.join(scene_output_dir, 'rgb', f'{frame_id:0>5d}.jpg') + save_depth_path = os.path.join(scene_output_dir, 'depth', f'{frame_id:0>5d}.png') + save_mask_path = os.path.join(scene_output_dir, 'masks', f'{frame_id:0>5d}.png') + os.makedirs(os.path.split(save_img_path)[0], exist_ok=True) + os.makedirs(os.path.split(save_depth_path)[0], exist_ok=True) + os.makedirs(os.path.split(save_mask_path)[0], exist_ok=True) + + input_rgb_image.save(save_img_path) + cv2.imwrite(save_depth_path, input_depthmap.astype(np.uint16)) + cv2.imwrite(save_mask_path, (input_mask * 255).astype(np.uint8)) + + save_meta_path = os.path.join(scene_output_dir, 'metadata', f'{frame_id:0>5d}.npz') + os.makedirs(os.path.split(save_meta_path)[0], exist_ok=True) + np.savez(save_meta_path, camera_intrinsics=input_camera_intrinsics, + camera_pose=camera_pose) + + return selected_sequences_numbers_dict + + +if __name__ == "__main__": + parser = get_parser() + args = parser.parse_args() + assert args.wildrgbd_dir != args.output_dir + + categories = sorted([ + dirname for dirname in os.listdir(args.wildrgbd_dir) + if os.path.isdir(os.path.join(args.wildrgbd_dir, dirname, 'scenes')) + ]) + + os.makedirs(args.output_dir, exist_ok=True) + + splits_num_sequences_per_object = [args.train_num_sequences_per_object, args.test_num_sequences_per_object] + for split, num_sequences_per_object in zip(['train', 'test'], splits_num_sequences_per_object): + selected_sequences_path = os.path.join(args.output_dir, f'selected_seqs_{split}.json') + if os.path.isfile(selected_sequences_path): + continue + all_selected_sequences = {} + for category in categories: + category_output_dir = osp.join(args.output_dir, category) + os.makedirs(category_output_dir, exist_ok=True) + category_selected_sequences_path = os.path.join(category_output_dir, f'selected_seqs_{split}.json') + if os.path.isfile(category_selected_sequences_path): + with open(category_selected_sequences_path, 'r') as fid: + category_selected_sequences = json.load(fid) + else: + print(f"Processing {split} - category = {category}") + category_selected_sequences = prepare_sequences( + category=category, + wildrgbd_dir=args.wildrgbd_dir, + output_dir=args.output_dir, + img_size=args.img_size, + split=split, + max_num_sequences_per_object=num_sequences_per_object, + output_num_frames=args.num_frames, + seed=args.seed + int("category".encode('ascii').hex(), 16), + ) + with open(category_selected_sequences_path, 'w') as file: + json.dump(category_selected_sequences, file) + + all_selected_sequences[category] = category_selected_sequences + with open(selected_sequences_path, 'w') as file: + json.dump(all_selected_sequences, file) diff --git a/submodules/mast3r/dust3r/demo.py b/submodules/mast3r/dust3r/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..326c6e5a49d5d352b4afb5445cee5d22571c3bdd --- /dev/null +++ b/submodules/mast3r/dust3r/demo.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# dust3r gradio demo executable +# -------------------------------------------------------- +import os +import torch +import tempfile + +from dust3r.model import AsymmetricCroCo3DStereo +from dust3r.demo import get_args_parser, main_demo, set_print_with_timestamp + +import matplotlib.pyplot as pl +pl.ion() + +torch.backends.cuda.matmul.allow_tf32 = True # for gpu >= Ampere and pytorch >= 1.12 + +if __name__ == '__main__': + parser = get_args_parser() + args = parser.parse_args() + set_print_with_timestamp() + + if args.tmp_dir is not None: + tmp_path = args.tmp_dir + os.makedirs(tmp_path, exist_ok=True) + tempfile.tempdir = tmp_path + + if args.server_name is not None: + server_name = args.server_name + else: + server_name = '0.0.0.0' if args.local_network else '127.0.0.1' + + if args.weights is not None: + weights_path = args.weights + else: + weights_path = "naver/" + args.model_name + model = AsymmetricCroCo3DStereo.from_pretrained(weights_path).to(args.device) + + # dust3r will write the 3D model inside tmpdirname + with tempfile.TemporaryDirectory(suffix='dust3r_gradio_demo') as tmpdirname: + if not args.silent: + print('Outputing stuff in', tmpdirname) + main_demo(tmpdirname, model, args.device, args.image_size, server_name, args.server_port, silent=args.silent) diff --git a/submodules/mast3r/dust3r/docker/docker-compose-cpu.yml b/submodules/mast3r/dust3r/docker/docker-compose-cpu.yml new file mode 100644 index 0000000000000000000000000000000000000000..2015fd771e8b6246d288c03a38f6fbb3f17dff20 --- /dev/null +++ b/submodules/mast3r/dust3r/docker/docker-compose-cpu.yml @@ -0,0 +1,16 @@ +version: '3.8' +services: + dust3r-demo: + build: + context: ./files + dockerfile: cpu.Dockerfile + ports: + - "7860:7860" + volumes: + - ./files/checkpoints:/dust3r/checkpoints + environment: + - DEVICE=cpu + - MODEL=${MODEL:-DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth} + cap_add: + - IPC_LOCK + - SYS_RESOURCE diff --git a/submodules/mast3r/dust3r/docker/docker-compose-cuda.yml b/submodules/mast3r/dust3r/docker/docker-compose-cuda.yml new file mode 100644 index 0000000000000000000000000000000000000000..85710af953d669fe618273de6ce3a062a7a84cca --- /dev/null +++ b/submodules/mast3r/dust3r/docker/docker-compose-cuda.yml @@ -0,0 +1,23 @@ +version: '3.8' +services: + dust3r-demo: + build: + context: ./files + dockerfile: cuda.Dockerfile + ports: + - "7860:7860" + environment: + - DEVICE=cuda + - MODEL=${MODEL:-DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth} + volumes: + - ./files/checkpoints:/dust3r/checkpoints + cap_add: + - IPC_LOCK + - SYS_RESOURCE + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] diff --git a/submodules/mast3r/dust3r/docker/files/cpu.Dockerfile b/submodules/mast3r/dust3r/docker/files/cpu.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..c9ccc39682dd7c7723f447ff47f12531a593446f --- /dev/null +++ b/submodules/mast3r/dust3r/docker/files/cpu.Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.11-slim + +LABEL description="Docker container for DUSt3R with dependencies installed. CPU VERSION" + +ENV DEVICE="cpu" +ENV MODEL="DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth" +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + git \ + libgl1-mesa-glx \ + libegl1-mesa \ + libxrandr2 \ + libxrandr2 \ + libxss1 \ + libxcursor1 \ + libxcomposite1 \ + libasound2 \ + libxi6 \ + libxtst6 \ + libglib2.0-0 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN git clone --recursive https://github.com/naver/dust3r /dust3r +WORKDIR /dust3r + +RUN pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu +RUN pip install -r requirements.txt +RUN pip install -r requirements_optional.txt +RUN pip install opencv-python==4.8.0.74 + +WORKDIR /dust3r + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/submodules/mast3r/dust3r/docker/files/cuda.Dockerfile b/submodules/mast3r/dust3r/docker/files/cuda.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..a1d2edce1a5e7cee2fa3d66faf4f6ee019595267 --- /dev/null +++ b/submodules/mast3r/dust3r/docker/files/cuda.Dockerfile @@ -0,0 +1,27 @@ +FROM nvcr.io/nvidia/pytorch:24.01-py3 + +LABEL description="Docker container for DUSt3R with dependencies installed. CUDA VERSION" +ENV DEVICE="cuda" +ENV MODEL="DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth" +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + git=1:2.34.1-1ubuntu1.10 \ + libglib2.0-0=2.72.4-0ubuntu2.2 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN git clone --recursive https://github.com/naver/dust3r /dust3r +WORKDIR /dust3r +RUN pip install -r requirements.txt +RUN pip install -r requirements_optional.txt +RUN pip install opencv-python==4.8.0.74 + +WORKDIR /dust3r/croco/models/curope/ +RUN python setup.py build_ext --inplace + +WORKDIR /dust3r +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/submodules/mast3r/dust3r/docker/files/entrypoint.sh b/submodules/mast3r/dust3r/docker/files/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..9637072a0af071f927ca0481bcaa4b600644b8b5 --- /dev/null +++ b/submodules/mast3r/dust3r/docker/files/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eux + +DEVICE=${DEVICE:-cuda} +MODEL=${MODEL:-DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth} + +exec python3 demo.py --weights "checkpoints/$MODEL" --device "$DEVICE" --local_network "$@" diff --git a/submodules/mast3r/dust3r/docker/run.sh b/submodules/mast3r/dust3r/docker/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..6c920363d607fc6019f10780d072edf49bee3046 --- /dev/null +++ b/submodules/mast3r/dust3r/docker/run.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +set -eux + +# Default model name +model_name="DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth" + +check_docker() { + if ! command -v docker &>/dev/null; then + echo "Docker could not be found. Please install Docker and try again." + exit 1 + fi +} + +download_model_checkpoint() { + if [ -f "./files/checkpoints/${model_name}" ]; then + echo "Model checkpoint ${model_name} already exists. Skipping download." + return + fi + echo "Downloading model checkpoint ${model_name}..." + wget "https://download.europe.naverlabs.com/ComputerVision/DUSt3R/${model_name}" -P ./files/checkpoints +} + +set_dcomp() { + if command -v docker-compose &>/dev/null; then + dcomp="docker-compose" + elif command -v docker &>/dev/null && docker compose version &>/dev/null; then + dcomp="docker compose" + else + echo "Docker Compose could not be found. Please install Docker Compose and try again." + exit 1 + fi +} + +run_docker() { + export MODEL=${model_name} + if [ "$with_cuda" -eq 1 ]; then + $dcomp -f docker-compose-cuda.yml up --build + else + $dcomp -f docker-compose-cpu.yml up --build + fi +} + +with_cuda=0 +for arg in "$@"; do + case $arg in + --with-cuda) + with_cuda=1 + ;; + --model_name=*) + model_name="${arg#*=}.pth" + ;; + *) + echo "Unknown parameter passed: $arg" + exit 1 + ;; + esac +done + + +main() { + check_docker + download_model_checkpoint + set_dcomp + run_docker +} + +main diff --git a/submodules/mast3r/dust3r/dust3r/__init__.py b/submodules/mast3r/dust3r/dust3r/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a32692113d830ddc4af4e6ed608f222fbe062e6e --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). diff --git a/submodules/mast3r/dust3r/dust3r/cloud_opt/__init__.py b/submodules/mast3r/dust3r/dust3r/cloud_opt/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..faf5cd279a317c1efb9ba947682992c0949c1bdc --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/cloud_opt/__init__.py @@ -0,0 +1,33 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# global alignment optimization wrapper function +# -------------------------------------------------------- +from enum import Enum + +from .optimizer import PointCloudOptimizer +from .modular_optimizer import ModularPointCloudOptimizer +from .pair_viewer import PairViewer + + +class GlobalAlignerMode(Enum): + PointCloudOptimizer = "PointCloudOptimizer" + ModularPointCloudOptimizer = "ModularPointCloudOptimizer" + PairViewer = "PairViewer" + + +def global_aligner(dust3r_output, device, mode=GlobalAlignerMode.PointCloudOptimizer, **optim_kw): + # extract all inputs + view1, view2, pred1, pred2 = [dust3r_output[k] for k in 'view1 view2 pred1 pred2'.split()] + # build the optimizer + if mode == GlobalAlignerMode.PointCloudOptimizer: + net = PointCloudOptimizer(view1, view2, pred1, pred2, **optim_kw).to(device) + elif mode == GlobalAlignerMode.ModularPointCloudOptimizer: + net = ModularPointCloudOptimizer(view1, view2, pred1, pred2, **optim_kw).to(device) + elif mode == GlobalAlignerMode.PairViewer: + net = PairViewer(view1, view2, pred1, pred2, **optim_kw).to(device) + else: + raise NotImplementedError(f'Unknown mode {mode}') + + return net diff --git a/submodules/mast3r/dust3r/dust3r/cloud_opt/base_opt.py b/submodules/mast3r/dust3r/dust3r/cloud_opt/base_opt.py new file mode 100644 index 0000000000000000000000000000000000000000..4d36e05bfca80509bced20add7c067987d538951 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/cloud_opt/base_opt.py @@ -0,0 +1,405 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Base class for the global alignement procedure +# -------------------------------------------------------- +from copy import deepcopy + +import numpy as np +import torch +import torch.nn as nn +import roma +from copy import deepcopy +import tqdm + +from dust3r.utils.geometry import inv, geotrf +from dust3r.utils.device import to_numpy +from dust3r.utils.image import rgb +from dust3r.viz import SceneViz, segment_sky, auto_cam_size +from dust3r.optim_factory import adjust_learning_rate_by_lr + +from dust3r.cloud_opt.commons import (edge_str, ALL_DISTS, NoGradParamDict, get_imshapes, signed_expm1, signed_log1p, + cosine_schedule, linear_schedule, get_conf_trf) +import dust3r.cloud_opt.init_im_poses as init_fun + + +class BasePCOptimizer (nn.Module): + """ Optimize a global scene, given a list of pairwise observations. + Graph node: images + Graph edges: observations = (pred1, pred2) + """ + + def __init__(self, *args, **kwargs): + if len(args) == 1 and len(kwargs) == 0: + other = deepcopy(args[0]) + attrs = '''edges is_symmetrized dist n_imgs pred_i pred_j imshapes + min_conf_thr conf_thr conf_i conf_j im_conf + base_scale norm_pw_scale POSE_DIM pw_poses + pw_adaptors pw_adaptors has_im_poses rand_pose imgs verbose'''.split() + self.__dict__.update({k: other[k] for k in attrs}) + else: + self._init_from_views(*args, **kwargs) + + def _init_from_views(self, view1, view2, pred1, pred2, + dist='l1', + conf='log', + min_conf_thr=3, + base_scale=0.5, + allow_pw_adaptors=False, + pw_break=20, + rand_pose=torch.randn, + iterationsCount=None, + verbose=True): + super().__init__() + if not isinstance(view1['idx'], list): + view1['idx'] = view1['idx'].tolist() + if not isinstance(view2['idx'], list): + view2['idx'] = view2['idx'].tolist() + self.edges = [(int(i), int(j)) for i, j in zip(view1['idx'], view2['idx'])] + self.is_symmetrized = set(self.edges) == {(j, i) for i, j in self.edges} + self.dist = ALL_DISTS[dist] + self.verbose = verbose + + self.n_imgs = self._check_edges() + + # input data + pred1_pts = pred1['pts3d'] + pred2_pts = pred2['pts3d_in_other_view'] + self.pred_i = NoGradParamDict({ij: pred1_pts[n] for n, ij in enumerate(self.str_edges)}) + self.pred_j = NoGradParamDict({ij: pred2_pts[n] for n, ij in enumerate(self.str_edges)}) + self.imshapes = get_imshapes(self.edges, pred1_pts, pred2_pts) + + # work in log-scale with conf + pred1_conf = pred1['conf'] + pred2_conf = pred2['conf'] + self.min_conf_thr = min_conf_thr + self.conf_trf = get_conf_trf(conf) + + self.conf_i = NoGradParamDict({ij: pred1_conf[n] for n, ij in enumerate(self.str_edges)}) + self.conf_j = NoGradParamDict({ij: pred2_conf[n] for n, ij in enumerate(self.str_edges)}) + self.im_conf = self._compute_img_conf(pred1_conf, pred2_conf) + for i in range(len(self.im_conf)): + self.im_conf[i].requires_grad = False + + # pairwise pose parameters + self.base_scale = base_scale + self.norm_pw_scale = True + self.pw_break = pw_break + self.POSE_DIM = 7 + self.pw_poses = nn.Parameter(rand_pose((self.n_edges, 1+self.POSE_DIM))) # pairwise poses + self.pw_adaptors = nn.Parameter(torch.zeros((self.n_edges, 2))) # slight xy/z adaptation + self.pw_adaptors.requires_grad_(allow_pw_adaptors) + self.has_im_poses = False + self.rand_pose = rand_pose + + # possibly store images for show_pointcloud + self.imgs = None + if 'img' in view1 and 'img' in view2: + imgs = [torch.zeros((3,)+hw) for hw in self.imshapes] + for v in range(len(self.edges)): + idx = view1['idx'][v] + imgs[idx] = view1['img'][v] + idx = view2['idx'][v] + imgs[idx] = view2['img'][v] + self.imgs = rgb(imgs) + + @property + def n_edges(self): + return len(self.edges) + + @property + def str_edges(self): + return [edge_str(i, j) for i, j in self.edges] + + @property + def imsizes(self): + return [(w, h) for h, w in self.imshapes] + + @property + def device(self): + return next(iter(self.parameters())).device + + def state_dict(self, trainable=True): + all_params = super().state_dict() + return {k: v for k, v in all_params.items() if k.startswith(('_', 'pred_i.', 'pred_j.', 'conf_i.', 'conf_j.')) != trainable} + + def load_state_dict(self, data): + return super().load_state_dict(self.state_dict(trainable=False) | data) + + def _check_edges(self): + indices = sorted({i for edge in self.edges for i in edge}) + assert indices == list(range(len(indices))), 'bad pair indices: missing values ' + return len(indices) + + @torch.no_grad() + def _compute_img_conf(self, pred1_conf, pred2_conf): + im_conf = nn.ParameterList([torch.zeros(hw, device=self.device) for hw in self.imshapes]) + for e, (i, j) in enumerate(self.edges): + im_conf[i] = torch.maximum(im_conf[i], pred1_conf[e]) + im_conf[j] = torch.maximum(im_conf[j], pred2_conf[e]) + return im_conf + + def get_adaptors(self): + adapt = self.pw_adaptors + adapt = torch.cat((adapt[:, 0:1], adapt), dim=-1) # (scale_xy, scale_xy, scale_z) + if self.norm_pw_scale: # normalize so that the product == 1 + adapt = adapt - adapt.mean(dim=1, keepdim=True) + return (adapt / self.pw_break).exp() + + def _get_poses(self, poses): + # normalize rotation + Q = poses[:, :4] + T = signed_expm1(poses[:, 4:7]) + RT = roma.RigidUnitQuat(Q, T).normalize().to_homogeneous() + return RT + + def _set_pose(self, poses, idx, R, T=None, scale=None, force=False): + # all poses == cam-to-world + pose = poses[idx] + if not (pose.requires_grad or force): + return pose + + if R.shape == (4, 4): + assert T is None + T = R[:3, 3] + R = R[:3, :3] + + if R is not None: + pose.data[0:4] = roma.rotmat_to_unitquat(R) + if T is not None: + pose.data[4:7] = signed_log1p(T / (scale or 1)) # translation is function of scale + + if scale is not None: + assert poses.shape[-1] in (8, 13) + pose.data[-1] = np.log(float(scale)) + return pose + + def get_pw_norm_scale_factor(self): + if self.norm_pw_scale: + # normalize scales so that things cannot go south + # we want that exp(scale) ~= self.base_scale + return (np.log(self.base_scale) - self.pw_poses[:, -1].mean()).exp() + else: + return 1 # don't norm scale for known poses + + def get_pw_scale(self): + scale = self.pw_poses[:, -1].exp() # (n_edges,) + scale = scale * self.get_pw_norm_scale_factor() + return scale + + def get_pw_poses(self): # cam to world + RT = self._get_poses(self.pw_poses) + scaled_RT = RT.clone() + scaled_RT[:, :3] *= self.get_pw_scale().view(-1, 1, 1) # scale the rotation AND translation + return scaled_RT + + def get_masks(self): + return [(conf > self.min_conf_thr) for conf in self.im_conf] + + def depth_to_pts3d(self): + raise NotImplementedError() + + def get_pts3d(self, raw=False): + res = self.depth_to_pts3d() + if not raw: + res = [dm[:h*w].view(h, w, 3) for dm, (h, w) in zip(res, self.imshapes)] + return res + + def _set_focal(self, idx, focal, force=False): + raise NotImplementedError() + + def get_focals(self): + raise NotImplementedError() + + def get_known_focal_mask(self): + raise NotImplementedError() + + def get_principal_points(self): + raise NotImplementedError() + + def get_conf(self, mode=None): + trf = self.conf_trf if mode is None else get_conf_trf(mode) + return [trf(c) for c in self.im_conf] + + def get_im_poses(self): + raise NotImplementedError() + + def _set_depthmap(self, idx, depth, force=False): + raise NotImplementedError() + + def get_depthmaps(self, raw=False): + raise NotImplementedError() + + def clean_pointcloud(self, **kw): + cams = inv(self.get_im_poses()) + K = self.get_intrinsics() + depthmaps = self.get_depthmaps() + all_pts3d = self.get_pts3d() + + new_im_confs = clean_pointcloud(self.im_conf, K, cams, depthmaps, all_pts3d, **kw) + + for i, new_conf in enumerate(new_im_confs): + self.im_conf[i].data[:] = new_conf + return self + + def forward(self, ret_details=False): + pw_poses = self.get_pw_poses() # cam-to-world + pw_adapt = self.get_adaptors() + proj_pts3d = self.get_pts3d() + # pre-compute pixel weights + weight_i = {i_j: self.conf_trf(c) for i_j, c in self.conf_i.items()} + weight_j = {i_j: self.conf_trf(c) for i_j, c in self.conf_j.items()} + + loss = 0 + if ret_details: + details = -torch.ones((self.n_imgs, self.n_imgs)) + + for e, (i, j) in enumerate(self.edges): + i_j = edge_str(i, j) + # distance in image i and j + aligned_pred_i = geotrf(pw_poses[e], pw_adapt[e] * self.pred_i[i_j]) + aligned_pred_j = geotrf(pw_poses[e], pw_adapt[e] * self.pred_j[i_j]) + li = self.dist(proj_pts3d[i], aligned_pred_i, weight=weight_i[i_j]).mean() + lj = self.dist(proj_pts3d[j], aligned_pred_j, weight=weight_j[i_j]).mean() + loss = loss + li + lj + + if ret_details: + details[i, j] = li + lj + loss /= self.n_edges # average over all pairs + + if ret_details: + return loss, details + return loss + + @torch.cuda.amp.autocast(enabled=False) + def compute_global_alignment(self, init=None, niter_PnP=10, **kw): + if init is None: + pass + elif init == 'msp' or init == 'mst': + init_fun.init_minimum_spanning_tree(self, niter_PnP=niter_PnP) + elif init == 'known_poses': + init_fun.init_from_known_poses(self, min_conf_thr=self.min_conf_thr, + niter_PnP=niter_PnP) + else: + raise ValueError(f'bad value for {init=}') + + return global_alignment_loop(self, **kw) + + @torch.no_grad() + def mask_sky(self): + res = deepcopy(self) + for i in range(self.n_imgs): + sky = segment_sky(self.imgs[i]) + res.im_conf[i][sky] = 0 + return res + + def show(self, show_pw_cams=False, show_pw_pts3d=False, cam_size=None, **kw): + viz = SceneViz() + if self.imgs is None: + colors = np.random.randint(0, 256, size=(self.n_imgs, 3)) + colors = list(map(tuple, colors.tolist())) + for n in range(self.n_imgs): + viz.add_pointcloud(self.get_pts3d()[n], colors[n], self.get_masks()[n]) + else: + viz.add_pointcloud(self.get_pts3d(), self.imgs, self.get_masks()) + colors = np.random.randint(256, size=(self.n_imgs, 3)) + + # camera poses + im_poses = to_numpy(self.get_im_poses()) + if cam_size is None: + cam_size = auto_cam_size(im_poses) + viz.add_cameras(im_poses, self.get_focals(), colors=colors, + images=self.imgs, imsizes=self.imsizes, cam_size=cam_size) + if show_pw_cams: + pw_poses = self.get_pw_poses() + viz.add_cameras(pw_poses, color=(192, 0, 192), cam_size=cam_size) + + if show_pw_pts3d: + pts = [geotrf(pw_poses[e], self.pred_i[edge_str(i, j)]) for e, (i, j) in enumerate(self.edges)] + viz.add_pointcloud(pts, (128, 0, 128)) + + viz.show(**kw) + return viz + + +def global_alignment_loop(net, lr=0.01, niter=300, schedule='cosine', lr_min=1e-6): + params = [p for p in net.parameters() if p.requires_grad] + if not params: + return net + + verbose = net.verbose + if verbose: + print('Global alignement - optimizing for:') + print([name for name, value in net.named_parameters() if value.requires_grad]) + + lr_base = lr + optimizer = torch.optim.Adam(params, lr=lr, betas=(0.9, 0.9)) + + loss = float('inf') + if verbose: + with tqdm.tqdm(total=niter) as bar: + while bar.n < bar.total: + loss, lr = global_alignment_iter(net, bar.n, niter, lr_base, lr_min, optimizer, schedule) + bar.set_postfix_str(f'{lr=:g} loss={loss:g}') + bar.update() + else: + for n in range(niter): + loss, _ = global_alignment_iter(net, n, niter, lr_base, lr_min, optimizer, schedule) + return loss + + +def global_alignment_iter(net, cur_iter, niter, lr_base, lr_min, optimizer, schedule): + t = cur_iter / niter + if schedule == 'cosine': + lr = cosine_schedule(t, lr_base, lr_min) + elif schedule == 'linear': + lr = linear_schedule(t, lr_base, lr_min) + else: + raise ValueError(f'bad lr {schedule=}') + adjust_learning_rate_by_lr(optimizer, lr) + optimizer.zero_grad() + loss = net() + loss.backward() + optimizer.step() + + return float(loss), lr + + +@torch.no_grad() +def clean_pointcloud( im_confs, K, cams, depthmaps, all_pts3d, + tol=0.001, bad_conf=0, dbg=()): + """ Method: + 1) express all 3d points in each camera coordinate frame + 2) if they're in front of a depthmap --> then lower their confidence + """ + assert len(im_confs) == len(cams) == len(K) == len(depthmaps) == len(all_pts3d) + assert 0 <= tol < 1 + res = [c.clone() for c in im_confs] + + # reshape appropriately + all_pts3d = [p.view(*c.shape,3) for p,c in zip(all_pts3d, im_confs)] + depthmaps = [d.view(*c.shape) for d,c in zip(depthmaps, im_confs)] + + for i, pts3d in enumerate(all_pts3d): + for j in range(len(all_pts3d)): + if i == j: continue + + # project 3dpts in other view + proj = geotrf(cams[j], pts3d) + proj_depth = proj[:,:,2] + u,v = geotrf(K[j], proj, norm=1, ncol=2).round().long().unbind(-1) + + # check which points are actually in the visible cone + H, W = im_confs[j].shape + msk_i = (proj_depth > 0) & (0 <= u) & (u < W) & (0 <= v) & (v < H) + msk_j = v[msk_i], u[msk_i] + + # find bad points = those in front but less confident + bad_points = (proj_depth[msk_i] < (1-tol) * depthmaps[j][msk_j]) & (res[i][msk_i] < res[j][msk_j]) + + bad_msk_i = msk_i.clone() + bad_msk_i[msk_i] = bad_points + res[i][bad_msk_i] = res[i][bad_msk_i].clip_(max=bad_conf) + + return res diff --git a/submodules/mast3r/dust3r/dust3r/cloud_opt/commons.py b/submodules/mast3r/dust3r/dust3r/cloud_opt/commons.py new file mode 100644 index 0000000000000000000000000000000000000000..3be9f855a69ea18c82dcc8e5769e0149a59649bd --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/cloud_opt/commons.py @@ -0,0 +1,90 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utility functions for global alignment +# -------------------------------------------------------- +import torch +import torch.nn as nn +import numpy as np + + +def edge_str(i, j): + return f'{i}_{j}' + + +def i_j_ij(ij): + return edge_str(*ij), ij + + +def edge_conf(conf_i, conf_j, edge): + return float(conf_i[edge].mean() * conf_j[edge].mean()) + + +def compute_edge_scores(edges, conf_i, conf_j): + return {(i, j): edge_conf(conf_i, conf_j, e) for e, (i, j) in edges} + + +def NoGradParamDict(x): + assert isinstance(x, dict) + return nn.ParameterDict(x).requires_grad_(False) + + +def get_imshapes(edges, pred_i, pred_j): + n_imgs = max(max(e) for e in edges) + 1 + imshapes = [None] * n_imgs + for e, (i, j) in enumerate(edges): + shape_i = tuple(pred_i[e].shape[0:2]) + shape_j = tuple(pred_j[e].shape[0:2]) + if imshapes[i]: + assert imshapes[i] == shape_i, f'incorrect shape for image {i}' + if imshapes[j]: + assert imshapes[j] == shape_j, f'incorrect shape for image {j}' + imshapes[i] = shape_i + imshapes[j] = shape_j + return imshapes + + +def get_conf_trf(mode): + if mode == 'log': + def conf_trf(x): return x.log() + elif mode == 'sqrt': + def conf_trf(x): return x.sqrt() + elif mode == 'm1': + def conf_trf(x): return x-1 + elif mode in ('id', 'none'): + def conf_trf(x): return x + else: + raise ValueError(f'bad mode for {mode=}') + return conf_trf + + +def l2_dist(a, b, weight): + return ((a - b).square().sum(dim=-1) * weight) + + +def l1_dist(a, b, weight): + return ((a - b).norm(dim=-1) * weight) + + +ALL_DISTS = dict(l1=l1_dist, l2=l2_dist) + + +def signed_log1p(x): + sign = torch.sign(x) + return sign * torch.log1p(torch.abs(x)) + + +def signed_expm1(x): + sign = torch.sign(x) + return sign * torch.expm1(torch.abs(x)) + + +def cosine_schedule(t, lr_start, lr_end): + assert 0 <= t <= 1 + return lr_end + (lr_start - lr_end) * (1+np.cos(t * np.pi))/2 + + +def linear_schedule(t, lr_start, lr_end): + assert 0 <= t <= 1 + return lr_start + (lr_end - lr_start) * t diff --git a/submodules/mast3r/dust3r/dust3r/cloud_opt/init_im_poses.py b/submodules/mast3r/dust3r/dust3r/cloud_opt/init_im_poses.py new file mode 100644 index 0000000000000000000000000000000000000000..7887c5cde27115273601e704b81ca0b0301f3715 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/cloud_opt/init_im_poses.py @@ -0,0 +1,316 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Initialization functions for global alignment +# -------------------------------------------------------- +from functools import cache + +import numpy as np +import scipy.sparse as sp +import torch +import cv2 +import roma +from tqdm import tqdm + +from dust3r.utils.geometry import geotrf, inv, get_med_dist_between_poses +from dust3r.post_process import estimate_focal_knowing_depth +from dust3r.viz import to_numpy + +from dust3r.cloud_opt.commons import edge_str, i_j_ij, compute_edge_scores + + +@torch.no_grad() +def init_from_known_poses(self, niter_PnP=10, min_conf_thr=3): + device = self.device + + # indices of known poses + nkp, known_poses_msk, known_poses = get_known_poses(self) + assert nkp == self.n_imgs, 'not all poses are known' + + # get all focals + nkf, _, im_focals = get_known_focals(self) + assert nkf == self.n_imgs + im_pp = self.get_principal_points() + + best_depthmaps = {} + # init all pairwise poses + for e, (i, j) in enumerate(tqdm(self.edges, disable=not self.verbose)): + i_j = edge_str(i, j) + + # find relative pose for this pair + P1 = torch.eye(4, device=device) + msk = self.conf_i[i_j] > min(min_conf_thr, self.conf_i[i_j].min() - 0.1) + _, P2 = fast_pnp(self.pred_j[i_j], float(im_focals[i].mean()), + pp=im_pp[i], msk=msk, device=device, niter_PnP=niter_PnP) + + # align the two predicted camera with the two gt cameras + s, R, T = align_multiple_poses(torch.stack((P1, P2)), known_poses[[i, j]]) + # normally we have known_poses[i] ~= sRT_to_4x4(s,R,T,device) @ P1 + # and geotrf(sRT_to_4x4(1,R,T,device), s*P2[:3,3]) + self._set_pose(self.pw_poses, e, R, T, scale=s) + + # remember if this is a good depthmap + score = float(self.conf_i[i_j].mean()) + if score > best_depthmaps.get(i, (0,))[0]: + best_depthmaps[i] = score, i_j, s + + # init all image poses + for n in range(self.n_imgs): + assert known_poses_msk[n] + _, i_j, scale = best_depthmaps[n] + depth = self.pred_i[i_j][:, :, 2] + self._set_depthmap(n, depth * scale) + + +@torch.no_grad() +def init_minimum_spanning_tree(self, **kw): + """ Init all camera poses (image-wise and pairwise poses) given + an initial set of pairwise estimations. + """ + device = self.device + pts3d, _, im_focals, im_poses = minimum_spanning_tree(self.imshapes, self.edges, + self.pred_i, self.pred_j, self.conf_i, self.conf_j, self.im_conf, self.min_conf_thr, + device, has_im_poses=self.has_im_poses, verbose=self.verbose, + **kw) + + return init_from_pts3d(self, pts3d, im_focals, im_poses) + + +def init_from_pts3d(self, pts3d, im_focals, im_poses): + # init poses + nkp, known_poses_msk, known_poses = get_known_poses(self) + if nkp == 1: + raise NotImplementedError("Would be simpler to just align everything afterwards on the single known pose") + elif nkp > 1: + # global rigid SE3 alignment + s, R, T = align_multiple_poses(im_poses[known_poses_msk], known_poses[known_poses_msk]) + trf = sRT_to_4x4(s, R, T, device=known_poses.device) + + # rotate everything + im_poses = trf @ im_poses + im_poses[:, :3, :3] /= s # undo scaling on the rotation part + for img_pts3d in pts3d: + img_pts3d[:] = geotrf(trf, img_pts3d) + + # set all pairwise poses + for e, (i, j) in enumerate(self.edges): + i_j = edge_str(i, j) + # compute transform that goes from cam to world + s, R, T = rigid_points_registration(self.pred_i[i_j], pts3d[i], conf=self.conf_i[i_j]) + self._set_pose(self.pw_poses, e, R, T, scale=s) + + # take into account the scale normalization + s_factor = self.get_pw_norm_scale_factor() + im_poses[:, :3, 3] *= s_factor # apply downscaling factor + for img_pts3d in pts3d: + img_pts3d *= s_factor + + # init all image poses + if self.has_im_poses: + for i in range(self.n_imgs): + cam2world = im_poses[i] + depth = geotrf(inv(cam2world), pts3d[i])[..., 2] + self._set_depthmap(i, depth) + self._set_pose(self.im_poses, i, cam2world) + if im_focals[i] is not None: + self._set_focal(i, im_focals[i]) + + if self.verbose: + print(' init loss =', float(self())) + + +def minimum_spanning_tree(imshapes, edges, pred_i, pred_j, conf_i, conf_j, im_conf, min_conf_thr, + device, has_im_poses=True, niter_PnP=10, verbose=True): + n_imgs = len(imshapes) + sparse_graph = -dict_to_sparse_graph(compute_edge_scores(map(i_j_ij, edges), conf_i, conf_j)) + msp = sp.csgraph.minimum_spanning_tree(sparse_graph).tocoo() + + # temp variable to store 3d points + pts3d = [None] * len(imshapes) + + todo = sorted(zip(-msp.data, msp.row, msp.col)) # sorted edges + im_poses = [None] * n_imgs + im_focals = [None] * n_imgs + + # init with strongest edge + score, i, j = todo.pop() + if verbose: + print(f' init edge ({i}*,{j}*) {score=}') + i_j = edge_str(i, j) + pts3d[i] = pred_i[i_j].clone() + pts3d[j] = pred_j[i_j].clone() + done = {i, j} + if has_im_poses: + im_poses[i] = torch.eye(4, device=device) + im_focals[i] = estimate_focal(pred_i[i_j]) + + # set initial pointcloud based on pairwise graph + msp_edges = [(i, j)] + while todo: + # each time, predict the next one + score, i, j = todo.pop() + + if im_focals[i] is None: + im_focals[i] = estimate_focal(pred_i[i_j]) + + if i in done: + if verbose: + print(f' init edge ({i},{j}*) {score=}') + assert j not in done + # align pred[i] with pts3d[i], and then set j accordingly + i_j = edge_str(i, j) + s, R, T = rigid_points_registration(pred_i[i_j], pts3d[i], conf=conf_i[i_j]) + trf = sRT_to_4x4(s, R, T, device) + pts3d[j] = geotrf(trf, pred_j[i_j]) + done.add(j) + msp_edges.append((i, j)) + + if has_im_poses and im_poses[i] is None: + im_poses[i] = sRT_to_4x4(1, R, T, device) + + elif j in done: + if verbose: + print(f' init edge ({i}*,{j}) {score=}') + assert i not in done + i_j = edge_str(i, j) + s, R, T = rigid_points_registration(pred_j[i_j], pts3d[j], conf=conf_j[i_j]) + trf = sRT_to_4x4(s, R, T, device) + pts3d[i] = geotrf(trf, pred_i[i_j]) + done.add(i) + msp_edges.append((i, j)) + + if has_im_poses and im_poses[i] is None: + im_poses[i] = sRT_to_4x4(1, R, T, device) + else: + # let's try again later + todo.insert(0, (score, i, j)) + + if has_im_poses: + # complete all missing informations + pair_scores = list(sparse_graph.values()) # already negative scores: less is best + edges_from_best_to_worse = np.array(list(sparse_graph.keys()))[np.argsort(pair_scores)] + for i, j in edges_from_best_to_worse.tolist(): + if im_focals[i] is None: + im_focals[i] = estimate_focal(pred_i[edge_str(i, j)]) + + for i in range(n_imgs): + if im_poses[i] is None: + msk = im_conf[i] > min_conf_thr + res = fast_pnp(pts3d[i], im_focals[i], msk=msk, device=device, niter_PnP=niter_PnP) + if res: + im_focals[i], im_poses[i] = res + if im_poses[i] is None: + im_poses[i] = torch.eye(4, device=device) + im_poses = torch.stack(im_poses) + else: + im_poses = im_focals = None + + return pts3d, msp_edges, im_focals, im_poses + + +def dict_to_sparse_graph(dic): + n_imgs = max(max(e) for e in dic) + 1 + res = sp.dok_array((n_imgs, n_imgs)) + for edge, value in dic.items(): + res[edge] = value + return res + + +def rigid_points_registration(pts1, pts2, conf): + R, T, s = roma.rigid_points_registration( + pts1.reshape(-1, 3), pts2.reshape(-1, 3), weights=conf.ravel(), compute_scaling=True) + return s, R, T # return un-scaled (R, T) + + +def sRT_to_4x4(scale, R, T, device): + trf = torch.eye(4, device=device) + trf[:3, :3] = R * scale + trf[:3, 3] = T.ravel() # doesn't need scaling + return trf + + +def estimate_focal(pts3d_i, pp=None): + if pp is None: + H, W, THREE = pts3d_i.shape + assert THREE == 3 + pp = torch.tensor((W/2, H/2), device=pts3d_i.device) + focal = estimate_focal_knowing_depth(pts3d_i.unsqueeze(0), pp.unsqueeze(0), focal_mode='weiszfeld').ravel() + return float(focal) + + +@cache +def pixel_grid(H, W): + return np.mgrid[:W, :H].T.astype(np.float32) + + +def fast_pnp(pts3d, focal, msk, device, pp=None, niter_PnP=10): + # extract camera poses and focals with RANSAC-PnP + if msk.sum() < 4: + return None # we need at least 4 points for PnP + pts3d, msk = map(to_numpy, (pts3d, msk)) + + H, W, THREE = pts3d.shape + assert THREE == 3 + pixels = pixel_grid(H, W) + + if focal is None: + S = max(W, H) + tentative_focals = np.geomspace(S/2, S*3, 21) + else: + tentative_focals = [focal] + + if pp is None: + pp = (W/2, H/2) + else: + pp = to_numpy(pp) + + best = 0, + for focal in tentative_focals: + K = np.float32([(focal, 0, pp[0]), (0, focal, pp[1]), (0, 0, 1)]) + + success, R, T, inliers = cv2.solvePnPRansac(pts3d[msk], pixels[msk], K, None, + iterationsCount=niter_PnP, reprojectionError=5, flags=cv2.SOLVEPNP_SQPNP) + if not success: + continue + + score = len(inliers) + if success and score > best[0]: + best = score, R, T, focal + + if not best[0]: + return None + + _, R, T, best_focal = best + R = cv2.Rodrigues(R)[0] # world to cam + R, T = map(torch.from_numpy, (R, T)) + return best_focal, inv(sRT_to_4x4(1, R, T, device)) # cam to world + + +def get_known_poses(self): + if self.has_im_poses: + known_poses_msk = torch.tensor([not (p.requires_grad) for p in self.im_poses]) + known_poses = self.get_im_poses() + return known_poses_msk.sum(), known_poses_msk, known_poses + else: + return 0, None, None + + +def get_known_focals(self): + if self.has_im_poses: + known_focal_msk = self.get_known_focal_mask() + known_focals = self.get_focals() + return known_focal_msk.sum(), known_focal_msk, known_focals + else: + return 0, None, None + + +def align_multiple_poses(src_poses, target_poses): + N = len(src_poses) + assert src_poses.shape == target_poses.shape == (N, 4, 4) + + def center_and_z(poses): + eps = get_med_dist_between_poses(poses) / 100 + return torch.cat((poses[:, :3, 3], poses[:, :3, 3] + eps*poses[:, :3, 2])) + R, T, s = roma.rigid_points_registration(center_and_z(src_poses), center_and_z(target_poses), compute_scaling=True) + return s, R, T diff --git a/submodules/mast3r/dust3r/dust3r/cloud_opt/modular_optimizer.py b/submodules/mast3r/dust3r/dust3r/cloud_opt/modular_optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..d06464b40276684385c18b9195be1491c6f47f07 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/cloud_opt/modular_optimizer.py @@ -0,0 +1,145 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Slower implementation of the global alignment that allows to freeze partial poses/intrinsics +# -------------------------------------------------------- +import numpy as np +import torch +import torch.nn as nn + +from dust3r.cloud_opt.base_opt import BasePCOptimizer +from dust3r.utils.geometry import geotrf +from dust3r.utils.device import to_cpu, to_numpy +from dust3r.utils.geometry import depthmap_to_pts3d + + +class ModularPointCloudOptimizer (BasePCOptimizer): + """ Optimize a global scene, given a list of pairwise observations. + Unlike PointCloudOptimizer, you can fix parts of the optimization process (partial poses/intrinsics) + Graph node: images + Graph edges: observations = (pred1, pred2) + """ + + def __init__(self, *args, optimize_pp=False, fx_and_fy=False, focal_brake=20, **kwargs): + super().__init__(*args, **kwargs) + self.has_im_poses = True # by definition of this class + self.focal_brake = focal_brake + + # adding thing to optimize + self.im_depthmaps = nn.ParameterList(torch.randn(H, W)/10-3 for H, W in self.imshapes) # log(depth) + self.im_poses = nn.ParameterList(self.rand_pose(self.POSE_DIM) for _ in range(self.n_imgs)) # camera poses + default_focals = [self.focal_brake * np.log(max(H, W)) for H, W in self.imshapes] + self.im_focals = nn.ParameterList(torch.FloatTensor([f, f] if fx_and_fy else [ + f]) for f in default_focals) # camera intrinsics + self.im_pp = nn.ParameterList(torch.zeros((2,)) for _ in range(self.n_imgs)) # camera intrinsics + self.im_pp.requires_grad_(optimize_pp) + + def preset_pose(self, known_poses, pose_msk=None): # cam-to-world + if isinstance(known_poses, torch.Tensor) and known_poses.ndim == 2: + known_poses = [known_poses] + for idx, pose in zip(self._get_msk_indices(pose_msk), known_poses): + if self.verbose: + print(f' (setting pose #{idx} = {pose[:3,3]})') + self._no_grad(self._set_pose(self.im_poses, idx, torch.tensor(pose), force=True)) + + # normalize scale if there's less than 1 known pose + n_known_poses = sum((p.requires_grad is False) for p in self.im_poses) + self.norm_pw_scale = (n_known_poses <= 1) + + def preset_intrinsics(self, known_intrinsics, msk=None): + if isinstance(known_intrinsics, torch.Tensor) and known_intrinsics.ndim == 2: + known_intrinsics = [known_intrinsics] + for K in known_intrinsics: + assert K.shape == (3, 3) + self.preset_focal([K.diagonal()[:2].mean() for K in known_intrinsics], msk) + self.preset_principal_point([K[:2, 2] for K in known_intrinsics], msk) + + def preset_focal(self, known_focals, msk=None): + for idx, focal in zip(self._get_msk_indices(msk), known_focals): + if self.verbose: + print(f' (setting focal #{idx} = {focal})') + self._no_grad(self._set_focal(idx, focal, force=True)) + + def preset_principal_point(self, known_pp, msk=None): + for idx, pp in zip(self._get_msk_indices(msk), known_pp): + if self.verbose: + print(f' (setting principal point #{idx} = {pp})') + self._no_grad(self._set_principal_point(idx, pp, force=True)) + + def _no_grad(self, tensor): + return tensor.requires_grad_(False) + + def _get_msk_indices(self, msk): + if msk is None: + return range(self.n_imgs) + elif isinstance(msk, int): + return [msk] + elif isinstance(msk, (tuple, list)): + return self._get_msk_indices(np.array(msk)) + elif msk.dtype in (bool, torch.bool, np.bool_): + assert len(msk) == self.n_imgs + return np.where(msk)[0] + elif np.issubdtype(msk.dtype, np.integer): + return msk + else: + raise ValueError(f'bad {msk=}') + + def _set_focal(self, idx, focal, force=False): + param = self.im_focals[idx] + if param.requires_grad or force: # can only init a parameter not already initialized + param.data[:] = self.focal_brake * np.log(focal) + return param + + def get_focals(self): + log_focals = torch.stack(list(self.im_focals), dim=0) + return (log_focals / self.focal_brake).exp() + + def _set_principal_point(self, idx, pp, force=False): + param = self.im_pp[idx] + H, W = self.imshapes[idx] + if param.requires_grad or force: # can only init a parameter not already initialized + param.data[:] = to_cpu(to_numpy(pp) - (W/2, H/2)) / 10 + return param + + def get_principal_points(self): + return torch.stack([pp.new((W/2, H/2))+10*pp for pp, (H, W) in zip(self.im_pp, self.imshapes)]) + + def get_intrinsics(self): + K = torch.zeros((self.n_imgs, 3, 3), device=self.device) + focals = self.get_focals().view(self.n_imgs, -1) + K[:, 0, 0] = focals[:, 0] + K[:, 1, 1] = focals[:, -1] + K[:, :2, 2] = self.get_principal_points() + K[:, 2, 2] = 1 + return K + + def get_im_poses(self): # cam to world + cam2world = self._get_poses(torch.stack(list(self.im_poses))) + return cam2world + + def _set_depthmap(self, idx, depth, force=False): + param = self.im_depthmaps[idx] + if param.requires_grad or force: # can only init a parameter not already initialized + param.data[:] = depth.log().nan_to_num(neginf=0) + return param + + def get_depthmaps(self): + return [d.exp() for d in self.im_depthmaps] + + def depth_to_pts3d(self): + # Get depths and projection params if not provided + focals = self.get_focals() + pp = self.get_principal_points() + im_poses = self.get_im_poses() + depth = self.get_depthmaps() + + # convert focal to (1,2,H,W) constant field + def focal_ex(i): return focals[i][..., None, None].expand(1, *focals[i].shape, *self.imshapes[i]) + # get pointmaps in camera frame + rel_ptmaps = [depthmap_to_pts3d(depth[i][None], focal_ex(i), pp=pp[i:i+1])[0] for i in range(im_poses.shape[0])] + # project to world frame + return [geotrf(pose, ptmap) for pose, ptmap in zip(im_poses, rel_ptmaps)] + + def get_pts3d(self): + return self.depth_to_pts3d() diff --git a/submodules/mast3r/dust3r/dust3r/cloud_opt/optimizer.py b/submodules/mast3r/dust3r/dust3r/cloud_opt/optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..42e48613e55faa4ede5a366d1c0bfc4d18ffae4f --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/cloud_opt/optimizer.py @@ -0,0 +1,248 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Main class for the implementation of the global alignment +# -------------------------------------------------------- +import numpy as np +import torch +import torch.nn as nn + +from dust3r.cloud_opt.base_opt import BasePCOptimizer +from dust3r.utils.geometry import xy_grid, geotrf +from dust3r.utils.device import to_cpu, to_numpy + + +class PointCloudOptimizer(BasePCOptimizer): + """ Optimize a global scene, given a list of pairwise observations. + Graph node: images + Graph edges: observations = (pred1, pred2) + """ + + def __init__(self, *args, optimize_pp=False, focal_break=20, **kwargs): + super().__init__(*args, **kwargs) + + self.has_im_poses = True # by definition of this class + self.focal_break = focal_break + + # adding thing to optimize + self.im_depthmaps = nn.ParameterList(torch.randn(H, W)/10-3 for H, W in self.imshapes) # log(depth) + self.im_poses = nn.ParameterList(self.rand_pose(self.POSE_DIM) for _ in range(self.n_imgs)) # camera poses + self.im_focals = nn.ParameterList(torch.FloatTensor( + [self.focal_break*np.log(max(H, W))]) for H, W in self.imshapes) # camera intrinsics + self.im_pp = nn.ParameterList(torch.zeros((2,)) for _ in range(self.n_imgs)) # camera intrinsics + self.im_pp.requires_grad_(optimize_pp) + + self.imshape = self.imshapes[0] + im_areas = [h*w for h, w in self.imshapes] + self.max_area = max(im_areas) + + # adding thing to optimize + self.im_depthmaps = ParameterStack(self.im_depthmaps, is_param=True, fill=self.max_area) + self.im_poses = ParameterStack(self.im_poses, is_param=True) + self.im_focals = ParameterStack(self.im_focals, is_param=True) + self.im_pp = ParameterStack(self.im_pp, is_param=True) + self.register_buffer('_pp', torch.tensor([(w/2, h/2) for h, w in self.imshapes])) + self.register_buffer('_grid', ParameterStack( + [xy_grid(W, H, device=self.device) for H, W in self.imshapes], fill=self.max_area)) + + # pre-compute pixel weights + self.register_buffer('_weight_i', ParameterStack( + [self.conf_trf(self.conf_i[i_j]) for i_j in self.str_edges], fill=self.max_area)) + self.register_buffer('_weight_j', ParameterStack( + [self.conf_trf(self.conf_j[i_j]) for i_j in self.str_edges], fill=self.max_area)) + + # precompute aa + self.register_buffer('_stacked_pred_i', ParameterStack(self.pred_i, self.str_edges, fill=self.max_area)) + self.register_buffer('_stacked_pred_j', ParameterStack(self.pred_j, self.str_edges, fill=self.max_area)) + self.register_buffer('_ei', torch.tensor([i for i, j in self.edges])) + self.register_buffer('_ej', torch.tensor([j for i, j in self.edges])) + self.total_area_i = sum([im_areas[i] for i, j in self.edges]) + self.total_area_j = sum([im_areas[j] for i, j in self.edges]) + + def _check_all_imgs_are_selected(self, msk): + assert np.all(self._get_msk_indices(msk) == np.arange(self.n_imgs)), 'incomplete mask!' + + def preset_pose(self, known_poses, pose_msk=None): # cam-to-world + self._check_all_imgs_are_selected(pose_msk) + + if isinstance(known_poses, torch.Tensor) and known_poses.ndim == 2: + known_poses = [known_poses] + for idx, pose in zip(self._get_msk_indices(pose_msk), known_poses): + if self.verbose: + print(f' (setting pose #{idx} = {pose[:3,3]})') + self._no_grad(self._set_pose(self.im_poses, idx, torch.tensor(pose))) + + # normalize scale if there's less than 1 known pose + n_known_poses = sum((p.requires_grad is False) for p in self.im_poses) + self.norm_pw_scale = (n_known_poses <= 1) + + self.im_poses.requires_grad_(False) + self.norm_pw_scale = False + + def preset_focal(self, known_focals, msk=None): + self._check_all_imgs_are_selected(msk) + + for idx, focal in zip(self._get_msk_indices(msk), known_focals): + if self.verbose: + print(f' (setting focal #{idx} = {focal})') + self._no_grad(self._set_focal(idx, focal)) + + self.im_focals.requires_grad_(False) + + def preset_principal_point(self, known_pp, msk=None): + self._check_all_imgs_are_selected(msk) + + for idx, pp in zip(self._get_msk_indices(msk), known_pp): + if self.verbose: + print(f' (setting principal point #{idx} = {pp})') + self._no_grad(self._set_principal_point(idx, pp)) + + self.im_pp.requires_grad_(False) + + def _get_msk_indices(self, msk): + if msk is None: + return range(self.n_imgs) + elif isinstance(msk, int): + return [msk] + elif isinstance(msk, (tuple, list)): + return self._get_msk_indices(np.array(msk)) + elif msk.dtype in (bool, torch.bool, np.bool_): + assert len(msk) == self.n_imgs + return np.where(msk)[0] + elif np.issubdtype(msk.dtype, np.integer): + return msk + else: + raise ValueError(f'bad {msk=}') + + def _no_grad(self, tensor): + assert tensor.requires_grad, 'it must be True at this point, otherwise no modification occurs' + + def _set_focal(self, idx, focal, force=False): + param = self.im_focals[idx] + if param.requires_grad or force: # can only init a parameter not already initialized + param.data[:] = self.focal_break * np.log(focal) + return param + + def get_focals(self): + log_focals = torch.stack(list(self.im_focals), dim=0) + return (log_focals / self.focal_break).exp() + + def get_known_focal_mask(self): + return torch.tensor([not (p.requires_grad) for p in self.im_focals]) + + def _set_principal_point(self, idx, pp, force=False): + param = self.im_pp[idx] + H, W = self.imshapes[idx] + if param.requires_grad or force: # can only init a parameter not already initialized + param.data[:] = to_cpu(to_numpy(pp) - (W/2, H/2)) / 10 + return param + + def get_principal_points(self): + return self._pp + 10 * self.im_pp + + def get_intrinsics(self): + K = torch.zeros((self.n_imgs, 3, 3), device=self.device) + focals = self.get_focals().flatten() + K[:, 0, 0] = K[:, 1, 1] = focals + K[:, :2, 2] = self.get_principal_points() + K[:, 2, 2] = 1 + return K + + def get_im_poses(self): # cam to world + cam2world = self._get_poses(self.im_poses) + return cam2world + + def _set_depthmap(self, idx, depth, force=False): + depth = _ravel_hw(depth, self.max_area) + + param = self.im_depthmaps[idx] + if param.requires_grad or force: # can only init a parameter not already initialized + param.data[:] = depth.log().nan_to_num(neginf=0) + return param + + def get_depthmaps(self, raw=False): + res = self.im_depthmaps.exp() + if not raw: + res = [dm[:h*w].view(h, w) for dm, (h, w) in zip(res, self.imshapes)] + return res + + def depth_to_pts3d(self): + # Get depths and projection params if not provided + focals = self.get_focals() + pp = self.get_principal_points() + im_poses = self.get_im_poses() + depth = self.get_depthmaps(raw=True) + + # get pointmaps in camera frame + rel_ptmaps = _fast_depthmap_to_pts3d(depth, self._grid, focals, pp=pp) + # project to world frame + return geotrf(im_poses, rel_ptmaps) + + def get_pts3d(self, raw=False): + res = self.depth_to_pts3d() + if not raw: + res = [dm[:h*w].view(h, w, 3) for dm, (h, w) in zip(res, self.imshapes)] + return res + + def forward(self): + pw_poses = self.get_pw_poses() # cam-to-world + pw_adapt = self.get_adaptors().unsqueeze(1) + proj_pts3d = self.get_pts3d(raw=True) + + # rotate pairwise prediction according to pw_poses + aligned_pred_i = geotrf(pw_poses, pw_adapt * self._stacked_pred_i) + aligned_pred_j = geotrf(pw_poses, pw_adapt * self._stacked_pred_j) + + # compute the less + li = self.dist(proj_pts3d[self._ei], aligned_pred_i, weight=self._weight_i).sum() / self.total_area_i + lj = self.dist(proj_pts3d[self._ej], aligned_pred_j, weight=self._weight_j).sum() / self.total_area_j + + return li + lj + + +def _fast_depthmap_to_pts3d(depth, pixel_grid, focal, pp): + pp = pp.unsqueeze(1) + focal = focal.unsqueeze(1) + assert focal.shape == (len(depth), 1, 1) + assert pp.shape == (len(depth), 1, 2) + assert pixel_grid.shape == depth.shape + (2,) + depth = depth.unsqueeze(-1) + return torch.cat((depth * (pixel_grid - pp) / focal, depth), dim=-1) + + +def ParameterStack(params, keys=None, is_param=None, fill=0): + if keys is not None: + params = [params[k] for k in keys] + + if fill > 0: + params = [_ravel_hw(p, fill) for p in params] + + requires_grad = params[0].requires_grad + assert all(p.requires_grad == requires_grad for p in params) + + params = torch.stack(list(params)).float().detach() + if is_param or requires_grad: + params = nn.Parameter(params) + params.requires_grad_(requires_grad) + return params + + +def _ravel_hw(tensor, fill=0): + # ravel H,W + tensor = tensor.view((tensor.shape[0] * tensor.shape[1],) + tensor.shape[2:]) + + if len(tensor) < fill: + tensor = torch.cat((tensor, tensor.new_zeros((fill - len(tensor),)+tensor.shape[1:]))) + return tensor + + +def acceptable_focal_range(H, W, minf=0.5, maxf=3.5): + focal_base = max(H, W) / (2 * np.tan(np.deg2rad(60) / 2)) # size / 1.1547005383792515 + return minf*focal_base, maxf*focal_base + + +def apply_mask(img, msk): + img = img.copy() + img[msk] = 0 + return img diff --git a/submodules/mast3r/dust3r/dust3r/cloud_opt/pair_viewer.py b/submodules/mast3r/dust3r/dust3r/cloud_opt/pair_viewer.py new file mode 100644 index 0000000000000000000000000000000000000000..62ae3b9a5fbca8b96711de051d9d6597830bd488 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/cloud_opt/pair_viewer.py @@ -0,0 +1,127 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dummy optimizer for visualizing pairs +# -------------------------------------------------------- +import numpy as np +import torch +import torch.nn as nn +import cv2 + +from dust3r.cloud_opt.base_opt import BasePCOptimizer +from dust3r.utils.geometry import inv, geotrf, depthmap_to_absolute_camera_coordinates +from dust3r.cloud_opt.commons import edge_str +from dust3r.post_process import estimate_focal_knowing_depth + + +class PairViewer (BasePCOptimizer): + """ + This a Dummy Optimizer. + To use only when the goal is to visualize the results for a pair of images (with is_symmetrized) + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + assert self.is_symmetrized and self.n_edges == 2 + self.has_im_poses = True + + # compute all parameters directly from raw input + self.focals = [] + self.pp = [] + rel_poses = [] + confs = [] + for i in range(self.n_imgs): + conf = float(self.conf_i[edge_str(i, 1-i)].mean() * self.conf_j[edge_str(i, 1-i)].mean()) + if self.verbose: + print(f' - {conf=:.3} for edge {i}-{1-i}') + confs.append(conf) + + H, W = self.imshapes[i] + pts3d = self.pred_i[edge_str(i, 1-i)] + pp = torch.tensor((W/2, H/2)) + focal = float(estimate_focal_knowing_depth(pts3d[None], pp, focal_mode='weiszfeld')) + self.focals.append(focal) + self.pp.append(pp) + + # estimate the pose of pts1 in image 2 + pixels = np.mgrid[:W, :H].T.astype(np.float32) + pts3d = self.pred_j[edge_str(1-i, i)].numpy() + assert pts3d.shape[:2] == (H, W) + msk = self.get_masks()[i].numpy() + K = np.float32([(focal, 0, pp[0]), (0, focal, pp[1]), (0, 0, 1)]) + + try: + res = cv2.solvePnPRansac(pts3d[msk], pixels[msk], K, None, + iterationsCount=100, reprojectionError=5, flags=cv2.SOLVEPNP_SQPNP) + success, R, T, inliers = res + assert success + + R = cv2.Rodrigues(R)[0] # world to cam + pose = inv(np.r_[np.c_[R, T], [(0, 0, 0, 1)]]) # cam to world + except: + pose = np.eye(4) + rel_poses.append(torch.from_numpy(pose.astype(np.float32))) + + # let's use the pair with the most confidence + if confs[0] > confs[1]: + # ptcloud is expressed in camera1 + self.im_poses = [torch.eye(4), rel_poses[1]] # I, cam2-to-cam1 + self.depth = [self.pred_i['0_1'][..., 2], geotrf(inv(rel_poses[1]), self.pred_j['0_1'])[..., 2]] + else: + # ptcloud is expressed in camera2 + self.im_poses = [rel_poses[0], torch.eye(4)] # I, cam1-to-cam2 + self.depth = [geotrf(inv(rel_poses[0]), self.pred_j['1_0'])[..., 2], self.pred_i['1_0'][..., 2]] + + self.im_poses = nn.Parameter(torch.stack(self.im_poses, dim=0), requires_grad=False) + self.focals = nn.Parameter(torch.tensor(self.focals), requires_grad=False) + self.pp = nn.Parameter(torch.stack(self.pp, dim=0), requires_grad=False) + self.depth = nn.ParameterList(self.depth) + for p in self.parameters(): + p.requires_grad = False + + def _set_depthmap(self, idx, depth, force=False): + if self.verbose: + print('_set_depthmap is ignored in PairViewer') + return + + def get_depthmaps(self, raw=False): + depth = [d.to(self.device) for d in self.depth] + return depth + + def _set_focal(self, idx, focal, force=False): + self.focals[idx] = focal + + def get_focals(self): + return self.focals + + def get_known_focal_mask(self): + return torch.tensor([not (p.requires_grad) for p in self.focals]) + + def get_principal_points(self): + return self.pp + + def get_intrinsics(self): + focals = self.get_focals() + pps = self.get_principal_points() + K = torch.zeros((len(focals), 3, 3), device=self.device) + for i in range(len(focals)): + K[i, 0, 0] = K[i, 1, 1] = focals[i] + K[i, :2, 2] = pps[i] + K[i, 2, 2] = 1 + return K + + def get_im_poses(self): + return self.im_poses + + def depth_to_pts3d(self): + pts3d = [] + for d, intrinsics, im_pose in zip(self.depth, self.get_intrinsics(), self.get_im_poses()): + pts, _ = depthmap_to_absolute_camera_coordinates(d.cpu().numpy(), + intrinsics.cpu().numpy(), + im_pose.cpu().numpy()) + pts3d.append(torch.from_numpy(pts).to(device=self.device)) + return pts3d + + def forward(self): + return float('nan') diff --git a/submodules/mast3r/dust3r/dust3r/datasets/__init__.py b/submodules/mast3r/dust3r/dust3r/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2123d09ec2840ab5ee9ca43057c35f93233bde89 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/__init__.py @@ -0,0 +1,50 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +from .utils.transforms import * +from .base.batched_sampler import BatchedRandomSampler # noqa +from .arkitscenes import ARKitScenes # noqa +from .blendedmvs import BlendedMVS # noqa +from .co3d import Co3d # noqa +from .habitat import Habitat # noqa +from .megadepth import MegaDepth # noqa +from .scannetpp import ScanNetpp # noqa +from .staticthings3d import StaticThings3D # noqa +from .waymo import Waymo # noqa +from .wildrgbd import WildRGBD # noqa + + +def get_data_loader(dataset, batch_size, num_workers=8, shuffle=True, drop_last=True, pin_mem=True): + import torch + from croco.utils.misc import get_world_size, get_rank + + # pytorch dataset + if isinstance(dataset, str): + dataset = eval(dataset) + + world_size = get_world_size() + rank = get_rank() + + try: + sampler = dataset.make_sampler(batch_size, shuffle=shuffle, world_size=world_size, + rank=rank, drop_last=drop_last) + except (AttributeError, NotImplementedError): + # not avail for this dataset + if torch.distributed.is_initialized(): + sampler = torch.utils.data.DistributedSampler( + dataset, num_replicas=world_size, rank=rank, shuffle=shuffle, drop_last=drop_last + ) + elif shuffle: + sampler = torch.utils.data.RandomSampler(dataset) + else: + sampler = torch.utils.data.SequentialSampler(dataset) + + data_loader = torch.utils.data.DataLoader( + dataset, + sampler=sampler, + batch_size=batch_size, + num_workers=num_workers, + pin_memory=pin_mem, + drop_last=drop_last, + ) + + return data_loader diff --git a/submodules/mast3r/dust3r/dust3r/datasets/arkitscenes.py b/submodules/mast3r/dust3r/dust3r/datasets/arkitscenes.py new file mode 100644 index 0000000000000000000000000000000000000000..4fad51acdc18b82cd6a4d227de0dac3b25783e33 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/arkitscenes.py @@ -0,0 +1,102 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed arkitscenes +# dataset at https://github.com/apple/ARKitScenes - Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Public License https://github.com/apple/ARKitScenes/tree/main?tab=readme-ov-file#license +# See datasets_preprocess/preprocess_arkitscenes.py +# -------------------------------------------------------- +import os.path as osp +import cv2 +import numpy as np + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class ARKitScenes(BaseStereoViewDataset): + def __init__(self, *args, split, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + if split == "train": + self.split = "Training" + elif split == "test": + self.split = "Test" + else: + raise ValueError("") + + self.loaded_data = self._load_data(self.split) + + def _load_data(self, split): + with np.load(osp.join(self.ROOT, split, 'all_metadata.npz')) as data: + self.scenes = data['scenes'] + self.sceneids = data['sceneids'] + self.images = data['images'] + self.intrinsics = data['intrinsics'].astype(np.float32) + self.trajectories = data['trajectories'].astype(np.float32) + self.pairs = data['pairs'][:, :2].astype(int) + + def __len__(self): + return len(self.pairs) + + def _get_views(self, idx, resolution, rng): + + image_idx1, image_idx2 = self.pairs[idx] + + views = [] + for view_idx in [image_idx1, image_idx2]: + scene_id = self.sceneids[view_idx] + scene_dir = osp.join(self.ROOT, self.split, self.scenes[scene_id]) + + intrinsics = self.intrinsics[view_idx] + camera_pose = self.trajectories[view_idx] + basename = self.images[view_idx] + + # Load RGB image + rgb_image = imread_cv2(osp.join(scene_dir, 'vga_wide', basename.replace('.png', '.jpg'))) + # Load depthmap + depthmap = imread_cv2(osp.join(scene_dir, 'lowres_depth', basename), cv2.IMREAD_UNCHANGED) + depthmap = depthmap.astype(np.float32) / 1000 + depthmap[~np.isfinite(depthmap)] = 0 # invalid + + rgb_image, depthmap, intrinsics = self._crop_resize_if_necessary( + rgb_image, depthmap, intrinsics, resolution, rng=rng, info=view_idx) + + views.append(dict( + img=rgb_image, + depthmap=depthmap.astype(np.float32), + camera_pose=camera_pose.astype(np.float32), + camera_intrinsics=intrinsics.astype(np.float32), + dataset='arkitscenes', + label=self.scenes[scene_id] + '_' + basename, + instance=f'{str(idx)}_{str(view_idx)}', + )) + + return views + + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = ARKitScenes(split='train', ROOT="data/arkitscenes_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/submodules/mast3r/dust3r/dust3r/datasets/base/__init__.py b/submodules/mast3r/dust3r/dust3r/datasets/base/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a32692113d830ddc4af4e6ed608f222fbe062e6e --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/base/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). diff --git a/submodules/mast3r/dust3r/dust3r/datasets/base/base_stereo_view_dataset.py b/submodules/mast3r/dust3r/dust3r/datasets/base/base_stereo_view_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..17390ca29d4437fc41f3c946b235888af9e4c888 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/base/base_stereo_view_dataset.py @@ -0,0 +1,220 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# base class for implementing datasets +# -------------------------------------------------------- +import PIL +import numpy as np +import torch + +from dust3r.datasets.base.easy_dataset import EasyDataset +from dust3r.datasets.utils.transforms import ImgNorm +from dust3r.utils.geometry import depthmap_to_absolute_camera_coordinates +import dust3r.datasets.utils.cropping as cropping + + +class BaseStereoViewDataset (EasyDataset): + """ Define all basic options. + + Usage: + class MyDataset (BaseStereoViewDataset): + def _get_views(self, idx, rng): + # overload here + views = [] + views.append(dict(img=, ...)) + return views + """ + + def __init__(self, *, # only keyword arguments + split=None, + resolution=None, # square_size or (width, height) or list of [(width,height), ...] + transform=ImgNorm, + aug_crop=False, + seed=None): + self.num_views = 2 + self.split = split + self._set_resolutions(resolution) + + self.transform = transform + if isinstance(transform, str): + transform = eval(transform) + + self.aug_crop = aug_crop + self.seed = seed + + def __len__(self): + return len(self.scenes) + + def get_stats(self): + return f"{len(self)} pairs" + + def __repr__(self): + resolutions_str = '['+';'.join(f'{w}x{h}' for w, h in self._resolutions)+']' + return f"""{type(self).__name__}({self.get_stats()}, + {self.split=}, + {self.seed=}, + resolutions={resolutions_str}, + {self.transform=})""".replace('self.', '').replace('\n', '').replace(' ', '') + + def _get_views(self, idx, resolution, rng): + raise NotImplementedError() + + def __getitem__(self, idx): + if isinstance(idx, tuple): + # the idx is specifying the aspect-ratio + idx, ar_idx = idx + else: + assert len(self._resolutions) == 1 + ar_idx = 0 + + # set-up the rng + if self.seed: # reseed for each __getitem__ + self._rng = np.random.default_rng(seed=self.seed + idx) + elif not hasattr(self, '_rng'): + seed = torch.initial_seed() # this is different for each dataloader process + self._rng = np.random.default_rng(seed=seed) + + # over-loaded code + resolution = self._resolutions[ar_idx] # DO NOT CHANGE THIS (compatible with BatchedRandomSampler) + views = self._get_views(idx, resolution, self._rng) + assert len(views) == self.num_views + + # check data-types + for v, view in enumerate(views): + assert 'pts3d' not in view, f"pts3d should not be there, they will be computed afterwards based on intrinsics+depthmap for view {view_name(view)}" + view['idx'] = (idx, ar_idx, v) + + # encode the image + width, height = view['img'].size + view['true_shape'] = np.int32((height, width)) + view['img'] = self.transform(view['img']) + + assert 'camera_intrinsics' in view + if 'camera_pose' not in view: + view['camera_pose'] = np.full((4, 4), np.nan, dtype=np.float32) + else: + assert np.isfinite(view['camera_pose']).all(), f'NaN in camera pose for view {view_name(view)}' + assert 'pts3d' not in view + assert 'valid_mask' not in view + assert np.isfinite(view['depthmap']).all(), f'NaN in depthmap for view {view_name(view)}' + pts3d, valid_mask = depthmap_to_absolute_camera_coordinates(**view) + + view['pts3d'] = pts3d + view['valid_mask'] = valid_mask & np.isfinite(pts3d).all(axis=-1) + + # check all datatypes + for key, val in view.items(): + res, err_msg = is_good_type(key, val) + assert res, f"{err_msg} with {key}={val} for view {view_name(view)}" + K = view['camera_intrinsics'] + + # last thing done! + for view in views: + # transpose to make sure all views are the same size + transpose_to_landscape(view) + # this allows to check whether the RNG is is the same state each time + view['rng'] = int.from_bytes(self._rng.bytes(4), 'big') + return views + + def _set_resolutions(self, resolutions): + assert resolutions is not None, 'undefined resolution' + + if not isinstance(resolutions, list): + resolutions = [resolutions] + + self._resolutions = [] + for resolution in resolutions: + if isinstance(resolution, int): + width = height = resolution + else: + width, height = resolution + assert isinstance(width, int), f'Bad type for {width=} {type(width)=}, should be int' + assert isinstance(height, int), f'Bad type for {height=} {type(height)=}, should be int' + assert width >= height + self._resolutions.append((width, height)) + + def _crop_resize_if_necessary(self, image, depthmap, intrinsics, resolution, rng=None, info=None): + """ This function: + - first downsizes the image with LANCZOS inteprolation, + which is better than bilinear interpolation in + """ + if not isinstance(image, PIL.Image.Image): + image = PIL.Image.fromarray(image) + + # downscale with lanczos interpolation so that image.size == resolution + # cropping centered on the principal point + W, H = image.size + cx, cy = intrinsics[:2, 2].round().astype(int) + min_margin_x = min(cx, W-cx) + min_margin_y = min(cy, H-cy) + assert min_margin_x > W/5, f'Bad principal point in view={info}' + assert min_margin_y > H/5, f'Bad principal point in view={info}' + # the new window will be a rectangle of size (2*min_margin_x, 2*min_margin_y) centered on (cx,cy) + l, t = cx - min_margin_x, cy - min_margin_y + r, b = cx + min_margin_x, cy + min_margin_y + crop_bbox = (l, t, r, b) + image, depthmap, intrinsics = cropping.crop_image_depthmap(image, depthmap, intrinsics, crop_bbox) + + # transpose the resolution if necessary + W, H = image.size # new size + assert resolution[0] >= resolution[1] + if H > 1.1*W: + # image is portrait mode + resolution = resolution[::-1] + elif 0.9 < H/W < 1.1 and resolution[0] != resolution[1]: + # image is square, so we chose (portrait, landscape) randomly + if rng.integers(2): + resolution = resolution[::-1] + + # high-quality Lanczos down-scaling + target_resolution = np.array(resolution) + if self.aug_crop > 1: + target_resolution += rng.integers(0, self.aug_crop) + image, depthmap, intrinsics = cropping.rescale_image_depthmap(image, depthmap, intrinsics, target_resolution) + + # actual cropping (if necessary) with bilinear interpolation + intrinsics2 = cropping.camera_matrix_of_crop(intrinsics, image.size, resolution, offset_factor=0.5) + crop_bbox = cropping.bbox_from_intrinsics_in_out(intrinsics, intrinsics2, resolution) + image, depthmap, intrinsics2 = cropping.crop_image_depthmap(image, depthmap, intrinsics, crop_bbox) + + return image, depthmap, intrinsics2 + + +def is_good_type(key, v): + """ returns (is_good, err_msg) + """ + if isinstance(v, (str, int, tuple)): + return True, None + if v.dtype not in (np.float32, torch.float32, bool, np.int32, np.int64, np.uint8): + return False, f"bad {v.dtype=}" + return True, None + + +def view_name(view, batch_index=None): + def sel(x): return x[batch_index] if batch_index not in (None, slice(None)) else x + db = sel(view['dataset']) + label = sel(view['label']) + instance = sel(view['instance']) + return f"{db}/{label}/{instance}" + + +def transpose_to_landscape(view): + height, width = view['true_shape'] + + if width < height: + # rectify portrait to landscape + assert view['img'].shape == (3, height, width) + view['img'] = view['img'].swapaxes(1, 2) + + assert view['valid_mask'].shape == (height, width) + view['valid_mask'] = view['valid_mask'].swapaxes(0, 1) + + assert view['depthmap'].shape == (height, width) + view['depthmap'] = view['depthmap'].swapaxes(0, 1) + + assert view['pts3d'].shape == (height, width, 3) + view['pts3d'] = view['pts3d'].swapaxes(0, 1) + + # transpose x and y pixels + view['camera_intrinsics'] = view['camera_intrinsics'][[1, 0, 2]] diff --git a/submodules/mast3r/dust3r/dust3r/datasets/base/batched_sampler.py b/submodules/mast3r/dust3r/dust3r/datasets/base/batched_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..85f58a65d41bb8101159e032d5b0aac26a7cf1a1 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/base/batched_sampler.py @@ -0,0 +1,74 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Random sampling under a constraint +# -------------------------------------------------------- +import numpy as np +import torch + + +class BatchedRandomSampler: + """ Random sampling under a constraint: each sample in the batch has the same feature, + which is chosen randomly from a known pool of 'features' for each batch. + + For instance, the 'feature' could be the image aspect-ratio. + + The index returned is a tuple (sample_idx, feat_idx). + This sampler ensures that each series of `batch_size` indices has the same `feat_idx`. + """ + + def __init__(self, dataset, batch_size, pool_size, world_size=1, rank=0, drop_last=True): + self.batch_size = batch_size + self.pool_size = pool_size + + self.len_dataset = N = len(dataset) + self.total_size = round_by(N, batch_size*world_size) if drop_last else N + assert world_size == 1 or drop_last, 'must drop the last batch in distributed mode' + + # distributed sampler + self.world_size = world_size + self.rank = rank + self.epoch = None + + def __len__(self): + return self.total_size // self.world_size + + def set_epoch(self, epoch): + self.epoch = epoch + + def __iter__(self): + # prepare RNG + if self.epoch is None: + assert self.world_size == 1 and self.rank == 0, 'use set_epoch() if distributed mode is used' + seed = int(torch.empty((), dtype=torch.int64).random_().item()) + else: + seed = self.epoch + 777 + rng = np.random.default_rng(seed=seed) + + # random indices (will restart from 0 if not drop_last) + sample_idxs = np.arange(self.total_size) + rng.shuffle(sample_idxs) + + # random feat_idxs (same across each batch) + n_batches = (self.total_size+self.batch_size-1) // self.batch_size + feat_idxs = rng.integers(self.pool_size, size=n_batches) + feat_idxs = np.broadcast_to(feat_idxs[:, None], (n_batches, self.batch_size)) + feat_idxs = feat_idxs.ravel()[:self.total_size] + + # put them together + idxs = np.c_[sample_idxs, feat_idxs] # shape = (total_size, 2) + + # Distributed sampler: we select a subset of batches + # make sure the slice for each node is aligned with batch_size + size_per_proc = self.batch_size * ((self.total_size + self.world_size * + self.batch_size-1) // (self.world_size * self.batch_size)) + idxs = idxs[self.rank*size_per_proc: (self.rank+1)*size_per_proc] + + yield from (tuple(idx) for idx in idxs) + + +def round_by(total, multiple, up=False): + if up: + total = total + multiple-1 + return (total//multiple) * multiple diff --git a/submodules/mast3r/dust3r/dust3r/datasets/base/easy_dataset.py b/submodules/mast3r/dust3r/dust3r/datasets/base/easy_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..4939a88f02715a1f80be943ddb6d808e1be84db7 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/base/easy_dataset.py @@ -0,0 +1,157 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# A dataset base class that you can easily resize and combine. +# -------------------------------------------------------- +import numpy as np +from dust3r.datasets.base.batched_sampler import BatchedRandomSampler + + +class EasyDataset: + """ a dataset that you can easily resize and combine. + Examples: + --------- + 2 * dataset ==> duplicate each element 2x + + 10 @ dataset ==> set the size to 10 (random sampling, duplicates if necessary) + + dataset1 + dataset2 ==> concatenate datasets + """ + + def __add__(self, other): + return CatDataset([self, other]) + + def __rmul__(self, factor): + return MulDataset(factor, self) + + def __rmatmul__(self, factor): + return ResizedDataset(factor, self) + + def set_epoch(self, epoch): + pass # nothing to do by default + + def make_sampler(self, batch_size, shuffle=True, world_size=1, rank=0, drop_last=True): + if not (shuffle): + raise NotImplementedError() # cannot deal yet + num_of_aspect_ratios = len(self._resolutions) + return BatchedRandomSampler(self, batch_size, num_of_aspect_ratios, world_size=world_size, rank=rank, drop_last=drop_last) + + +class MulDataset (EasyDataset): + """ Artifically augmenting the size of a dataset. + """ + multiplicator: int + + def __init__(self, multiplicator, dataset): + assert isinstance(multiplicator, int) and multiplicator > 0 + self.multiplicator = multiplicator + self.dataset = dataset + + def __len__(self): + return self.multiplicator * len(self.dataset) + + def __repr__(self): + return f'{self.multiplicator}*{repr(self.dataset)}' + + def __getitem__(self, idx): + if isinstance(idx, tuple): + idx, other = idx + return self.dataset[idx // self.multiplicator, other] + else: + return self.dataset[idx // self.multiplicator] + + @property + def _resolutions(self): + return self.dataset._resolutions + + +class ResizedDataset (EasyDataset): + """ Artifically changing the size of a dataset. + """ + new_size: int + + def __init__(self, new_size, dataset): + assert isinstance(new_size, int) and new_size > 0 + self.new_size = new_size + self.dataset = dataset + + def __len__(self): + return self.new_size + + def __repr__(self): + size_str = str(self.new_size) + for i in range((len(size_str)-1) // 3): + sep = -4*i-3 + size_str = size_str[:sep] + '_' + size_str[sep:] + return f'{size_str} @ {repr(self.dataset)}' + + def set_epoch(self, epoch): + # this random shuffle only depends on the epoch + rng = np.random.default_rng(seed=epoch+777) + + # shuffle all indices + perm = rng.permutation(len(self.dataset)) + + # rotary extension until target size is met + shuffled_idxs = np.concatenate([perm] * (1 + (len(self)-1) // len(self.dataset))) + self._idxs_mapping = shuffled_idxs[:self.new_size] + + assert len(self._idxs_mapping) == self.new_size + + def __getitem__(self, idx): + assert hasattr(self, '_idxs_mapping'), 'You need to call dataset.set_epoch() to use ResizedDataset.__getitem__()' + if isinstance(idx, tuple): + idx, other = idx + return self.dataset[self._idxs_mapping[idx], other] + else: + return self.dataset[self._idxs_mapping[idx]] + + @property + def _resolutions(self): + return self.dataset._resolutions + + +class CatDataset (EasyDataset): + """ Concatenation of several datasets + """ + + def __init__(self, datasets): + for dataset in datasets: + assert isinstance(dataset, EasyDataset) + self.datasets = datasets + self._cum_sizes = np.cumsum([len(dataset) for dataset in datasets]) + + def __len__(self): + return self._cum_sizes[-1] + + def __repr__(self): + # remove uselessly long transform + return ' + '.join(repr(dataset).replace(',transform=Compose( ToTensor() Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)))', '') for dataset in self.datasets) + + def set_epoch(self, epoch): + for dataset in self.datasets: + dataset.set_epoch(epoch) + + def __getitem__(self, idx): + other = None + if isinstance(idx, tuple): + idx, other = idx + + if not (0 <= idx < len(self)): + raise IndexError() + + db_idx = np.searchsorted(self._cum_sizes, idx, 'right') + dataset = self.datasets[db_idx] + new_idx = idx - (self._cum_sizes[db_idx - 1] if db_idx > 0 else 0) + + if other is not None: + new_idx = (new_idx, other) + return dataset[new_idx] + + @property + def _resolutions(self): + resolutions = self.datasets[0]._resolutions + for dataset in self.datasets[1:]: + assert tuple(dataset._resolutions) == tuple(resolutions) + return resolutions diff --git a/submodules/mast3r/dust3r/dust3r/datasets/blendedmvs.py b/submodules/mast3r/dust3r/dust3r/datasets/blendedmvs.py new file mode 100644 index 0000000000000000000000000000000000000000..93e68c28620cc47a7b1743834e45f82d576126d0 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/blendedmvs.py @@ -0,0 +1,104 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed BlendedMVS +# dataset at https://github.com/YoYo000/BlendedMVS +# See datasets_preprocess/preprocess_blendedmvs.py +# -------------------------------------------------------- +import os.path as osp +import numpy as np + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class BlendedMVS (BaseStereoViewDataset): + """ Dataset of outdoor street scenes, 5 images each time + """ + + def __init__(self, *args, ROOT, split=None, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + self._load_data(split) + + def _load_data(self, split): + pairs = np.load(osp.join(self.ROOT, 'blendedmvs_pairs.npy')) + if split is None: + selection = slice(None) + if split == 'train': + # select 90% of all scenes + selection = (pairs['seq_low'] % 10) > 0 + if split == 'val': + # select 10% of all scenes + selection = (pairs['seq_low'] % 10) == 0 + self.pairs = pairs[selection] + + # list of all scenes + self.scenes = np.unique(self.pairs['seq_low']) # low is unique enough + + def __len__(self): + return len(self.pairs) + + def get_stats(self): + return f'{len(self)} pairs from {len(self.scenes)} scenes' + + def _get_views(self, pair_idx, resolution, rng): + seqh, seql, img1, img2, score = self.pairs[pair_idx] + + seq = f"{seqh:08x}{seql:016x}" + seq_path = osp.join(self.ROOT, seq) + + views = [] + + for view_index in [img1, img2]: + impath = f"{view_index:08n}" + image = imread_cv2(osp.join(seq_path, impath + ".jpg")) + depthmap = imread_cv2(osp.join(seq_path, impath + ".exr")) + camera_params = np.load(osp.join(seq_path, impath + ".npz")) + + intrinsics = np.float32(camera_params['intrinsics']) + camera_pose = np.eye(4, dtype=np.float32) + camera_pose[:3, :3] = camera_params['R_cam2world'] + camera_pose[:3, 3] = camera_params['t_cam2world'] + + image, depthmap, intrinsics = self._crop_resize_if_necessary( + image, depthmap, intrinsics, resolution, rng, info=(seq_path, impath)) + + views.append(dict( + img=image, + depthmap=depthmap, + camera_pose=camera_pose, # cam2world + camera_intrinsics=intrinsics, + dataset='BlendedMVS', + label=osp.relpath(seq_path, self.ROOT), + instance=impath)) + + return views + + +if __name__ == '__main__': + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = BlendedMVS(split='train', ROOT="data/blendedmvs_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(idx, view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/submodules/mast3r/dust3r/dust3r/datasets/co3d.py b/submodules/mast3r/dust3r/dust3r/datasets/co3d.py new file mode 100644 index 0000000000000000000000000000000000000000..2ea5c8555d34b776e7a48396dcd0eecece713e34 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/co3d.py @@ -0,0 +1,165 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed Co3d_v2 +# dataset at https://github.com/facebookresearch/co3d - Creative Commons Attribution-NonCommercial 4.0 International +# See datasets_preprocess/preprocess_co3d.py +# -------------------------------------------------------- +import os.path as osp +import json +import itertools +from collections import deque + +import cv2 +import numpy as np + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class Co3d(BaseStereoViewDataset): + def __init__(self, mask_bg=True, *args, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + assert mask_bg in (True, False, 'rand') + self.mask_bg = mask_bg + self.dataset_label = 'Co3d_v2' + + # load all scenes + with open(osp.join(self.ROOT, f'selected_seqs_{self.split}.json'), 'r') as f: + self.scenes = json.load(f) + self.scenes = {k: v for k, v in self.scenes.items() if len(v) > 0} + self.scenes = {(k, k2): v2 for k, v in self.scenes.items() + for k2, v2 in v.items()} + self.scene_list = list(self.scenes.keys()) + + # for each scene, we have 100 images ==> 360 degrees (so 25 frames ~= 90 degrees) + # we prepare all combinations such that i-j = +/- [5, 10, .., 90] degrees + self.combinations = [(i, j) + for i, j in itertools.combinations(range(100), 2) + if 0 < abs(i - j) <= 30 and abs(i - j) % 5 == 0] + + self.invalidate = {scene: {} for scene in self.scene_list} + + def __len__(self): + return len(self.scene_list) * len(self.combinations) + + def _get_metadatapath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'images', f'frame{view_idx:06n}.npz') + + def _get_impath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'images', f'frame{view_idx:06n}.jpg') + + def _get_depthpath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'depths', f'frame{view_idx:06n}.jpg.geometric.png') + + def _get_maskpath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'masks', f'frame{view_idx:06n}.png') + + def _read_depthmap(self, depthpath, input_metadata): + depthmap = imread_cv2(depthpath, cv2.IMREAD_UNCHANGED) + depthmap = (depthmap.astype(np.float32) / 65535) * np.nan_to_num(input_metadata['maximum_depth']) + return depthmap + + def _get_views(self, idx, resolution, rng): + # choose a scene + obj, instance = self.scene_list[idx // len(self.combinations)] + image_pool = self.scenes[obj, instance] + im1_idx, im2_idx = self.combinations[idx % len(self.combinations)] + + # add a bit of randomness + last = len(image_pool) - 1 + + if resolution not in self.invalidate[obj, instance]: # flag invalid images + self.invalidate[obj, instance][resolution] = [False for _ in range(len(image_pool))] + + # decide now if we mask the bg + mask_bg = (self.mask_bg == True) or (self.mask_bg == 'rand' and rng.choice(2)) + + views = [] + imgs_idxs = [max(0, min(im_idx + rng.integers(-4, 5), last)) for im_idx in [im2_idx, im1_idx]] + imgs_idxs = deque(imgs_idxs) + while len(imgs_idxs) > 0: # some images (few) have zero depth + im_idx = imgs_idxs.pop() + + if self.invalidate[obj, instance][resolution][im_idx]: + # search for a valid image + random_direction = 2 * rng.choice(2) - 1 + for offset in range(1, len(image_pool)): + tentative_im_idx = (im_idx + (random_direction * offset)) % len(image_pool) + if not self.invalidate[obj, instance][resolution][tentative_im_idx]: + im_idx = tentative_im_idx + break + + view_idx = image_pool[im_idx] + + impath = self._get_impath(obj, instance, view_idx) + depthpath = self._get_depthpath(obj, instance, view_idx) + + # load camera params + metadata_path = self._get_metadatapath(obj, instance, view_idx) + input_metadata = np.load(metadata_path) + camera_pose = input_metadata['camera_pose'].astype(np.float32) + intrinsics = input_metadata['camera_intrinsics'].astype(np.float32) + + # load image and depth + rgb_image = imread_cv2(impath) + depthmap = self._read_depthmap(depthpath, input_metadata) + + if mask_bg: + # load object mask + maskpath = self._get_maskpath(obj, instance, view_idx) + maskmap = imread_cv2(maskpath, cv2.IMREAD_UNCHANGED).astype(np.float32) + maskmap = (maskmap / 255.0) > 0.1 + + # update the depthmap with mask + depthmap *= maskmap + + rgb_image, depthmap, intrinsics = self._crop_resize_if_necessary( + rgb_image, depthmap, intrinsics, resolution, rng=rng, info=impath) + + num_valid = (depthmap > 0.0).sum() + if num_valid == 0: + # problem, invalidate image and retry + self.invalidate[obj, instance][resolution][im_idx] = True + imgs_idxs.append(im_idx) + continue + + views.append(dict( + img=rgb_image, + depthmap=depthmap, + camera_pose=camera_pose, + camera_intrinsics=intrinsics, + dataset=self.dataset_label, + label=osp.join(obj, instance), + instance=osp.split(impath)[1], + )) + return views + + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = Co3d(split='train', ROOT="data/co3d_subset_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/submodules/mast3r/dust3r/dust3r/datasets/habitat.py b/submodules/mast3r/dust3r/dust3r/datasets/habitat.py new file mode 100644 index 0000000000000000000000000000000000000000..11ce8a0ffb2134387d5fb794df89834db3ea8c9f --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/habitat.py @@ -0,0 +1,107 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed habitat +# dataset at https://github.com/facebookresearch/habitat-sim/blob/main/DATASETS.md +# See datasets_preprocess/habitat for more details +# -------------------------------------------------------- +import os.path as osp +import os +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" # noqa +import cv2 # noqa +import numpy as np +from PIL import Image +import json + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset + + +class Habitat(BaseStereoViewDataset): + def __init__(self, size, *args, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + assert self.split is not None + # loading list of scenes + with open(osp.join(self.ROOT, f'Habitat_{size}_scenes_{self.split}.txt')) as f: + self.scenes = f.read().splitlines() + self.instances = list(range(1, 5)) + + def filter_scene(self, label, instance=None): + if instance: + subscene, instance = instance.split('_') + label += '/' + subscene + self.instances = [int(instance) - 1] + valid = np.bool_([scene.startswith(label) for scene in self.scenes]) + assert sum(valid), 'no scene was selected for {label=} {instance=}' + self.scenes = [scene for i, scene in enumerate(self.scenes) if valid[i]] + + def _get_views(self, idx, resolution, rng): + scene = self.scenes[idx] + data_path, key = osp.split(osp.join(self.ROOT, scene)) + views = [] + two_random_views = [0, rng.choice(self.instances)] # view 0 is connected with all other views + for view_index in two_random_views: + # load the view (and use the next one if this one's broken) + for ii in range(view_index, view_index + 5): + image, depthmap, intrinsics, camera_pose = self._load_one_view(data_path, key, ii % 5, resolution, rng) + if np.isfinite(camera_pose).all(): + break + views.append(dict( + img=image, + depthmap=depthmap, + camera_pose=camera_pose, # cam2world + camera_intrinsics=intrinsics, + dataset='Habitat', + label=osp.relpath(data_path, self.ROOT), + instance=f"{key}_{view_index}")) + return views + + def _load_one_view(self, data_path, key, view_index, resolution, rng): + view_index += 1 # file indices starts at 1 + impath = osp.join(data_path, f"{key}_{view_index}.jpeg") + image = Image.open(impath) + + depthmap_filename = osp.join(data_path, f"{key}_{view_index}_depth.exr") + depthmap = cv2.imread(depthmap_filename, cv2.IMREAD_GRAYSCALE | cv2.IMREAD_ANYDEPTH) + + camera_params_filename = osp.join(data_path, f"{key}_{view_index}_camera_params.json") + with open(camera_params_filename, 'r') as f: + camera_params = json.load(f) + + intrinsics = np.float32(camera_params['camera_intrinsics']) + camera_pose = np.eye(4, dtype=np.float32) + camera_pose[:3, :3] = camera_params['R_cam2world'] + camera_pose[:3, 3] = camera_params['t_cam2world'] + + image, depthmap, intrinsics = self._crop_resize_if_necessary( + image, depthmap, intrinsics, resolution, rng, info=impath) + return image, depthmap, intrinsics, camera_pose + + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = Habitat(1_000_000, split='train', ROOT="data/habitat_processed", + resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/submodules/mast3r/dust3r/dust3r/datasets/megadepth.py b/submodules/mast3r/dust3r/dust3r/datasets/megadepth.py new file mode 100644 index 0000000000000000000000000000000000000000..8131498b76d855e5293fe79b3686fc42bf87eea8 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/megadepth.py @@ -0,0 +1,123 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed MegaDepth +# dataset at https://www.cs.cornell.edu/projects/megadepth/ +# See datasets_preprocess/preprocess_megadepth.py +# -------------------------------------------------------- +import os.path as osp +import numpy as np + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class MegaDepth(BaseStereoViewDataset): + def __init__(self, *args, split, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + self.loaded_data = self._load_data(self.split) + + if self.split is None: + pass + elif self.split == 'train': + self.select_scene(('0015', '0022'), opposite=True) + elif self.split == 'val': + self.select_scene(('0015', '0022')) + else: + raise ValueError(f'bad {self.split=}') + + def _load_data(self, split): + with np.load(osp.join(self.ROOT, 'all_metadata.npz')) as data: + self.all_scenes = data['scenes'] + self.all_images = data['images'] + self.pairs = data['pairs'] + + def __len__(self): + return len(self.pairs) + + def get_stats(self): + return f'{len(self)} pairs from {len(self.all_scenes)} scenes' + + def select_scene(self, scene, *instances, opposite=False): + scenes = (scene,) if isinstance(scene, str) else tuple(scene) + scene_id = [s.startswith(scenes) for s in self.all_scenes] + assert any(scene_id), 'no scene found' + + valid = np.in1d(self.pairs['scene_id'], np.nonzero(scene_id)[0]) + if instances: + image_id = [i.startswith(instances) for i in self.all_images] + image_id = np.nonzero(image_id)[0] + assert len(image_id), 'no instance found' + # both together? + if len(instances) == 2: + valid &= np.in1d(self.pairs['im1_id'], image_id) & np.in1d(self.pairs['im2_id'], image_id) + else: + valid &= np.in1d(self.pairs['im1_id'], image_id) | np.in1d(self.pairs['im2_id'], image_id) + + if opposite: + valid = ~valid + assert valid.any() + self.pairs = self.pairs[valid] + + def _get_views(self, pair_idx, resolution, rng): + scene_id, im1_id, im2_id, score = self.pairs[pair_idx] + + scene, subscene = self.all_scenes[scene_id].split() + seq_path = osp.join(self.ROOT, scene, subscene) + + views = [] + + for im_id in [im1_id, im2_id]: + img = self.all_images[im_id] + try: + image = imread_cv2(osp.join(seq_path, img + '.jpg')) + depthmap = imread_cv2(osp.join(seq_path, img + ".exr")) + camera_params = np.load(osp.join(seq_path, img + ".npz")) + except Exception as e: + raise OSError(f'cannot load {img}, got exception {e}') + + intrinsics = np.float32(camera_params['intrinsics']) + camera_pose = np.float32(camera_params['cam2world']) + + image, depthmap, intrinsics = self._crop_resize_if_necessary( + image, depthmap, intrinsics, resolution, rng, info=(seq_path, img)) + + views.append(dict( + img=image, + depthmap=depthmap, + camera_pose=camera_pose, # cam2world + camera_intrinsics=intrinsics, + dataset='MegaDepth', + label=osp.relpath(seq_path, self.ROOT), + instance=img)) + + return views + + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = MegaDepth(split='train', ROOT="data/megadepth_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(idx, view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/submodules/mast3r/dust3r/dust3r/datasets/scannetpp.py b/submodules/mast3r/dust3r/dust3r/datasets/scannetpp.py new file mode 100644 index 0000000000000000000000000000000000000000..520deedd0eb8cba8663af941731d89e0b2e71a80 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/scannetpp.py @@ -0,0 +1,96 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed scannet++ +# dataset at https://github.com/scannetpp/scannetpp - non-commercial research and educational purposes +# https://kaldir.vc.in.tum.de/scannetpp/static/scannetpp-terms-of-use.pdf +# See datasets_preprocess/preprocess_scannetpp.py +# -------------------------------------------------------- +import os.path as osp +import cv2 +import numpy as np + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class ScanNetpp(BaseStereoViewDataset): + def __init__(self, *args, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + assert self.split == 'train' + self.loaded_data = self._load_data() + + def _load_data(self): + with np.load(osp.join(self.ROOT, 'all_metadata.npz')) as data: + self.scenes = data['scenes'] + self.sceneids = data['sceneids'] + self.images = data['images'] + self.intrinsics = data['intrinsics'].astype(np.float32) + self.trajectories = data['trajectories'].astype(np.float32) + self.pairs = data['pairs'][:, :2].astype(int) + + def __len__(self): + return len(self.pairs) + + def _get_views(self, idx, resolution, rng): + + image_idx1, image_idx2 = self.pairs[idx] + + views = [] + for view_idx in [image_idx1, image_idx2]: + scene_id = self.sceneids[view_idx] + scene_dir = osp.join(self.ROOT, self.scenes[scene_id]) + + intrinsics = self.intrinsics[view_idx] + camera_pose = self.trajectories[view_idx] + basename = self.images[view_idx] + + # Load RGB image + rgb_image = imread_cv2(osp.join(scene_dir, 'images', basename + '.jpg')) + # Load depthmap + depthmap = imread_cv2(osp.join(scene_dir, 'depth', basename + '.png'), cv2.IMREAD_UNCHANGED) + depthmap = depthmap.astype(np.float32) / 1000 + depthmap[~np.isfinite(depthmap)] = 0 # invalid + + rgb_image, depthmap, intrinsics = self._crop_resize_if_necessary( + rgb_image, depthmap, intrinsics, resolution, rng=rng, info=view_idx) + + views.append(dict( + img=rgb_image, + depthmap=depthmap.astype(np.float32), + camera_pose=camera_pose.astype(np.float32), + camera_intrinsics=intrinsics.astype(np.float32), + dataset='ScanNet++', + label=self.scenes[scene_id] + '_' + basename, + instance=f'{str(idx)}_{str(view_idx)}', + )) + return views + + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = ScanNetpp(split='train', ROOT="data/scannetpp_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx*255, (1 - idx)*255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/submodules/mast3r/dust3r/dust3r/datasets/staticthings3d.py b/submodules/mast3r/dust3r/dust3r/datasets/staticthings3d.py new file mode 100644 index 0000000000000000000000000000000000000000..e7f70f0ee7bf8c8ab6bb1702aa2481f3d16df413 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/staticthings3d.py @@ -0,0 +1,96 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed StaticThings3D +# dataset at https://github.com/lmb-freiburg/robustmvd/ +# See datasets_preprocess/preprocess_staticthings3d.py +# -------------------------------------------------------- +import os.path as osp +import numpy as np + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class StaticThings3D (BaseStereoViewDataset): + """ Dataset of indoor scenes, 5 images each time + """ + def __init__(self, ROOT, *args, mask_bg='rand', **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + + assert mask_bg in (True, False, 'rand') + self.mask_bg = mask_bg + + # loading all pairs + assert self.split is None + self.pairs = np.load(osp.join(ROOT, 'staticthings_pairs.npy')) + + def __len__(self): + return len(self.pairs) + + def get_stats(self): + return f'{len(self)} pairs' + + def _get_views(self, pair_idx, resolution, rng): + scene, seq, cam1, im1, cam2, im2 = self.pairs[pair_idx] + seq_path = osp.join('TRAIN', scene.decode('ascii'), f'{seq:04d}') + + views = [] + + mask_bg = (self.mask_bg == True) or (self.mask_bg == 'rand' and rng.choice(2)) + + CAM = {b'l':'left', b'r':'right'} + for cam, idx in [(CAM[cam1], im1), (CAM[cam2], im2)]: + num = f"{idx:04n}" + img = num+"_clean.jpg" if rng.choice(2) else num+"_final.jpg" + image = imread_cv2(osp.join(self.ROOT, seq_path, cam, img)) + depthmap = imread_cv2(osp.join(self.ROOT, seq_path, cam, num+".exr")) + camera_params = np.load(osp.join(self.ROOT, seq_path, cam, num+".npz")) + + intrinsics = camera_params['intrinsics'] + camera_pose = camera_params['cam2world'] + + if mask_bg: + depthmap[depthmap > 200] = 0 + + image, depthmap, intrinsics = self._crop_resize_if_necessary(image, depthmap, intrinsics, resolution, rng, info=(seq_path,cam,img)) + + views.append(dict( + img = image, + depthmap = depthmap, + camera_pose = camera_pose, # cam2world + camera_intrinsics = intrinsics, + dataset = 'StaticThings3D', + label = seq_path, + instance = cam+'_'+img)) + + return views + + +if __name__ == '__main__': + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = StaticThings3D(ROOT="data/staticthings3d_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(idx, view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx*255, (1 - idx)*255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/submodules/mast3r/dust3r/dust3r/datasets/utils/__init__.py b/submodules/mast3r/dust3r/dust3r/datasets/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a32692113d830ddc4af4e6ed608f222fbe062e6e --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). diff --git a/submodules/mast3r/dust3r/dust3r/datasets/utils/cropping.py b/submodules/mast3r/dust3r/dust3r/datasets/utils/cropping.py new file mode 100644 index 0000000000000000000000000000000000000000..574b865277b118df9e0e29bc4e2c7e933d45a8f8 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/utils/cropping.py @@ -0,0 +1,126 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# croppping utilities +# -------------------------------------------------------- +import PIL.Image +import os +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 # noqa +import numpy as np # noqa +from dust3r.utils.geometry import colmap_to_opencv_intrinsics, opencv_to_colmap_intrinsics # noqa +try: + lanczos = PIL.Image.Resampling.LANCZOS + bicubic = PIL.Image.Resampling.BICUBIC +except AttributeError: + lanczos = PIL.Image.LANCZOS + bicubic = PIL.Image.BICUBIC + + +class ImageList: + """ Convenience class to aply the same operation to a whole set of images. + """ + + def __init__(self, images): + if not isinstance(images, (tuple, list, set)): + images = [images] + self.images = [] + for image in images: + if not isinstance(image, PIL.Image.Image): + image = PIL.Image.fromarray(image) + self.images.append(image) + + def __len__(self): + return len(self.images) + + def to_pil(self): + return tuple(self.images) if len(self.images) > 1 else self.images[0] + + @property + def size(self): + sizes = [im.size for im in self.images] + assert all(sizes[0] == s for s in sizes) + return sizes[0] + + def resize(self, *args, **kwargs): + return ImageList(self._dispatch('resize', *args, **kwargs)) + + def crop(self, *args, **kwargs): + return ImageList(self._dispatch('crop', *args, **kwargs)) + + def _dispatch(self, func, *args, **kwargs): + return [getattr(im, func)(*args, **kwargs) for im in self.images] + + +def rescale_image_depthmap(image, depthmap, camera_intrinsics, output_resolution, force=True): + """ Jointly rescale a (image, depthmap) + so that (out_width, out_height) >= output_res + """ + image = ImageList(image) + input_resolution = np.array(image.size) # (W,H) + output_resolution = np.array(output_resolution) + if depthmap is not None: + # can also use this with masks instead of depthmaps + assert tuple(depthmap.shape[:2]) == image.size[::-1] + + # define output resolution + assert output_resolution.shape == (2,) + scale_final = max(output_resolution / image.size) + 1e-8 + if scale_final >= 1 and not force: # image is already smaller than what is asked + return (image.to_pil(), depthmap, camera_intrinsics) + output_resolution = np.floor(input_resolution * scale_final).astype(int) + + # first rescale the image so that it contains the crop + # convert output_resolution to tuple + output_resolution = tuple(output_resolution) + image = image.resize(output_resolution, resample=lanczos if scale_final < 1 else bicubic) + if depthmap is not None: + depthmap = cv2.resize(depthmap, output_resolution, fx=scale_final, + fy=scale_final, interpolation=cv2.INTER_NEAREST) + + # no offset here; simple rescaling + camera_intrinsics = camera_matrix_of_crop( + camera_intrinsics, input_resolution, output_resolution, scaling=scale_final) + + return image.to_pil(), depthmap, camera_intrinsics + + +def camera_matrix_of_crop(input_camera_matrix, input_resolution, output_resolution, scaling=1, offset_factor=0.5, offset=None): + # Margins to offset the origin + margins = np.asarray(input_resolution) * scaling - output_resolution + assert np.all(margins >= 0.0) + if offset is None: + offset = offset_factor * margins + + # Generate new camera parameters + output_camera_matrix_colmap = opencv_to_colmap_intrinsics(input_camera_matrix) + output_camera_matrix_colmap[:2, :] *= scaling + output_camera_matrix_colmap[:2, 2] -= offset + output_camera_matrix = colmap_to_opencv_intrinsics(output_camera_matrix_colmap) + + return output_camera_matrix + + +def crop_image_depthmap(image, depthmap, camera_intrinsics, crop_bbox): + """ + Return a crop of the input view. + """ + image = ImageList(image) + l, t, r, b = crop_bbox + + image = image.crop((l, t, r, b)) + depthmap = depthmap[t:b, l:r] + + camera_intrinsics = camera_intrinsics.copy() + camera_intrinsics[0, 2] -= l + camera_intrinsics[1, 2] -= t + + return image.to_pil(), depthmap, camera_intrinsics + + +def bbox_from_intrinsics_in_out(input_camera_matrix, output_camera_matrix, output_resolution): + out_width, out_height = output_resolution + l, t = np.int32(np.round(input_camera_matrix[:2, 2] - output_camera_matrix[:2, 2])) + crop_bbox = (l, t, l + out_width, t + out_height) + return crop_bbox diff --git a/submodules/mast3r/dust3r/dust3r/datasets/utils/transforms.py b/submodules/mast3r/dust3r/dust3r/datasets/utils/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..eb34f2f01d3f8f829ba71a7e03e181bf18f72c25 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/utils/transforms.py @@ -0,0 +1,11 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# DUST3R default transforms +# -------------------------------------------------------- +import torchvision.transforms as tvf +from dust3r.utils.image import ImgNorm + +# define the standard image transforms +ColorJitter = tvf.Compose([tvf.ColorJitter(0.5, 0.5, 0.5, 0.1), ImgNorm]) diff --git a/submodules/mast3r/dust3r/dust3r/datasets/waymo.py b/submodules/mast3r/dust3r/dust3r/datasets/waymo.py new file mode 100644 index 0000000000000000000000000000000000000000..b9a135152cd8973532405b491450c22942dcd6ca --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/waymo.py @@ -0,0 +1,93 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed WayMo +# dataset at https://github.com/waymo-research/waymo-open-dataset +# See datasets_preprocess/preprocess_waymo.py +# -------------------------------------------------------- +import os.path as osp +import numpy as np + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class Waymo (BaseStereoViewDataset): + """ Dataset of outdoor street scenes, 5 images each time + """ + + def __init__(self, *args, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + self._load_data() + + def _load_data(self): + with np.load(osp.join(self.ROOT, 'waymo_pairs.npz')) as data: + self.scenes = data['scenes'] + self.frames = data['frames'] + self.inv_frames = {frame: i for i, frame in enumerate(data['frames'])} + self.pairs = data['pairs'] # (array of (scene_id, img1_id, img2_id) + assert self.pairs[:, 0].max() == len(self.scenes) - 1 + + def __len__(self): + return len(self.pairs) + + def get_stats(self): + return f'{len(self)} pairs from {len(self.scenes)} scenes' + + def _get_views(self, pair_idx, resolution, rng): + seq, img1, img2 = self.pairs[pair_idx] + seq_path = osp.join(self.ROOT, self.scenes[seq]) + + views = [] + + for view_index in [img1, img2]: + impath = self.frames[view_index] + image = imread_cv2(osp.join(seq_path, impath + ".jpg")) + depthmap = imread_cv2(osp.join(seq_path, impath + ".exr")) + camera_params = np.load(osp.join(seq_path, impath + ".npz")) + + intrinsics = np.float32(camera_params['intrinsics']) + camera_pose = np.float32(camera_params['cam2world']) + + image, depthmap, intrinsics = self._crop_resize_if_necessary( + image, depthmap, intrinsics, resolution, rng, info=(seq_path, impath)) + + views.append(dict( + img=image, + depthmap=depthmap, + camera_pose=camera_pose, # cam2world + camera_intrinsics=intrinsics, + dataset='Waymo', + label=osp.relpath(seq_path, self.ROOT), + instance=impath)) + + return views + + +if __name__ == '__main__': + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = Waymo(split='train', ROOT="data/megadepth_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(idx, view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/submodules/mast3r/dust3r/dust3r/datasets/wildrgbd.py b/submodules/mast3r/dust3r/dust3r/datasets/wildrgbd.py new file mode 100644 index 0000000000000000000000000000000000000000..c41dd0b78402bf8ff1e62c6a50de338aa916e0af --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/datasets/wildrgbd.py @@ -0,0 +1,67 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed WildRGB-D +# dataset at https://github.com/wildrgbd/wildrgbd/ +# See datasets_preprocess/preprocess_wildrgbd.py +# -------------------------------------------------------- +import os.path as osp + +import cv2 +import numpy as np + +from dust3r.datasets.co3d import Co3d +from dust3r.utils.image import imread_cv2 + + +class WildRGBD(Co3d): + def __init__(self, mask_bg=True, *args, ROOT, **kwargs): + super().__init__(mask_bg, *args, ROOT=ROOT, **kwargs) + self.dataset_label = 'WildRGBD' + + def _get_metadatapath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'metadata', f'{view_idx:0>5d}.npz') + + def _get_impath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'rgb', f'{view_idx:0>5d}.jpg') + + def _get_depthpath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'depth', f'{view_idx:0>5d}.png') + + def _get_maskpath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'masks', f'{view_idx:0>5d}.png') + + def _read_depthmap(self, depthpath, input_metadata): + # We store depths in the depth scale of 1000. + # That is, when we load depth image and divide by 1000, we could get depth in meters. + depthmap = imread_cv2(depthpath, cv2.IMREAD_UNCHANGED) + depthmap = depthmap.astype(np.float32) / 1000.0 + return depthmap + + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = WildRGBD(split='train', ROOT="data/wildrgbd_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/submodules/mast3r/dust3r/dust3r/demo.py b/submodules/mast3r/dust3r/dust3r/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..c491be097b71ec38ea981dadf4f456d6e9829d48 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/demo.py @@ -0,0 +1,283 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# gradio demo +# -------------------------------------------------------- +import argparse +import math +import builtins +import datetime +import gradio +import os +import torch +import numpy as np +import functools +import trimesh +import copy +from scipy.spatial.transform import Rotation + +from dust3r.inference import inference +from dust3r.image_pairs import make_pairs +from dust3r.utils.image import load_images, rgb +from dust3r.utils.device import to_numpy +from dust3r.viz import add_scene_cam, CAM_COLORS, OPENGL, pts3d_to_trimesh, cat_meshes +from dust3r.cloud_opt import global_aligner, GlobalAlignerMode + +import matplotlib.pyplot as pl + + +def get_args_parser(): + parser = argparse.ArgumentParser() + parser_url = parser.add_mutually_exclusive_group() + parser_url.add_argument("--local_network", action='store_true', default=False, + help="make app accessible on local network: address will be set to 0.0.0.0") + parser_url.add_argument("--server_name", type=str, default=None, help="server url, default is 127.0.0.1") + parser.add_argument("--image_size", type=int, default=512, choices=[512, 224], help="image size") + parser.add_argument("--server_port", type=int, help=("will start gradio app on this port (if available). " + "If None, will search for an available port starting at 7860."), + default=None) + parser_weights = parser.add_mutually_exclusive_group(required=True) + parser_weights.add_argument("--weights", type=str, help="path to the model weights", default=None) + parser_weights.add_argument("--model_name", type=str, help="name of the model weights", + choices=["DUSt3R_ViTLarge_BaseDecoder_512_dpt", + "DUSt3R_ViTLarge_BaseDecoder_512_linear", + "DUSt3R_ViTLarge_BaseDecoder_224_linear"]) + parser.add_argument("--device", type=str, default='cuda', help="pytorch device") + parser.add_argument("--tmp_dir", type=str, default=None, help="value for tempfile.tempdir") + parser.add_argument("--silent", action='store_true', default=False, + help="silence logs") + return parser + + +def set_print_with_timestamp(time_format="%Y-%m-%d %H:%M:%S"): + builtin_print = builtins.print + + def print_with_timestamp(*args, **kwargs): + now = datetime.datetime.now() + formatted_date_time = now.strftime(time_format) + + builtin_print(f'[{formatted_date_time}] ', end='') # print with time stamp + builtin_print(*args, **kwargs) + + builtins.print = print_with_timestamp + + +def _convert_scene_output_to_glb(outdir, imgs, pts3d, mask, focals, cams2world, cam_size=0.05, + cam_color=None, as_pointcloud=False, + transparent_cams=False, silent=False): + assert len(pts3d) == len(mask) <= len(imgs) <= len(cams2world) == len(focals) + pts3d = to_numpy(pts3d) + imgs = to_numpy(imgs) + focals = to_numpy(focals) + cams2world = to_numpy(cams2world) + + scene = trimesh.Scene() + + # full pointcloud + if as_pointcloud: + pts = np.concatenate([p[m] for p, m in zip(pts3d, mask)]) + col = np.concatenate([p[m] for p, m in zip(imgs, mask)]) + pct = trimesh.PointCloud(pts.reshape(-1, 3), colors=col.reshape(-1, 3)) + scene.add_geometry(pct) + else: + meshes = [] + for i in range(len(imgs)): + meshes.append(pts3d_to_trimesh(imgs[i], pts3d[i], mask[i])) + mesh = trimesh.Trimesh(**cat_meshes(meshes)) + scene.add_geometry(mesh) + + # add each camera + for i, pose_c2w in enumerate(cams2world): + if isinstance(cam_color, list): + camera_edge_color = cam_color[i] + else: + camera_edge_color = cam_color or CAM_COLORS[i % len(CAM_COLORS)] + add_scene_cam(scene, pose_c2w, camera_edge_color, + None if transparent_cams else imgs[i], focals[i], + imsize=imgs[i].shape[1::-1], screen_width=cam_size) + + rot = np.eye(4) + rot[:3, :3] = Rotation.from_euler('y', np.deg2rad(180)).as_matrix() + scene.apply_transform(np.linalg.inv(cams2world[0] @ OPENGL @ rot)) + outfile = os.path.join(outdir, 'scene.glb') + if not silent: + print('(exporting 3D scene to', outfile, ')') + scene.export(file_obj=outfile) + return outfile + + +def get_3D_model_from_scene(outdir, silent, scene, min_conf_thr=3, as_pointcloud=False, mask_sky=False, + clean_depth=False, transparent_cams=False, cam_size=0.05): + """ + extract 3D_model (glb file) from a reconstructed scene + """ + if scene is None: + return None + # post processes + if clean_depth: + scene = scene.clean_pointcloud() + if mask_sky: + scene = scene.mask_sky() + + # get optimized values from scene + rgbimg = scene.imgs + focals = scene.get_focals().cpu() + cams2world = scene.get_im_poses().cpu() + # 3D pointcloud from depthmap, poses and intrinsics + pts3d = to_numpy(scene.get_pts3d()) + scene.min_conf_thr = float(scene.conf_trf(torch.tensor(min_conf_thr))) + msk = to_numpy(scene.get_masks()) + return _convert_scene_output_to_glb(outdir, rgbimg, pts3d, msk, focals, cams2world, as_pointcloud=as_pointcloud, + transparent_cams=transparent_cams, cam_size=cam_size, silent=silent) + + +def get_reconstructed_scene(outdir, model, device, silent, image_size, filelist, schedule, niter, min_conf_thr, + as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size, + scenegraph_type, winsize, refid): + """ + from a list of images, run dust3r inference, global aligner. + then run get_3D_model_from_scene + """ + imgs = load_images(filelist, size=image_size, verbose=not silent) + if len(imgs) == 1: + imgs = [imgs[0], copy.deepcopy(imgs[0])] + imgs[1]['idx'] = 1 + if scenegraph_type == "swin": + scenegraph_type = scenegraph_type + "-" + str(winsize) + elif scenegraph_type == "oneref": + scenegraph_type = scenegraph_type + "-" + str(refid) + + pairs = make_pairs(imgs, scene_graph=scenegraph_type, prefilter=None, symmetrize=True) + output = inference(pairs, model, device, batch_size=1, verbose=not silent) + + mode = GlobalAlignerMode.PointCloudOptimizer if len(imgs) > 2 else GlobalAlignerMode.PairViewer + scene = global_aligner(output, device=device, mode=mode, verbose=not silent) + lr = 0.01 + + if mode == GlobalAlignerMode.PointCloudOptimizer: + loss = scene.compute_global_alignment(init='mst', niter=niter, schedule=schedule, lr=lr) + + outfile = get_3D_model_from_scene(outdir, silent, scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size) + + # also return rgb, depth and confidence imgs + # depth is normalized with the max value for all images + # we apply the jet colormap on the confidence maps + rgbimg = scene.imgs + depths = to_numpy(scene.get_depthmaps()) + confs = to_numpy([c for c in scene.im_conf]) + cmap = pl.get_cmap('jet') + depths_max = max([d.max() for d in depths]) + depths = [d / depths_max for d in depths] + confs_max = max([d.max() for d in confs]) + confs = [cmap(d / confs_max) for d in confs] + + imgs = [] + for i in range(len(rgbimg)): + imgs.append(rgbimg[i]) + imgs.append(rgb(depths[i])) + imgs.append(rgb(confs[i])) + + return scene, outfile, imgs + + +def set_scenegraph_options(inputfiles, winsize, refid, scenegraph_type): + num_files = len(inputfiles) if inputfiles is not None else 1 + max_winsize = max(1, math.ceil((num_files - 1) / 2)) + if scenegraph_type == "swin": + winsize = gradio.Slider(label="Scene Graph: Window Size", value=max_winsize, + minimum=1, maximum=max_winsize, step=1, visible=True) + refid = gradio.Slider(label="Scene Graph: Id", value=0, minimum=0, + maximum=num_files - 1, step=1, visible=False) + elif scenegraph_type == "oneref": + winsize = gradio.Slider(label="Scene Graph: Window Size", value=max_winsize, + minimum=1, maximum=max_winsize, step=1, visible=False) + refid = gradio.Slider(label="Scene Graph: Id", value=0, minimum=0, + maximum=num_files - 1, step=1, visible=True) + else: + winsize = gradio.Slider(label="Scene Graph: Window Size", value=max_winsize, + minimum=1, maximum=max_winsize, step=1, visible=False) + refid = gradio.Slider(label="Scene Graph: Id", value=0, minimum=0, + maximum=num_files - 1, step=1, visible=False) + return winsize, refid + + +def main_demo(tmpdirname, model, device, image_size, server_name, server_port, silent=False): + recon_fun = functools.partial(get_reconstructed_scene, tmpdirname, model, device, silent, image_size) + model_from_scene_fun = functools.partial(get_3D_model_from_scene, tmpdirname, silent) + with gradio.Blocks(css=""".gradio-container {margin: 0 !important; min-width: 100%};""", title="DUSt3R Demo") as demo: + # scene state is save so that you can change conf_thr, cam_size... without rerunning the inference + scene = gradio.State(None) + gradio.HTML('

DUSt3R Demo

') + with gradio.Column(): + inputfiles = gradio.File(file_count="multiple") + with gradio.Row(): + schedule = gradio.Dropdown(["linear", "cosine"], + value='linear', label="schedule", info="For global alignment!") + niter = gradio.Number(value=300, precision=0, minimum=0, maximum=5000, + label="num_iterations", info="For global alignment!") + scenegraph_type = gradio.Dropdown([("complete: all possible image pairs", "complete"), + ("swin: sliding window", "swin"), + ("oneref: match one image with all", "oneref")], + value='complete', label="Scenegraph", + info="Define how to make pairs", + interactive=True) + winsize = gradio.Slider(label="Scene Graph: Window Size", value=1, + minimum=1, maximum=1, step=1, visible=False) + refid = gradio.Slider(label="Scene Graph: Id", value=0, minimum=0, maximum=0, step=1, visible=False) + + run_btn = gradio.Button("Run") + + with gradio.Row(): + # adjust the confidence threshold + min_conf_thr = gradio.Slider(label="min_conf_thr", value=3.0, minimum=1.0, maximum=20, step=0.1) + # adjust the camera size in the output pointcloud + cam_size = gradio.Slider(label="cam_size", value=0.05, minimum=0.001, maximum=0.1, step=0.001) + with gradio.Row(): + as_pointcloud = gradio.Checkbox(value=False, label="As pointcloud") + # two post process implemented + mask_sky = gradio.Checkbox(value=False, label="Mask sky") + clean_depth = gradio.Checkbox(value=True, label="Clean-up depthmaps") + transparent_cams = gradio.Checkbox(value=False, label="Transparent cameras") + + outmodel = gradio.Model3D() + outgallery = gradio.Gallery(label='rgb,depth,confidence', columns=3, height="100%") + + # events + scenegraph_type.change(set_scenegraph_options, + inputs=[inputfiles, winsize, refid, scenegraph_type], + outputs=[winsize, refid]) + inputfiles.change(set_scenegraph_options, + inputs=[inputfiles, winsize, refid, scenegraph_type], + outputs=[winsize, refid]) + run_btn.click(fn=recon_fun, + inputs=[inputfiles, schedule, niter, min_conf_thr, as_pointcloud, + mask_sky, clean_depth, transparent_cams, cam_size, + scenegraph_type, winsize, refid], + outputs=[scene, outmodel, outgallery]) + min_conf_thr.release(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size], + outputs=outmodel) + cam_size.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size], + outputs=outmodel) + as_pointcloud.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size], + outputs=outmodel) + mask_sky.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size], + outputs=outmodel) + clean_depth.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size], + outputs=outmodel) + transparent_cams.change(model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size], + outputs=outmodel) + demo.launch(share=False, server_name=server_name, server_port=server_port) diff --git a/submodules/mast3r/dust3r/dust3r/heads/__init__.py b/submodules/mast3r/dust3r/dust3r/heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..53d0aa5610cae95f34f96bdb3ff9e835a2d6208e --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/heads/__init__.py @@ -0,0 +1,19 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# head factory +# -------------------------------------------------------- +from .linear_head import LinearPts3d +from .dpt_head import create_dpt_head + + +def head_factory(head_type, output_mode, net, has_conf=False): + """" build a prediction head for the decoder + """ + if head_type == 'linear' and output_mode == 'pts3d': + return LinearPts3d(net, has_conf) + elif head_type == 'dpt' and output_mode == 'pts3d': + return create_dpt_head(net, has_conf=has_conf) + else: + raise NotImplementedError(f"unexpected {head_type=} and {output_mode=}") diff --git a/submodules/mast3r/dust3r/dust3r/heads/dpt_head.py b/submodules/mast3r/dust3r/dust3r/heads/dpt_head.py new file mode 100644 index 0000000000000000000000000000000000000000..b7bdc9ff587eef3ec8978a22f63659fbf3c277d6 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/heads/dpt_head.py @@ -0,0 +1,115 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# dpt head implementation for DUST3R +# Downstream heads assume inputs of size B x N x C (where N is the number of tokens) ; +# or if it takes as input the output at every layer, the attribute return_all_layers should be set to True +# the forward function also takes as input a dictionnary img_info with key "height" and "width" +# for PixelwiseTask, the output will be of dimension B x num_channels x H x W +# -------------------------------------------------------- +from einops import rearrange +from typing import List +import torch +import torch.nn as nn +from dust3r.heads.postprocess import postprocess +import dust3r.utils.path_to_croco # noqa: F401 +from models.dpt_block import DPTOutputAdapter # noqa + + +class DPTOutputAdapter_fix(DPTOutputAdapter): + """ + Adapt croco's DPTOutputAdapter implementation for dust3r: + remove duplicated weigths, and fix forward for dust3r + """ + + def init(self, dim_tokens_enc=768): + super().init(dim_tokens_enc) + # these are duplicated weights + del self.act_1_postprocess + del self.act_2_postprocess + del self.act_3_postprocess + del self.act_4_postprocess + + def forward(self, encoder_tokens: List[torch.Tensor], image_size=None): + assert self.dim_tokens_enc is not None, 'Need to call init(dim_tokens_enc) function first' + # H, W = input_info['image_size'] + image_size = self.image_size if image_size is None else image_size + H, W = image_size + # Number of patches in height and width + N_H = H // (self.stride_level * self.P_H) + N_W = W // (self.stride_level * self.P_W) + + # Hook decoder onto 4 layers from specified ViT layers + layers = [encoder_tokens[hook] for hook in self.hooks] + + # Extract only task-relevant tokens and ignore global tokens. + layers = [self.adapt_tokens(l) for l in layers] + + # Reshape tokens to spatial representation + layers = [rearrange(l, 'b (nh nw) c -> b c nh nw', nh=N_H, nw=N_W) for l in layers] + + layers = [self.act_postprocess[idx](l) for idx, l in enumerate(layers)] + # Project layers to chosen feature dim + layers = [self.scratch.layer_rn[idx](l) for idx, l in enumerate(layers)] + + # Fuse layers using refinement stages + path_4 = self.scratch.refinenet4(layers[3])[:, :, :layers[2].shape[2], :layers[2].shape[3]] + path_3 = self.scratch.refinenet3(path_4, layers[2]) + path_2 = self.scratch.refinenet2(path_3, layers[1]) + path_1 = self.scratch.refinenet1(path_2, layers[0]) + + # Output head + out = self.head(path_1) + + return out + + +class PixelwiseTaskWithDPT(nn.Module): + """ DPT module for dust3r, can return 3D points + confidence for all pixels""" + + def __init__(self, *, n_cls_token=0, hooks_idx=None, dim_tokens=None, + output_width_ratio=1, num_channels=1, postprocess=None, depth_mode=None, conf_mode=None, **kwargs): + super(PixelwiseTaskWithDPT, self).__init__() + self.return_all_layers = True # backbone needs to return all layers + self.postprocess = postprocess + self.depth_mode = depth_mode + self.conf_mode = conf_mode + + assert n_cls_token == 0, "Not implemented" + dpt_args = dict(output_width_ratio=output_width_ratio, + num_channels=num_channels, + **kwargs) + if hooks_idx is not None: + dpt_args.update(hooks=hooks_idx) + self.dpt = DPTOutputAdapter_fix(**dpt_args) + dpt_init_args = {} if dim_tokens is None else {'dim_tokens_enc': dim_tokens} + self.dpt.init(**dpt_init_args) + + def forward(self, x, img_info): + out = self.dpt(x, image_size=(img_info[0], img_info[1])) + if self.postprocess: + out = self.postprocess(out, self.depth_mode, self.conf_mode) + return out + + +def create_dpt_head(net, has_conf=False): + """ + return PixelwiseTaskWithDPT for given net params + """ + assert net.dec_depth > 9 + l2 = net.dec_depth + feature_dim = 256 + last_dim = feature_dim//2 + out_nchan = 3 + ed = net.enc_embed_dim + dd = net.dec_embed_dim + return PixelwiseTaskWithDPT(num_channels=out_nchan + has_conf, + feature_dim=feature_dim, + last_dim=last_dim, + hooks_idx=[0, l2*2//4, l2*3//4, l2], + dim_tokens=[ed, dd, dd, dd], + postprocess=postprocess, + depth_mode=net.depth_mode, + conf_mode=net.conf_mode, + head_type='regression') diff --git a/submodules/mast3r/dust3r/dust3r/heads/linear_head.py b/submodules/mast3r/dust3r/dust3r/heads/linear_head.py new file mode 100644 index 0000000000000000000000000000000000000000..6b697f29eaa6f43fad0a3e27a8d9b8f1a602a833 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/heads/linear_head.py @@ -0,0 +1,41 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# linear head implementation for DUST3R +# -------------------------------------------------------- +import torch.nn as nn +import torch.nn.functional as F +from dust3r.heads.postprocess import postprocess + + +class LinearPts3d (nn.Module): + """ + Linear head for dust3r + Each token outputs: - 16x16 3D points (+ confidence) + """ + + def __init__(self, net, has_conf=False): + super().__init__() + self.patch_size = net.patch_embed.patch_size[0] + self.depth_mode = net.depth_mode + self.conf_mode = net.conf_mode + self.has_conf = has_conf + + self.proj = nn.Linear(net.dec_embed_dim, (3 + has_conf)*self.patch_size**2) + + def setup(self, croconet): + pass + + def forward(self, decout, img_shape): + H, W = img_shape + tokens = decout[-1] + B, S, D = tokens.shape + + # extract 3D points + feat = self.proj(tokens) # B,S,D + feat = feat.transpose(-1, -2).view(B, -1, H//self.patch_size, W//self.patch_size) + feat = F.pixel_shuffle(feat, self.patch_size) # B,3,H,W + + # permute + norm depth + return postprocess(feat, self.depth_mode, self.conf_mode) diff --git a/submodules/mast3r/dust3r/dust3r/heads/postprocess.py b/submodules/mast3r/dust3r/dust3r/heads/postprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..cd68a90d89b8dcd7d8a4b4ea06ef8b17eb5da093 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/heads/postprocess.py @@ -0,0 +1,58 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# post process function for all heads: extract 3D points/confidence from output +# -------------------------------------------------------- +import torch + + +def postprocess(out, depth_mode, conf_mode): + """ + extract 3D points/confidence from prediction head output + """ + fmap = out.permute(0, 2, 3, 1) # B,H,W,3 + res = dict(pts3d=reg_dense_depth(fmap[:, :, :, 0:3], mode=depth_mode)) + + if conf_mode is not None: + res['conf'] = reg_dense_conf(fmap[:, :, :, 3], mode=conf_mode) + return res + + +def reg_dense_depth(xyz, mode): + """ + extract 3D points from prediction head output + """ + mode, vmin, vmax = mode + + no_bounds = (vmin == -float('inf')) and (vmax == float('inf')) + assert no_bounds + + if mode == 'linear': + if no_bounds: + return xyz # [-inf, +inf] + return xyz.clip(min=vmin, max=vmax) + + # distance to origin + d = xyz.norm(dim=-1, keepdim=True) + xyz = xyz / d.clip(min=1e-8) + + if mode == 'square': + return xyz * d.square() + + if mode == 'exp': + return xyz * torch.expm1(d) + + raise ValueError(f'bad {mode=}') + + +def reg_dense_conf(x, mode): + """ + extract confidence from prediction head output + """ + mode, vmin, vmax = mode + if mode == 'exp': + return vmin + x.exp().clip(max=vmax-vmin) + if mode == 'sigmoid': + return (vmax - vmin) * torch.sigmoid(x) + vmin + raise ValueError(f'bad {mode=}') diff --git a/submodules/mast3r/dust3r/dust3r/image_pairs.py b/submodules/mast3r/dust3r/dust3r/image_pairs.py new file mode 100644 index 0000000000000000000000000000000000000000..ebcf902b4d07b83fe83ffceba3f45ca0d74dfcf7 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/image_pairs.py @@ -0,0 +1,104 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilities needed to load image pairs +# -------------------------------------------------------- +import numpy as np +import torch + + +def make_pairs(imgs, scene_graph='complete', prefilter=None, symmetrize=True): + pairs = [] + if scene_graph == 'complete': # complete graph + for i in range(len(imgs)): + for j in range(i): + pairs.append((imgs[i], imgs[j])) + elif scene_graph.startswith('swin'): + iscyclic = not scene_graph.endswith('noncyclic') + try: + winsize = int(scene_graph.split('-')[1]) + except Exception as e: + winsize = 3 + pairsid = set() + for i in range(len(imgs)): + for j in range(1, winsize + 1): + idx = (i + j) + if iscyclic: + idx = idx % len(imgs) # explicit loop closure + if idx >= len(imgs): + continue + pairsid.add((i, idx) if i < idx else (idx, i)) + for i, j in pairsid: + pairs.append((imgs[i], imgs[j])) + elif scene_graph.startswith('logwin'): + iscyclic = not scene_graph.endswith('noncyclic') + try: + winsize = int(scene_graph.split('-')[1]) + except Exception as e: + winsize = 3 + offsets = [2**i for i in range(winsize)] + pairsid = set() + for i in range(len(imgs)): + ixs_l = [i - off for off in offsets] + ixs_r = [i + off for off in offsets] + for j in ixs_l + ixs_r: + if iscyclic: + j = j % len(imgs) # Explicit loop closure + if j < 0 or j >= len(imgs) or j == i: + continue + pairsid.add((i, j) if i < j else (j, i)) + for i, j in pairsid: + pairs.append((imgs[i], imgs[j])) + elif scene_graph.startswith('oneref'): + refid = int(scene_graph.split('-')[1]) if '-' in scene_graph else 0 + for j in range(len(imgs)): + if j != refid: + pairs.append((imgs[refid], imgs[j])) + if symmetrize: + pairs += [(img2, img1) for img1, img2 in pairs] + + # now, remove edges + if isinstance(prefilter, str) and prefilter.startswith('seq'): + pairs = filter_pairs_seq(pairs, int(prefilter[3:])) + + if isinstance(prefilter, str) and prefilter.startswith('cyc'): + pairs = filter_pairs_seq(pairs, int(prefilter[3:]), cyclic=True) + + return pairs + + +def sel(x, kept): + if isinstance(x, dict): + return {k: sel(v, kept) for k, v in x.items()} + if isinstance(x, (torch.Tensor, np.ndarray)): + return x[kept] + if isinstance(x, (tuple, list)): + return type(x)([x[k] for k in kept]) + + +def _filter_edges_seq(edges, seq_dis_thr, cyclic=False): + # number of images + n = max(max(e) for e in edges) + 1 + + kept = [] + for e, (i, j) in enumerate(edges): + dis = abs(i - j) + if cyclic: + dis = min(dis, abs(i + n - j), abs(i - n - j)) + if dis <= seq_dis_thr: + kept.append(e) + return kept + + +def filter_pairs_seq(pairs, seq_dis_thr, cyclic=False): + edges = [(img1['idx'], img2['idx']) for img1, img2 in pairs] + kept = _filter_edges_seq(edges, seq_dis_thr, cyclic=cyclic) + return [pairs[i] for i in kept] + + +def filter_edges_seq(view1, view2, pred1, pred2, seq_dis_thr, cyclic=False): + edges = [(int(i), int(j)) for i, j in zip(view1['idx'], view2['idx'])] + kept = _filter_edges_seq(edges, seq_dis_thr, cyclic=cyclic) + print(f'>> Filtering edges more than {seq_dis_thr} frames apart: kept {len(kept)}/{len(edges)} edges') + return sel(view1, kept), sel(view2, kept), sel(pred1, kept), sel(pred2, kept) diff --git a/submodules/mast3r/dust3r/dust3r/inference.py b/submodules/mast3r/dust3r/dust3r/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..90540486b077add90ca50f62a5072e082cb2f2d7 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/inference.py @@ -0,0 +1,150 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilities needed for the inference +# -------------------------------------------------------- +import tqdm +import torch +from dust3r.utils.device import to_cpu, collate_with_cat +from dust3r.utils.misc import invalid_to_nans +from dust3r.utils.geometry import depthmap_to_pts3d, geotrf + + +def _interleave_imgs(img1, img2): + res = {} + for key, value1 in img1.items(): + value2 = img2[key] + if isinstance(value1, torch.Tensor): + value = torch.stack((value1, value2), dim=1).flatten(0, 1) + else: + value = [x for pair in zip(value1, value2) for x in pair] + res[key] = value + return res + + +def make_batch_symmetric(batch): + view1, view2 = batch + view1, view2 = (_interleave_imgs(view1, view2), _interleave_imgs(view2, view1)) + return view1, view2 + + +def loss_of_one_batch(batch, model, criterion, device, symmetrize_batch=False, use_amp=False, ret=None): + view1, view2 = batch + ignore_keys = set(['depthmap', 'dataset', 'label', 'instance', 'idx', 'true_shape', 'rng']) + for view in batch: + for name in view.keys(): # pseudo_focal + if name in ignore_keys: + continue + view[name] = view[name].to(device, non_blocking=True) + + if symmetrize_batch: + view1, view2 = make_batch_symmetric(batch) + + with torch.cuda.amp.autocast(enabled=bool(use_amp)): + pred1, pred2 = model(view1, view2) + + # loss is supposed to be symmetric + with torch.cuda.amp.autocast(enabled=False): + loss = criterion(view1, view2, pred1, pred2) if criterion is not None else None + + result = dict(view1=view1, view2=view2, pred1=pred1, pred2=pred2, loss=loss) + return result[ret] if ret else result + + +@torch.no_grad() +def inference(pairs, model, device, batch_size=8, verbose=True): + if verbose: + print(f'>> Inference with model on {len(pairs)} image pairs') + result = [] + + # first, check if all images have the same size + multiple_shapes = not (check_if_same_size(pairs)) + if multiple_shapes: # force bs=1 + batch_size = 1 + + for i in tqdm.trange(0, len(pairs), batch_size, disable=not verbose): + res = loss_of_one_batch(collate_with_cat(pairs[i:i + batch_size]), model, None, device) + result.append(to_cpu(res)) + + result = collate_with_cat(result, lists=multiple_shapes) + + return result + + +def check_if_same_size(pairs): + shapes1 = [img1['img'].shape[-2:] for img1, img2 in pairs] + shapes2 = [img2['img'].shape[-2:] for img1, img2 in pairs] + return all(shapes1[0] == s for s in shapes1) and all(shapes2[0] == s for s in shapes2) + + +def get_pred_pts3d(gt, pred, use_pose=False): + if 'depth' in pred and 'pseudo_focal' in pred: + try: + pp = gt['camera_intrinsics'][..., :2, 2] + except KeyError: + pp = None + pts3d = depthmap_to_pts3d(**pred, pp=pp) + + elif 'pts3d' in pred: + # pts3d from my camera + pts3d = pred['pts3d'] + + elif 'pts3d_in_other_view' in pred: + # pts3d from the other camera, already transformed + assert use_pose is True + return pred['pts3d_in_other_view'] # return! + + if use_pose: + camera_pose = pred.get('camera_pose') + assert camera_pose is not None + pts3d = geotrf(camera_pose, pts3d) + + return pts3d + + +def find_opt_scaling(gt_pts1, gt_pts2, pr_pts1, pr_pts2=None, fit_mode='weiszfeld_stop_grad', valid1=None, valid2=None): + assert gt_pts1.ndim == pr_pts1.ndim == 4 + assert gt_pts1.shape == pr_pts1.shape + if gt_pts2 is not None: + assert gt_pts2.ndim == pr_pts2.ndim == 4 + assert gt_pts2.shape == pr_pts2.shape + + # concat the pointcloud + nan_gt_pts1 = invalid_to_nans(gt_pts1, valid1).flatten(1, 2) + nan_gt_pts2 = invalid_to_nans(gt_pts2, valid2).flatten(1, 2) if gt_pts2 is not None else None + + pr_pts1 = invalid_to_nans(pr_pts1, valid1).flatten(1, 2) + pr_pts2 = invalid_to_nans(pr_pts2, valid2).flatten(1, 2) if pr_pts2 is not None else None + + all_gt = torch.cat((nan_gt_pts1, nan_gt_pts2), dim=1) if gt_pts2 is not None else nan_gt_pts1 + all_pr = torch.cat((pr_pts1, pr_pts2), dim=1) if pr_pts2 is not None else pr_pts1 + + dot_gt_pr = (all_pr * all_gt).sum(dim=-1) + dot_gt_gt = all_gt.square().sum(dim=-1) + + if fit_mode.startswith('avg'): + # scaling = (all_pr / all_gt).view(B, -1).mean(dim=1) + scaling = dot_gt_pr.nanmean(dim=1) / dot_gt_gt.nanmean(dim=1) + elif fit_mode.startswith('median'): + scaling = (dot_gt_pr / dot_gt_gt).nanmedian(dim=1).values + elif fit_mode.startswith('weiszfeld'): + # init scaling with l2 closed form + scaling = dot_gt_pr.nanmean(dim=1) / dot_gt_gt.nanmean(dim=1) + # iterative re-weighted least-squares + for iter in range(10): + # re-weighting by inverse of distance + dis = (all_pr - scaling.view(-1, 1, 1) * all_gt).norm(dim=-1) + # print(dis.nanmean(-1)) + w = dis.clip_(min=1e-8).reciprocal() + # update the scaling with the new weights + scaling = (w * dot_gt_pr).nanmean(dim=1) / (w * dot_gt_gt).nanmean(dim=1) + else: + raise ValueError(f'bad {fit_mode=}') + + if fit_mode.endswith('stop_grad'): + scaling = scaling.detach() + + scaling = scaling.clip(min=1e-3) + # assert scaling.isfinite().all(), bb() + return scaling diff --git a/submodules/mast3r/dust3r/dust3r/losses.py b/submodules/mast3r/dust3r/dust3r/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..4f8febff1a2dd674e759bcf83d023099a59cc934 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/losses.py @@ -0,0 +1,299 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Implementation of DUSt3R training losses +# -------------------------------------------------------- +from copy import copy, deepcopy +import torch +import torch.nn as nn + +from dust3r.inference import get_pred_pts3d, find_opt_scaling +from dust3r.utils.geometry import inv, geotrf, normalize_pointcloud +from dust3r.utils.geometry import get_joint_pointcloud_depth, get_joint_pointcloud_center_scale + + +def Sum(*losses_and_masks): + loss, mask = losses_and_masks[0] + if loss.ndim > 0: + # we are actually returning the loss for every pixels + return losses_and_masks + else: + # we are returning the global loss + for loss2, mask2 in losses_and_masks[1:]: + loss = loss + loss2 + return loss + + +class BaseCriterion(nn.Module): + def __init__(self, reduction='mean'): + super().__init__() + self.reduction = reduction + + +class LLoss (BaseCriterion): + """ L-norm loss + """ + + def forward(self, a, b): + assert a.shape == b.shape and a.ndim >= 2 and 1 <= a.shape[-1] <= 3, f'Bad shape = {a.shape}' + dist = self.distance(a, b) + assert dist.ndim == a.ndim - 1 # one dimension less + if self.reduction == 'none': + return dist + if self.reduction == 'sum': + return dist.sum() + if self.reduction == 'mean': + return dist.mean() if dist.numel() > 0 else dist.new_zeros(()) + raise ValueError(f'bad {self.reduction=} mode') + + def distance(self, a, b): + raise NotImplementedError() + + +class L21Loss (LLoss): + """ Euclidean distance between 3d points """ + + def distance(self, a, b): + return torch.norm(a - b, dim=-1) # normalized L2 distance + + +L21 = L21Loss() + + +class Criterion (nn.Module): + def __init__(self, criterion=None): + super().__init__() + assert isinstance(criterion, BaseCriterion), f'{criterion} is not a proper criterion!' + self.criterion = copy(criterion) + + def get_name(self): + return f'{type(self).__name__}({self.criterion})' + + def with_reduction(self, mode='none'): + res = loss = deepcopy(self) + while loss is not None: + assert isinstance(loss, Criterion) + loss.criterion.reduction = mode # make it return the loss for each sample + loss = loss._loss2 # we assume loss is a Multiloss + return res + + +class MultiLoss (nn.Module): + """ Easily combinable losses (also keep track of individual loss values): + loss = MyLoss1() + 0.1*MyLoss2() + Usage: + Inherit from this class and override get_name() and compute_loss() + """ + + def __init__(self): + super().__init__() + self._alpha = 1 + self._loss2 = None + + def compute_loss(self, *args, **kwargs): + raise NotImplementedError() + + def get_name(self): + raise NotImplementedError() + + def __mul__(self, alpha): + assert isinstance(alpha, (int, float)) + res = copy(self) + res._alpha = alpha + return res + __rmul__ = __mul__ # same + + def __add__(self, loss2): + assert isinstance(loss2, MultiLoss) + res = cur = copy(self) + # find the end of the chain + while cur._loss2 is not None: + cur = cur._loss2 + cur._loss2 = loss2 + return res + + def __repr__(self): + name = self.get_name() + if self._alpha != 1: + name = f'{self._alpha:g}*{name}' + if self._loss2: + name = f'{name} + {self._loss2}' + return name + + def forward(self, *args, **kwargs): + loss = self.compute_loss(*args, **kwargs) + if isinstance(loss, tuple): + loss, details = loss + elif loss.ndim == 0: + details = {self.get_name(): float(loss)} + else: + details = {} + loss = loss * self._alpha + + if self._loss2: + loss2, details2 = self._loss2(*args, **kwargs) + loss = loss + loss2 + details |= details2 + + return loss, details + + +class Regr3D (Criterion, MultiLoss): + """ Ensure that all 3D points are correct. + Asymmetric loss: view1 is supposed to be the anchor. + + P1 = RT1 @ D1 + P2 = RT2 @ D2 + loss1 = (I @ pred_D1) - (RT1^-1 @ RT1 @ D1) + loss2 = (RT21 @ pred_D2) - (RT1^-1 @ P2) + = (RT21 @ pred_D2) - (RT1^-1 @ RT2 @ D2) + """ + + def __init__(self, criterion, norm_mode='avg_dis', gt_scale=False): + super().__init__(criterion) + self.norm_mode = norm_mode + self.gt_scale = gt_scale + + def get_all_pts3d(self, gt1, gt2, pred1, pred2, dist_clip=None): + # everything is normalized w.r.t. camera of view1 + in_camera1 = inv(gt1['camera_pose']) + gt_pts1 = geotrf(in_camera1, gt1['pts3d']) # B,H,W,3 + gt_pts2 = geotrf(in_camera1, gt2['pts3d']) # B,H,W,3 + + valid1 = gt1['valid_mask'].clone() + valid2 = gt2['valid_mask'].clone() + + if dist_clip is not None: + # points that are too far-away == invalid + dis1 = gt_pts1.norm(dim=-1) # (B, H, W) + dis2 = gt_pts2.norm(dim=-1) # (B, H, W) + valid1 = valid1 & (dis1 <= dist_clip) + valid2 = valid2 & (dis2 <= dist_clip) + + pr_pts1 = get_pred_pts3d(gt1, pred1, use_pose=False) + pr_pts2 = get_pred_pts3d(gt2, pred2, use_pose=True) + + # normalize 3d points + if self.norm_mode: + pr_pts1, pr_pts2 = normalize_pointcloud(pr_pts1, pr_pts2, self.norm_mode, valid1, valid2) + if self.norm_mode and not self.gt_scale: + gt_pts1, gt_pts2 = normalize_pointcloud(gt_pts1, gt_pts2, self.norm_mode, valid1, valid2) + + return gt_pts1, gt_pts2, pr_pts1, pr_pts2, valid1, valid2, {} + + def compute_loss(self, gt1, gt2, pred1, pred2, **kw): + gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, monitoring = \ + self.get_all_pts3d(gt1, gt2, pred1, pred2, **kw) + # loss on img1 side + l1 = self.criterion(pred_pts1[mask1], gt_pts1[mask1]) + # loss on gt2 side + l2 = self.criterion(pred_pts2[mask2], gt_pts2[mask2]) + self_name = type(self).__name__ + details = {self_name + '_pts3d_1': float(l1.mean()), self_name + '_pts3d_2': float(l2.mean())} + return Sum((l1, mask1), (l2, mask2)), (details | monitoring) + + +class ConfLoss (MultiLoss): + """ Weighted regression by learned confidence. + Assuming the input pixel_loss is a pixel-level regression loss. + + Principle: + high-confidence means high conf = 0.1 ==> conf_loss = x / 10 + alpha*log(10) + low confidence means low conf = 10 ==> conf_loss = x * 10 - alpha*log(10) + + alpha: hyperparameter + """ + + def __init__(self, pixel_loss, alpha=1): + super().__init__() + assert alpha > 0 + self.alpha = alpha + self.pixel_loss = pixel_loss.with_reduction('none') + + def get_name(self): + return f'ConfLoss({self.pixel_loss})' + + def get_conf_log(self, x): + return x, torch.log(x) + + def compute_loss(self, gt1, gt2, pred1, pred2, **kw): + # compute per-pixel loss + ((loss1, msk1), (loss2, msk2)), details = self.pixel_loss(gt1, gt2, pred1, pred2, **kw) + if loss1.numel() == 0: + print('NO VALID POINTS in img1', force=True) + if loss2.numel() == 0: + print('NO VALID POINTS in img2', force=True) + + # weight by confidence + conf1, log_conf1 = self.get_conf_log(pred1['conf'][msk1]) + conf2, log_conf2 = self.get_conf_log(pred2['conf'][msk2]) + conf_loss1 = loss1 * conf1 - self.alpha * log_conf1 + conf_loss2 = loss2 * conf2 - self.alpha * log_conf2 + + # average + nan protection (in case of no valid pixels at all) + conf_loss1 = conf_loss1.mean() if conf_loss1.numel() > 0 else 0 + conf_loss2 = conf_loss2.mean() if conf_loss2.numel() > 0 else 0 + + return conf_loss1 + conf_loss2, dict(conf_loss_1=float(conf_loss1), conf_loss2=float(conf_loss2), **details) + + +class Regr3D_ShiftInv (Regr3D): + """ Same than Regr3D but invariant to depth shift. + """ + + def get_all_pts3d(self, gt1, gt2, pred1, pred2): + # compute unnormalized points + gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, monitoring = \ + super().get_all_pts3d(gt1, gt2, pred1, pred2) + + # compute median depth + gt_z1, gt_z2 = gt_pts1[..., 2], gt_pts2[..., 2] + pred_z1, pred_z2 = pred_pts1[..., 2], pred_pts2[..., 2] + gt_shift_z = get_joint_pointcloud_depth(gt_z1, gt_z2, mask1, mask2)[:, None, None] + pred_shift_z = get_joint_pointcloud_depth(pred_z1, pred_z2, mask1, mask2)[:, None, None] + + # subtract the median depth + gt_z1 -= gt_shift_z + gt_z2 -= gt_shift_z + pred_z1 -= pred_shift_z + pred_z2 -= pred_shift_z + + # monitoring = dict(monitoring, gt_shift_z=gt_shift_z.mean().detach(), pred_shift_z=pred_shift_z.mean().detach()) + return gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, monitoring + + +class Regr3D_ScaleInv (Regr3D): + """ Same than Regr3D but invariant to depth shift. + if gt_scale == True: enforce the prediction to take the same scale than GT + """ + + def get_all_pts3d(self, gt1, gt2, pred1, pred2): + # compute depth-normalized points + gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, monitoring = super().get_all_pts3d(gt1, gt2, pred1, pred2) + + # measure scene scale + _, gt_scale = get_joint_pointcloud_center_scale(gt_pts1, gt_pts2, mask1, mask2) + _, pred_scale = get_joint_pointcloud_center_scale(pred_pts1, pred_pts2, mask1, mask2) + + # prevent predictions to be in a ridiculous range + pred_scale = pred_scale.clip(min=1e-3, max=1e3) + + # subtract the median depth + if self.gt_scale: + pred_pts1 *= gt_scale / pred_scale + pred_pts2 *= gt_scale / pred_scale + # monitoring = dict(monitoring, pred_scale=(pred_scale/gt_scale).mean()) + else: + gt_pts1 /= gt_scale + gt_pts2 /= gt_scale + pred_pts1 /= pred_scale + pred_pts2 /= pred_scale + # monitoring = dict(monitoring, gt_scale=gt_scale.mean(), pred_scale=pred_scale.mean().detach()) + + return gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, monitoring + + +class Regr3D_ScaleShiftInv (Regr3D_ScaleInv, Regr3D_ShiftInv): + # calls Regr3D_ShiftInv first, then Regr3D_ScaleInv + pass diff --git a/submodules/mast3r/dust3r/dust3r/model.py b/submodules/mast3r/dust3r/dust3r/model.py new file mode 100644 index 0000000000000000000000000000000000000000..41c3a4f78eb5fbafdeb7ab8523468de320886c64 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/model.py @@ -0,0 +1,210 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# DUSt3R model class +# -------------------------------------------------------- +from copy import deepcopy +import torch +import os +from packaging import version +import huggingface_hub + +from .utils.misc import fill_default_args, freeze_all_params, is_symmetrized, interleave, transpose_to_landscape +from .heads import head_factory +from dust3r.patch_embed import get_patch_embed + +import dust3r.utils.path_to_croco # noqa: F401 +from models.croco import CroCoNet # noqa + +inf = float('inf') + +hf_version_number = huggingface_hub.__version__ +assert version.parse(hf_version_number) >= version.parse("0.22.0"), ("Outdated huggingface_hub version, " + "please reinstall requirements.txt") + + +def load_model(model_path, device, verbose=True): + if verbose: + print('... loading model from', model_path) + ckpt = torch.load(model_path, map_location='cpu') + args = ckpt['args'].model.replace("ManyAR_PatchEmbed", "PatchEmbedDust3R") + if 'landscape_only' not in args: + args = args[:-1] + ', landscape_only=False)' + else: + args = args.replace(" ", "").replace('landscape_only=True', 'landscape_only=False') + assert "landscape_only=False" in args + if verbose: + print(f"instantiating : {args}") + net = eval(args) + s = net.load_state_dict(ckpt['model'], strict=False) + if verbose: + print(s) + return net.to(device) + + +class AsymmetricCroCo3DStereo ( + CroCoNet, + huggingface_hub.PyTorchModelHubMixin, + library_name="dust3r", + repo_url="https://github.com/naver/dust3r", + tags=["image-to-3d"], +): + """ Two siamese encoders, followed by two decoders. + The goal is to output 3d points directly, both images in view1's frame + (hence the asymmetry). + """ + + def __init__(self, + output_mode='pts3d', + head_type='linear', + depth_mode=('exp', -inf, inf), + conf_mode=('exp', 1, inf), + freeze='none', + landscape_only=True, + patch_embed_cls='PatchEmbedDust3R', # PatchEmbedDust3R or ManyAR_PatchEmbed + **croco_kwargs): + self.patch_embed_cls = patch_embed_cls + self.croco_args = fill_default_args(croco_kwargs, super().__init__) + super().__init__(**croco_kwargs) + + # dust3r specific initialization + self.dec_blocks2 = deepcopy(self.dec_blocks) + self.set_downstream_head(output_mode, head_type, landscape_only, depth_mode, conf_mode, **croco_kwargs) + self.set_freeze(freeze) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, **kw): + if os.path.isfile(pretrained_model_name_or_path): + return load_model(pretrained_model_name_or_path, device='cpu') + else: + try: + model = super(AsymmetricCroCo3DStereo, cls).from_pretrained(pretrained_model_name_or_path, **kw) + except TypeError as e: + raise Exception(f'tried to load {pretrained_model_name_or_path} from huggingface, but failed') + return model + + def _set_patch_embed(self, img_size=224, patch_size=16, enc_embed_dim=768): + self.patch_embed = get_patch_embed(self.patch_embed_cls, img_size, patch_size, enc_embed_dim) + + def load_state_dict(self, ckpt, **kw): + # duplicate all weights for the second decoder if not present + new_ckpt = dict(ckpt) + if not any(k.startswith('dec_blocks2') for k in ckpt): + for key, value in ckpt.items(): + if key.startswith('dec_blocks'): + new_ckpt[key.replace('dec_blocks', 'dec_blocks2')] = value + return super().load_state_dict(new_ckpt, **kw) + + def set_freeze(self, freeze): # this is for use by downstream models + self.freeze = freeze + to_be_frozen = { + 'none': [], + 'mask': [self.mask_token], + 'encoder': [self.mask_token, self.patch_embed, self.enc_blocks], + } + freeze_all_params(to_be_frozen[freeze]) + + def _set_prediction_head(self, *args, **kwargs): + """ No prediction head """ + return + + def set_downstream_head(self, output_mode, head_type, landscape_only, depth_mode, conf_mode, patch_size, img_size, + **kw): + assert img_size[0] % patch_size == 0 and img_size[1] % patch_size == 0, \ + f'{img_size=} must be multiple of {patch_size=}' + self.output_mode = output_mode + self.head_type = head_type + self.depth_mode = depth_mode + self.conf_mode = conf_mode + # allocate heads + self.downstream_head1 = head_factory(head_type, output_mode, self, has_conf=bool(conf_mode)) + self.downstream_head2 = head_factory(head_type, output_mode, self, has_conf=bool(conf_mode)) + # magic wrapper + self.head1 = transpose_to_landscape(self.downstream_head1, activate=landscape_only) + self.head2 = transpose_to_landscape(self.downstream_head2, activate=landscape_only) + + def _encode_image(self, image, true_shape): + # embed the image into patches (x has size B x Npatches x C) + x, pos = self.patch_embed(image, true_shape=true_shape) + + # add positional embedding without cls token + assert self.enc_pos_embed is None + + # now apply the transformer encoder and normalization + for blk in self.enc_blocks: + x = blk(x, pos) + + x = self.enc_norm(x) + return x, pos, None + + def _encode_image_pairs(self, img1, img2, true_shape1, true_shape2): + if img1.shape[-2:] == img2.shape[-2:]: + out, pos, _ = self._encode_image(torch.cat((img1, img2), dim=0), + torch.cat((true_shape1, true_shape2), dim=0)) + out, out2 = out.chunk(2, dim=0) + pos, pos2 = pos.chunk(2, dim=0) + else: + out, pos, _ = self._encode_image(img1, true_shape1) + out2, pos2, _ = self._encode_image(img2, true_shape2) + return out, out2, pos, pos2 + + def _encode_symmetrized(self, view1, view2): + img1 = view1['img'] + img2 = view2['img'] + B = img1.shape[0] + # Recover true_shape when available, otherwise assume that the img shape is the true one + shape1 = view1.get('true_shape', torch.tensor(img1.shape[-2:])[None].repeat(B, 1)) + shape2 = view2.get('true_shape', torch.tensor(img2.shape[-2:])[None].repeat(B, 1)) + # warning! maybe the images have different portrait/landscape orientations + + if is_symmetrized(view1, view2): + # computing half of forward pass!' + feat1, feat2, pos1, pos2 = self._encode_image_pairs(img1[::2], img2[::2], shape1[::2], shape2[::2]) + feat1, feat2 = interleave(feat1, feat2) + pos1, pos2 = interleave(pos1, pos2) + else: + feat1, feat2, pos1, pos2 = self._encode_image_pairs(img1, img2, shape1, shape2) + + return (shape1, shape2), (feat1, feat2), (pos1, pos2) + + def _decoder(self, f1, pos1, f2, pos2): + final_output = [(f1, f2)] # before projection + + # project to decoder dim + f1 = self.decoder_embed(f1) + f2 = self.decoder_embed(f2) + + final_output.append((f1, f2)) + for blk1, blk2 in zip(self.dec_blocks, self.dec_blocks2): + # img1 side + f1, _ = blk1(*final_output[-1][::+1], pos1, pos2) + # img2 side + f2, _ = blk2(*final_output[-1][::-1], pos2, pos1) + # store the result + final_output.append((f1, f2)) + + # normalize last output + del final_output[1] # duplicate with final_output[0] + final_output[-1] = tuple(map(self.dec_norm, final_output[-1])) + return zip(*final_output) + + def _downstream_head(self, head_num, decout, img_shape): + B, S, D = decout[-1].shape + # img_shape = tuple(map(int, img_shape)) + head = getattr(self, f'head{head_num}') + return head(decout, img_shape) + + def forward(self, view1, view2): + # encode the two images --> B,S,D + (shape1, shape2), (feat1, feat2), (pos1, pos2) = self._encode_symmetrized(view1, view2) + + # combine all ref images into object-centric representation + dec1, dec2 = self._decoder(feat1, pos1, feat2, pos2) + + with torch.cuda.amp.autocast(enabled=False): + res1 = self._downstream_head(1, [tok.float() for tok in dec1], shape1) + res2 = self._downstream_head(2, [tok.float() for tok in dec2], shape2) + + res2['pts3d_in_other_view'] = res2.pop('pts3d') # predict view2's pts3d in view1's frame + return res1, res2 diff --git a/submodules/mast3r/dust3r/dust3r/optim_factory.py b/submodules/mast3r/dust3r/dust3r/optim_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..9b9c16e0e0fda3fd03c3def61abc1f354f75c584 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/optim_factory.py @@ -0,0 +1,14 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# optimization functions +# -------------------------------------------------------- + + +def adjust_learning_rate_by_lr(optimizer, lr): + for param_group in optimizer.param_groups: + if "lr_scale" in param_group: + param_group["lr"] = lr * param_group["lr_scale"] + else: + param_group["lr"] = lr diff --git a/submodules/mast3r/dust3r/dust3r/patch_embed.py b/submodules/mast3r/dust3r/dust3r/patch_embed.py new file mode 100644 index 0000000000000000000000000000000000000000..07bb184bccb9d16657581576779904065d2dc857 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/patch_embed.py @@ -0,0 +1,70 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# PatchEmbed implementation for DUST3R, +# in particular ManyAR_PatchEmbed that Handle images with non-square aspect ratio +# -------------------------------------------------------- +import torch +import dust3r.utils.path_to_croco # noqa: F401 +from models.blocks import PatchEmbed # noqa + + +def get_patch_embed(patch_embed_cls, img_size, patch_size, enc_embed_dim): + assert patch_embed_cls in ['PatchEmbedDust3R', 'ManyAR_PatchEmbed'] + patch_embed = eval(patch_embed_cls)(img_size, patch_size, 3, enc_embed_dim) + return patch_embed + + +class PatchEmbedDust3R(PatchEmbed): + def forward(self, x, **kw): + B, C, H, W = x.shape + assert H % self.patch_size[0] == 0, f"Input image height ({H}) is not a multiple of patch size ({self.patch_size[0]})." + assert W % self.patch_size[1] == 0, f"Input image width ({W}) is not a multiple of patch size ({self.patch_size[1]})." + x = self.proj(x) + pos = self.position_getter(B, x.size(2), x.size(3), x.device) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x, pos + + +class ManyAR_PatchEmbed (PatchEmbed): + """ Handle images with non-square aspect ratio. + All images in the same batch have the same aspect ratio. + true_shape = [(height, width) ...] indicates the actual shape of each image. + """ + + def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True): + self.embed_dim = embed_dim + super().__init__(img_size, patch_size, in_chans, embed_dim, norm_layer, flatten) + + def forward(self, img, true_shape): + B, C, H, W = img.shape + assert W >= H, f'img should be in landscape mode, but got {W=} {H=}' + assert H % self.patch_size[0] == 0, f"Input image height ({H}) is not a multiple of patch size ({self.patch_size[0]})." + assert W % self.patch_size[1] == 0, f"Input image width ({W}) is not a multiple of patch size ({self.patch_size[1]})." + assert true_shape.shape == (B, 2), f"true_shape has the wrong shape={true_shape.shape}" + + # size expressed in tokens + W //= self.patch_size[0] + H //= self.patch_size[1] + n_tokens = H * W + + height, width = true_shape.T + is_landscape = (width >= height) + is_portrait = ~is_landscape + + # allocate result + x = img.new_zeros((B, n_tokens, self.embed_dim)) + pos = img.new_zeros((B, n_tokens, 2), dtype=torch.int64) + + # linear projection, transposed if necessary + x[is_landscape] = self.proj(img[is_landscape]).permute(0, 2, 3, 1).flatten(1, 2).float() + x[is_portrait] = self.proj(img[is_portrait].swapaxes(-1, -2)).permute(0, 2, 3, 1).flatten(1, 2).float() + + pos[is_landscape] = self.position_getter(1, H, W, pos.device) + pos[is_portrait] = self.position_getter(1, W, H, pos.device) + + x = self.norm(x) + return x, pos diff --git a/submodules/mast3r/dust3r/dust3r/post_process.py b/submodules/mast3r/dust3r/dust3r/post_process.py new file mode 100644 index 0000000000000000000000000000000000000000..550a9b41025ad003228ef16f97d045fc238746e4 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/post_process.py @@ -0,0 +1,60 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilities for interpreting the DUST3R output +# -------------------------------------------------------- +import numpy as np +import torch +from dust3r.utils.geometry import xy_grid + + +def estimate_focal_knowing_depth(pts3d, pp, focal_mode='median', min_focal=0., max_focal=np.inf): + """ Reprojection method, for when the absolute depth is known: + 1) estimate the camera focal using a robust estimator + 2) reproject points onto true rays, minimizing a certain error + """ + B, H, W, THREE = pts3d.shape + assert THREE == 3 + + # centered pixel grid + pixels = xy_grid(W, H, device=pts3d.device).view(1, -1, 2) - pp.view(-1, 1, 2) # B,HW,2 + pts3d = pts3d.flatten(1, 2) # (B, HW, 3) + + if focal_mode == 'median': + with torch.no_grad(): + # direct estimation of focal + u, v = pixels.unbind(dim=-1) + x, y, z = pts3d.unbind(dim=-1) + fx_votes = (u * z) / x + fy_votes = (v * z) / y + + # assume square pixels, hence same focal for X and Y + f_votes = torch.cat((fx_votes.view(B, -1), fy_votes.view(B, -1)), dim=-1) + focal = torch.nanmedian(f_votes, dim=-1).values + + elif focal_mode == 'weiszfeld': + # init focal with l2 closed form + # we try to find focal = argmin Sum | pixel - focal * (x,y)/z| + xy_over_z = (pts3d[..., :2] / pts3d[..., 2:3]).nan_to_num(posinf=0, neginf=0) # homogeneous (x,y,1) + + dot_xy_px = (xy_over_z * pixels).sum(dim=-1) + dot_xy_xy = xy_over_z.square().sum(dim=-1) + + focal = dot_xy_px.mean(dim=1) / dot_xy_xy.mean(dim=1) + + # iterative re-weighted least-squares + for iter in range(10): + # re-weighting by inverse of distance + dis = (pixels - focal.view(-1, 1, 1) * xy_over_z).norm(dim=-1) + # print(dis.nanmean(-1)) + w = dis.clip(min=1e-8).reciprocal() + # update the scaling with the new weights + focal = (w * dot_xy_px).mean(dim=1) / (w * dot_xy_xy).mean(dim=1) + else: + raise ValueError(f'bad {focal_mode=}') + + focal_base = max(H, W) / (2 * np.tan(np.deg2rad(60) / 2)) # size / 1.1547005383792515 + focal = focal.clip(min=min_focal*focal_base, max=max_focal*focal_base) + # print(focal) + return focal diff --git a/submodules/mast3r/dust3r/dust3r/training.py b/submodules/mast3r/dust3r/dust3r/training.py new file mode 100644 index 0000000000000000000000000000000000000000..9d3549f99b899514eae203c7ea506f4a6b9deaf2 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/training.py @@ -0,0 +1,378 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# training code for DUSt3R +# -------------------------------------------------------- +# References: +# MAE: https://github.com/facebookresearch/mae +# DeiT: https://github.com/facebookresearch/deit +# BEiT: https://github.com/microsoft/unilm/tree/master/beit +# -------------------------------------------------------- +import argparse +import datetime +import json +import numpy as np +import os +import sys +import time +import math +from collections import defaultdict +from pathlib import Path +from typing import Sized + +import torch +import torch.backends.cudnn as cudnn +from torch.utils.tensorboard import SummaryWriter +torch.backends.cuda.matmul.allow_tf32 = True # for gpu >= Ampere and pytorch >= 1.12 + +from dust3r.model import AsymmetricCroCo3DStereo, inf # noqa: F401, needed when loading the model +from dust3r.datasets import get_data_loader # noqa +from dust3r.losses import * # noqa: F401, needed when loading the model +# from dust3r.inference import loss_of_one_batch # noqa +from src.losses import loss_of_one_batch # noqa + +import dust3r.utils.path_to_croco # noqa: F401 +import croco.utils.misc as misc # noqa +from croco.utils.misc import NativeScalerWithGradNormCount as NativeScaler # noqa + + +def get_args_parser(): + parser = argparse.ArgumentParser('DUST3R training', add_help=False) + # model and criterion + parser.add_argument('--model', default="AsymmetricCroCo3DStereo(patch_embed_cls='ManyAR_PatchEmbed')", + type=str, help="string containing the model to build") + parser.add_argument('--pretrained', default=None, help='path of a starting checkpoint') + parser.add_argument('--train_criterion', default="ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)", + type=str, help="train criterion") + parser.add_argument('--test_criterion', default=None, type=str, help="test criterion") + + # dataset + parser.add_argument('--train_dataset', required=True, type=str, help="training set") + parser.add_argument('--test_dataset', default='[None]', type=str, help="testing set") + + # training + parser.add_argument('--seed', default=0, type=int, help="Random seed") + parser.add_argument('--batch_size', default=64, type=int, + help="Batch size per GPU (effective batch size is batch_size * accum_iter * # gpus") + parser.add_argument('--accum_iter', default=1, type=int, + help="Accumulate gradient iterations (for increasing the effective batch size under memory constraints)") + parser.add_argument('--epochs', default=800, type=int, help="Maximum number of epochs for the scheduler") + + parser.add_argument('--weight_decay', type=float, default=0.05, help="weight decay (default: 0.05)") + parser.add_argument('--lr', type=float, default=None, metavar='LR', help='learning rate (absolute lr)') + parser.add_argument('--blr', type=float, default=1.5e-4, metavar='LR', + help='base learning rate: absolute_lr = base_lr * total_batch_size / 256') + parser.add_argument('--min_lr', type=float, default=0., metavar='LR', + help='lower lr bound for cyclic schedulers that hit 0') + parser.add_argument('--warmup_epochs', type=int, default=40, metavar='N', help='epochs to warmup LR') + + parser.add_argument('--amp', type=int, default=0, + choices=[0, 1], help="Use Automatic Mixed Precision for pretraining") + parser.add_argument("--disable_cudnn_benchmark", action='store_true', default=False, + help="set cudnn.benchmark = False") + # others + parser.add_argument('--num_workers', default=8, type=int) + parser.add_argument('--world_size', default=1, type=int, help='number of distributed processes') + parser.add_argument('--local_rank', default=-1, type=int) + parser.add_argument('--dist_url', default='env://', help='url used to set up distributed training') + + parser.add_argument('--eval_freq', type=int, default=1, help='Test loss evaluation frequency') + parser.add_argument('--save_freq', default=1, type=int, + help='frequence (number of epochs) to save checkpoint in checkpoint-last.pth') + parser.add_argument('--keep_freq', default=20, type=int, + help='frequence (number of epochs) to save checkpoint in checkpoint-%d.pth') + parser.add_argument('--print_freq', default=20, type=int, + help='frequence (number of iterations) to print infos while training') + + # output dir + parser.add_argument('--output_dir', default='./output/', type=str, help="path where to save the output") + return parser + + +def train(args): + misc.init_distributed_mode(args) + global_rank = misc.get_rank() + world_size = misc.get_world_size() + + print("output_dir: " + args.output_dir) + if args.output_dir: + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + + # auto resume + last_ckpt_fname = os.path.join(args.output_dir, f'checkpoint-last.pth') + args.resume = last_ckpt_fname if os.path.isfile(last_ckpt_fname) else None + + print('job dir: {}'.format(os.path.dirname(os.path.realpath(__file__)))) + print("{}".format(args).replace(', ', ',\n')) + + device = "cuda" if torch.cuda.is_available() else "cpu" + device = torch.device(device) + + # fix the seed + seed = args.seed + misc.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + + cudnn.benchmark = not args.disable_cudnn_benchmark + + # training dataset and loader + print('Building train dataset {:s}'.format(args.train_dataset)) + # dataset and loader + data_loader_train = build_dataset(args.train_dataset, args.batch_size, args.num_workers, test=False) + print('Building test dataset {:s}'.format(args.train_dataset)) + data_loader_test = {dataset.split('(')[0]: build_dataset(dataset, args.batch_size, args.num_workers, test=True) + for dataset in args.test_dataset.split('+')} + + # model + print('Loading model: {:s}'.format(args.model)) + model = eval(args.model) + print(f'>> Creating train criterion = {args.train_criterion}') + train_criterion = eval(args.train_criterion).to(device) + print(f'>> Creating test criterion = {args.test_criterion or args.train_criterion}') + test_criterion = eval(args.test_criterion or args.criterion).to(device) + + model.to(device) + model_without_ddp = model + print("Model = %s" % str(model_without_ddp)) + + if args.pretrained and not args.resume: + print('Loading pretrained: ', args.pretrained) + ckpt = torch.load(args.pretrained, map_location=device) + print(model.load_state_dict(ckpt['model'], strict=False)) + del ckpt # in case it occupies memory + + eff_batch_size = args.batch_size * args.accum_iter * misc.get_world_size() + if args.lr is None: # only base_lr is specified + args.lr = args.blr * eff_batch_size / 256 + print("base lr: %.2e" % (args.lr * 256 / eff_batch_size)) + print("actual lr: %.2e" % args.lr) + print("accumulate grad iterations: %d" % args.accum_iter) + print("effective batch size: %d" % eff_batch_size) + + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel( + model, device_ids=[args.gpu], find_unused_parameters=True, static_graph=True) + model_without_ddp = model.module + + # following timm: set wd as 0 for bias and norm layers + param_groups = misc.get_parameter_groups(model_without_ddp, args.weight_decay) + optimizer = torch.optim.AdamW(param_groups, lr=args.lr, betas=(0.9, 0.95)) + print(optimizer) + loss_scaler = NativeScaler() + + def write_log_stats(epoch, train_stats, test_stats): + if misc.is_main_process(): + if log_writer is not None: + log_writer.flush() + + log_stats = dict(epoch=epoch, **{f'train_{k}': v for k, v in train_stats.items()}) + for test_name in data_loader_test: + if test_name not in test_stats: + continue + log_stats.update({test_name + '_' + k: v for k, v in test_stats[test_name].items()}) + + with open(os.path.join(args.output_dir, "log.txt"), mode="a", encoding="utf-8") as f: + f.write(json.dumps(log_stats) + "\n") + + def save_model(epoch, fname, best_so_far): + misc.save_model(args=args, model_without_ddp=model_without_ddp, optimizer=optimizer, + loss_scaler=loss_scaler, epoch=epoch, fname=fname, best_so_far=best_so_far) + + best_so_far = misc.load_model(args=args, model_without_ddp=model_without_ddp, + optimizer=optimizer, loss_scaler=loss_scaler) + if best_so_far is None: + best_so_far = float('inf') + if global_rank == 0 and args.output_dir is not None: + log_writer = SummaryWriter(log_dir=args.output_dir) + else: + log_writer = None + + print(f"Start training for {args.epochs} epochs") + start_time = time.time() + train_stats = test_stats = {} + for epoch in range(args.start_epoch, args.epochs + 1): + + # Save immediately the last checkpoint + if epoch > args.start_epoch: + if args.save_freq and epoch % args.save_freq == 0 or epoch == args.epochs: + save_model(epoch - 1, 'last', best_so_far) + + # Test on multiple datasets + new_best = False + if (epoch > 0 and args.eval_freq > 0 and epoch % args.eval_freq == 0): + test_stats = {} + for test_name, testset in data_loader_test.items(): + stats = test_one_epoch(model, test_criterion, testset, + device, epoch, log_writer=log_writer, args=args, prefix=test_name) + test_stats[test_name] = stats + + # Save best of all + if stats['loss_med'] < best_so_far: + best_so_far = stats['loss_med'] + new_best = True + + # Save more stuff + write_log_stats(epoch, train_stats, test_stats) + + if epoch > args.start_epoch: + if args.keep_freq and epoch % args.keep_freq == 0: + save_model(epoch - 1, str(epoch), best_so_far) + if new_best: + save_model(epoch - 1, 'best', best_so_far) + if epoch >= args.epochs: + break # exit after writing last test to disk + + # Train + train_stats = train_one_epoch( + model, train_criterion, data_loader_train, + optimizer, device, epoch, loss_scaler, + log_writer=log_writer, + args=args) + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + save_final_model(args, args.epochs, model_without_ddp, best_so_far=best_so_far) + + +def save_final_model(args, epoch, model_without_ddp, best_so_far=None): + output_dir = Path(args.output_dir) + checkpoint_path = output_dir / 'checkpoint-final.pth' + to_save = { + 'args': args, + 'model': model_without_ddp if isinstance(model_without_ddp, dict) else model_without_ddp.cpu().state_dict(), + 'epoch': epoch + } + if best_so_far is not None: + to_save['best_so_far'] = best_so_far + print(f'>> Saving model to {checkpoint_path} ...') + misc.save_on_master(to_save, checkpoint_path) + + +def build_dataset(dataset, batch_size, num_workers, test=False): + split = ['Train', 'Test'][test] + print(f'Building {split} Data loader for dataset: ', dataset) + loader = get_data_loader(dataset, + batch_size=batch_size, + num_workers=num_workers, + pin_mem=True, + shuffle=not (test), + drop_last=not (test)) + + print(f"{split} dataset length: ", len(loader)) + return loader + + +def train_one_epoch(model: torch.nn.Module, criterion: torch.nn.Module, + data_loader: Sized, optimizer: torch.optim.Optimizer, + device: torch.device, epoch: int, loss_scaler, + args, + log_writer=None): + assert torch.backends.cuda.matmul.allow_tf32 == True + + model.train(True) + metric_logger = misc.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', misc.SmoothedValue(window_size=1, fmt='{value:.6f}')) + header = 'Epoch: [{}]'.format(epoch) + accum_iter = args.accum_iter + + if log_writer is not None: + print('log_dir: {}'.format(log_writer.log_dir)) + + if hasattr(data_loader, 'dataset') and hasattr(data_loader.dataset, 'set_epoch'): + data_loader.dataset.set_epoch(epoch) + if hasattr(data_loader, 'sampler') and hasattr(data_loader.sampler, 'set_epoch'): + data_loader.sampler.set_epoch(epoch) + + optimizer.zero_grad() + + for data_iter_step, batch in enumerate(metric_logger.log_every(data_loader, args.print_freq, header)): + epoch_f = epoch + data_iter_step / len(data_loader) + + # we use a per iteration (instead of per epoch) lr scheduler + if data_iter_step % accum_iter == 0: + misc.adjust_learning_rate(optimizer, epoch_f, args) + + loss_tuple = loss_of_one_batch(batch, model, criterion, device, + symmetrize_batch=False, + use_amp=bool(args.amp), ret='loss') + loss, loss_details = loss_tuple # criterion returns two values + loss_value = float(loss) + + if not math.isfinite(loss_value): + print("Loss is {}, stopping training".format(loss_value), force=True) + sys.exit(1) + + loss /= accum_iter + loss_scaler(loss, optimizer, parameters=model.parameters(), + update_grad=(data_iter_step + 1) % accum_iter == 0) + if (data_iter_step + 1) % accum_iter == 0: + optimizer.zero_grad() + + del loss + del batch + + lr = optimizer.param_groups[0]["lr"] + metric_logger.update(epoch=epoch_f) + metric_logger.update(lr=lr) + metric_logger.update(loss=loss_value, **loss_details) + + if (data_iter_step + 1) % accum_iter == 0 and ((data_iter_step + 1) % (accum_iter * args.print_freq)) == 0: + loss_value_reduce = misc.all_reduce_mean(loss_value) # MUST BE EXECUTED BY ALL NODES + if log_writer is None: + continue + """ We use epoch_1000x as the x-axis in tensorboard. + This calibrates different curves when batch size changes. + """ + epoch_1000x = int(epoch_f * 1000) + log_writer.add_scalar('train_loss', loss_value_reduce, epoch_1000x) + log_writer.add_scalar('train_lr', lr, epoch_1000x) + log_writer.add_scalar('train_iter', epoch_1000x, epoch_1000x) + for name, val in loss_details.items(): + log_writer.add_scalar('train_' + name, val, epoch_1000x) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + return {k: meter.global_avg for k, meter in metric_logger.meters.items()} + + +@torch.no_grad() +def test_one_epoch(model: torch.nn.Module, criterion: torch.nn.Module, + data_loader: Sized, device: torch.device, epoch: int, + args, log_writer=None, prefix='test'): + + model.eval() + metric_logger = misc.MetricLogger(delimiter=" ") + metric_logger.meters = defaultdict(lambda: misc.SmoothedValue(window_size=9**9)) + header = 'Test Epoch: [{}]'.format(epoch) + + if log_writer is not None: + print('log_dir: {}'.format(log_writer.log_dir)) + + if hasattr(data_loader, 'dataset') and hasattr(data_loader.dataset, 'set_epoch'): + data_loader.dataset.set_epoch(epoch) + if hasattr(data_loader, 'sampler') and hasattr(data_loader.sampler, 'set_epoch'): + data_loader.sampler.set_epoch(epoch) + + for _, batch in enumerate(metric_logger.log_every(data_loader, args.print_freq, header)): + loss_tuple = loss_of_one_batch(batch, model, criterion, device, + symmetrize_batch=False, + use_amp=bool(args.amp), ret='loss') + loss_value, loss_details = loss_tuple # criterion returns two values + metric_logger.update(loss=float(loss_value), **loss_details) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + + aggs = [('avg', 'global_avg'), ('med', 'median')] + results = {f'{k}_{tag}': getattr(meter, attr) for k, meter in metric_logger.meters.items() for tag, attr in aggs} + + if log_writer is not None: + for name, val in results.items(): + log_writer.add_scalar(prefix + '_' + name, val, 1000 * epoch) + + return results diff --git a/submodules/mast3r/dust3r/dust3r/utils/__init__.py b/submodules/mast3r/dust3r/dust3r/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a32692113d830ddc4af4e6ed608f222fbe062e6e --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). diff --git a/submodules/mast3r/dust3r/dust3r/utils/device.py b/submodules/mast3r/dust3r/dust3r/utils/device.py new file mode 100644 index 0000000000000000000000000000000000000000..e3b6a74dac05a2e1ba3a2b2f0faa8cea08ece745 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/utils/device.py @@ -0,0 +1,76 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilitary functions for DUSt3R +# -------------------------------------------------------- +import numpy as np +import torch + + +def todevice(batch, device, callback=None, non_blocking=False): + ''' Transfer some variables to another device (i.e. GPU, CPU:torch, CPU:numpy). + + batch: list, tuple, dict of tensors or other things + device: pytorch device or 'numpy' + callback: function that would be called on every sub-elements. + ''' + if callback: + batch = callback(batch) + + if isinstance(batch, dict): + return {k: todevice(v, device) for k, v in batch.items()} + + if isinstance(batch, (tuple, list)): + return type(batch)(todevice(x, device) for x in batch) + + x = batch + if device == 'numpy': + if isinstance(x, torch.Tensor): + x = x.detach().cpu().numpy() + elif x is not None: + if isinstance(x, np.ndarray): + x = torch.from_numpy(x) + if torch.is_tensor(x): + x = x.to(device, non_blocking=non_blocking) + return x + + +to_device = todevice # alias + + +def to_numpy(x): return todevice(x, 'numpy') +def to_cpu(x): return todevice(x, 'cpu') +def to_cuda(x): return todevice(x, 'cuda') + + +def collate_with_cat(whatever, lists=False): + if isinstance(whatever, dict): + return {k: collate_with_cat(vals, lists=lists) for k, vals in whatever.items()} + + elif isinstance(whatever, (tuple, list)): + if len(whatever) == 0: + return whatever + elem = whatever[0] + T = type(whatever) + + if elem is None: + return None + if isinstance(elem, (bool, float, int, str)): + return whatever + if isinstance(elem, tuple): + return T(collate_with_cat(x, lists=lists) for x in zip(*whatever)) + if isinstance(elem, dict): + return {k: collate_with_cat([e[k] for e in whatever], lists=lists) for k in elem} + + if isinstance(elem, torch.Tensor): + return listify(whatever) if lists else torch.cat(whatever) + if isinstance(elem, np.ndarray): + return listify(whatever) if lists else torch.cat([torch.from_numpy(x) for x in whatever]) + + # otherwise, we just chain lists + return sum(whatever, T()) + + +def listify(elems): + return [x for e in elems for x in e] diff --git a/submodules/mast3r/dust3r/dust3r/utils/geometry.py b/submodules/mast3r/dust3r/dust3r/utils/geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..ce365faf2acb97ffaafa1b80cb8ee0c28de0b6d6 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/utils/geometry.py @@ -0,0 +1,366 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# geometry utilitary functions +# -------------------------------------------------------- +import torch +import numpy as np +from scipy.spatial import cKDTree as KDTree + +from dust3r.utils.misc import invalid_to_zeros, invalid_to_nans +from dust3r.utils.device import to_numpy + + +def xy_grid(W, H, device=None, origin=(0, 0), unsqueeze=None, cat_dim=-1, homogeneous=False, **arange_kw): + """ Output a (H,W,2) array of int32 + with output[j,i,0] = i + origin[0] + output[j,i,1] = j + origin[1] + """ + if device is None: + # numpy + arange, meshgrid, stack, ones = np.arange, np.meshgrid, np.stack, np.ones + else: + # torch + arange = lambda *a, **kw: torch.arange(*a, device=device, **kw) + meshgrid, stack = torch.meshgrid, torch.stack + ones = lambda *a: torch.ones(*a, device=device) + + tw, th = [arange(o, o + s, **arange_kw) for s, o in zip((W, H), origin)] + grid = meshgrid(tw, th, indexing='xy') + if homogeneous: + grid = grid + (ones((H, W)),) + if unsqueeze is not None: + grid = (grid[0].unsqueeze(unsqueeze), grid[1].unsqueeze(unsqueeze)) + if cat_dim is not None: + grid = stack(grid, cat_dim) + return grid + + +def geotrf(Trf, pts, ncol=None, norm=False): + """ Apply a geometric transformation to a list of 3-D points. + + H: 3x3 or 4x4 projection matrix (typically a Homography) + p: numpy/torch/tuple of coordinates. Shape must be (...,2) or (...,3) + + ncol: int. number of columns of the result (2 or 3) + norm: float. if != 0, the resut is projected on the z=norm plane. + + Returns an array of projected 2d points. + """ + assert Trf.ndim >= 2 + if isinstance(Trf, np.ndarray): + pts = np.asarray(pts) + elif isinstance(Trf, torch.Tensor): + pts = torch.as_tensor(pts, dtype=Trf.dtype) + + # adapt shape if necessary + output_reshape = pts.shape[:-1] + ncol = ncol or pts.shape[-1] + + # optimized code + if (isinstance(Trf, torch.Tensor) and isinstance(pts, torch.Tensor) and + Trf.ndim == 3 and pts.ndim == 4): + d = pts.shape[3] + if Trf.shape[-1] == d: + pts = torch.einsum("bij, bhwj -> bhwi", Trf, pts) + elif Trf.shape[-1] == d + 1: + pts = torch.einsum("bij, bhwj -> bhwi", Trf[:, :d, :d], pts) + Trf[:, None, None, :d, d] + else: + raise ValueError(f'bad shape, not ending with 3 or 4, for {pts.shape=}') + else: + if Trf.ndim >= 3: + n = Trf.ndim - 2 + assert Trf.shape[:n] == pts.shape[:n], 'batch size does not match' + Trf = Trf.reshape(-1, Trf.shape[-2], Trf.shape[-1]) + + if pts.ndim > Trf.ndim: + # Trf == (B,d,d) & pts == (B,H,W,d) --> (B, H*W, d) + pts = pts.reshape(Trf.shape[0], -1, pts.shape[-1]) + elif pts.ndim == 2: + # Trf == (B,d,d) & pts == (B,d) --> (B, 1, d) + pts = pts[:, None, :] + + if pts.shape[-1] + 1 == Trf.shape[-1]: + Trf = Trf.swapaxes(-1, -2) # transpose Trf + pts = pts @ Trf[..., :-1, :] + Trf[..., -1:, :] + elif pts.shape[-1] == Trf.shape[-1]: + Trf = Trf.swapaxes(-1, -2) # transpose Trf + pts = pts @ Trf + else: + pts = Trf @ pts.T + if pts.ndim >= 2: + pts = pts.swapaxes(-1, -2) + + if norm: + pts = pts / pts[..., -1:] # DONT DO /= BECAUSE OF WEIRD PYTORCH BUG + if norm != 1: + pts *= norm + + res = pts[..., :ncol].reshape(*output_reshape, ncol) + return res + + +def inv(mat): + """ Invert a torch or numpy matrix + """ + if isinstance(mat, torch.Tensor): + return torch.linalg.inv(mat) + if isinstance(mat, np.ndarray): + return np.linalg.inv(mat) + raise ValueError(f'bad matrix type = {type(mat)}') + + +def depthmap_to_pts3d(depth, pseudo_focal, pp=None, **_): + """ + Args: + - depthmap (BxHxW array): + - pseudo_focal: [B,H,W] ; [B,2,H,W] or [B,1,H,W] + Returns: + pointmap of absolute coordinates (BxHxWx3 array) + """ + + if len(depth.shape) == 4: + B, H, W, n = depth.shape + else: + B, H, W = depth.shape + n = None + + if len(pseudo_focal.shape) == 3: # [B,H,W] + pseudo_focalx = pseudo_focaly = pseudo_focal + elif len(pseudo_focal.shape) == 4: # [B,2,H,W] or [B,1,H,W] + pseudo_focalx = pseudo_focal[:, 0] + if pseudo_focal.shape[1] == 2: + pseudo_focaly = pseudo_focal[:, 1] + else: + pseudo_focaly = pseudo_focalx + else: + raise NotImplementedError("Error, unknown input focal shape format.") + + assert pseudo_focalx.shape == depth.shape[:3] + assert pseudo_focaly.shape == depth.shape[:3] + grid_x, grid_y = xy_grid(W, H, cat_dim=0, device=depth.device)[:, None] + + # set principal point + if pp is None: + grid_x = grid_x - (W - 1) / 2 + grid_y = grid_y - (H - 1) / 2 + else: + grid_x = grid_x.expand(B, -1, -1) - pp[:, 0, None, None] + grid_y = grid_y.expand(B, -1, -1) - pp[:, 1, None, None] + + if n is None: + pts3d = torch.empty((B, H, W, 3), device=depth.device) + pts3d[..., 0] = depth * grid_x / pseudo_focalx + pts3d[..., 1] = depth * grid_y / pseudo_focaly + pts3d[..., 2] = depth + else: + pts3d = torch.empty((B, H, W, 3, n), device=depth.device) + pts3d[..., 0, :] = depth * (grid_x / pseudo_focalx)[..., None] + pts3d[..., 1, :] = depth * (grid_y / pseudo_focaly)[..., None] + pts3d[..., 2, :] = depth + return pts3d + + +def depthmap_to_camera_coordinates(depthmap, camera_intrinsics, pseudo_focal=None): + """ + Args: + - depthmap (HxW array): + - camera_intrinsics: a 3x3 matrix + Returns: + pointmap of absolute coordinates (HxWx3 array), and a mask specifying valid pixels. + """ + camera_intrinsics = np.float32(camera_intrinsics) + H, W = depthmap.shape + + # Compute 3D ray associated with each pixel + # Strong assumption: there are no skew terms + assert camera_intrinsics[0, 1] == 0.0 + assert camera_intrinsics[1, 0] == 0.0 + if pseudo_focal is None: + fu = camera_intrinsics[0, 0] + fv = camera_intrinsics[1, 1] + else: + assert pseudo_focal.shape == (H, W) + fu = fv = pseudo_focal + cu = camera_intrinsics[0, 2] + cv = camera_intrinsics[1, 2] + + u, v = np.meshgrid(np.arange(W), np.arange(H)) + z_cam = depthmap + x_cam = (u - cu) * z_cam / fu + y_cam = (v - cv) * z_cam / fv + X_cam = np.stack((x_cam, y_cam, z_cam), axis=-1).astype(np.float32) + + # Mask for valid coordinates + valid_mask = (depthmap > 0.0) + return X_cam, valid_mask + + +def depthmap_to_absolute_camera_coordinates(depthmap, camera_intrinsics, camera_pose, **kw): + """ + Args: + - depthmap (HxW array): + - camera_intrinsics: a 3x3 matrix + - camera_pose: a 4x3 or 4x4 cam2world matrix + Returns: + pointmap of absolute coordinates (HxWx3 array), and a mask specifying valid pixels.""" + X_cam, valid_mask = depthmap_to_camera_coordinates(depthmap, camera_intrinsics) + + X_world = X_cam # default + if camera_pose is not None: + # R_cam2world = np.float32(camera_params["R_cam2world"]) + # t_cam2world = np.float32(camera_params["t_cam2world"]).squeeze() + R_cam2world = camera_pose[:3, :3] + t_cam2world = camera_pose[:3, 3] + + # Express in absolute coordinates (invalid depth values) + X_world = np.einsum("ik, vuk -> vui", R_cam2world, X_cam) + t_cam2world[None, None, :] + + return X_world, valid_mask + + +def colmap_to_opencv_intrinsics(K): + """ + Modify camera intrinsics to follow a different convention. + Coordinates of the center of the top-left pixels are by default: + - (0.5, 0.5) in Colmap + - (0,0) in OpenCV + """ + K = K.copy() + K[0, 2] -= 0.5 + K[1, 2] -= 0.5 + return K + + +def opencv_to_colmap_intrinsics(K): + """ + Modify camera intrinsics to follow a different convention. + Coordinates of the center of the top-left pixels are by default: + - (0.5, 0.5) in Colmap + - (0,0) in OpenCV + """ + K = K.copy() + K[0, 2] += 0.5 + K[1, 2] += 0.5 + return K + + +def normalize_pointcloud(pts1, pts2, norm_mode='avg_dis', valid1=None, valid2=None, ret_factor=False): + """ renorm pointmaps pts1, pts2 with norm_mode + """ + assert pts1.ndim >= 3 and pts1.shape[-1] == 3 + assert pts2 is None or (pts2.ndim >= 3 and pts2.shape[-1] == 3) + norm_mode, dis_mode = norm_mode.split('_') + + if norm_mode == 'avg': + # gather all points together (joint normalization) + nan_pts1, nnz1 = invalid_to_zeros(pts1, valid1, ndim=3) + nan_pts2, nnz2 = invalid_to_zeros(pts2, valid2, ndim=3) if pts2 is not None else (None, 0) + all_pts = torch.cat((nan_pts1, nan_pts2), dim=1) if pts2 is not None else nan_pts1 + + # compute distance to origin + all_dis = all_pts.norm(dim=-1) + if dis_mode == 'dis': + pass # do nothing + elif dis_mode == 'log1p': + all_dis = torch.log1p(all_dis) + elif dis_mode == 'warp-log1p': + # actually warp input points before normalizing them + log_dis = torch.log1p(all_dis) + warp_factor = log_dis / all_dis.clip(min=1e-8) + H1, W1 = pts1.shape[1:-1] + pts1 = pts1 * warp_factor[:, :W1 * H1].view(-1, H1, W1, 1) + if pts2 is not None: + H2, W2 = pts2.shape[1:-1] + pts2 = pts2 * warp_factor[:, W1 * H1:].view(-1, H2, W2, 1) + all_dis = log_dis # this is their true distance afterwards + else: + raise ValueError(f'bad {dis_mode=}') + + norm_factor = all_dis.sum(dim=1) / (nnz1 + nnz2 + 1e-8) + else: + # gather all points together (joint normalization) + nan_pts1 = invalid_to_nans(pts1, valid1, ndim=3) + nan_pts2 = invalid_to_nans(pts2, valid2, ndim=3) if pts2 is not None else None + all_pts = torch.cat((nan_pts1, nan_pts2), dim=1) if pts2 is not None else nan_pts1 + + # compute distance to origin + all_dis = all_pts.norm(dim=-1) + + if norm_mode == 'avg': + norm_factor = all_dis.nanmean(dim=1) + elif norm_mode == 'median': + norm_factor = all_dis.nanmedian(dim=1).values.detach() + elif norm_mode == 'sqrt': + norm_factor = all_dis.sqrt().nanmean(dim=1)**2 + else: + raise ValueError(f'bad {norm_mode=}') + + norm_factor = norm_factor.clip(min=1e-8) + while norm_factor.ndim < pts1.ndim: + norm_factor.unsqueeze_(-1) + + res = pts1 / norm_factor + if pts2 is not None: + res = (res, pts2 / norm_factor) + if ret_factor: + res = res + (norm_factor,) + return res + + +@torch.no_grad() +def get_joint_pointcloud_depth(z1, z2, valid_mask1, valid_mask2=None, quantile=0.5): + # set invalid points to NaN + _z1 = invalid_to_nans(z1, valid_mask1).reshape(len(z1), -1) + _z2 = invalid_to_nans(z2, valid_mask2).reshape(len(z2), -1) if z2 is not None else None + _z = torch.cat((_z1, _z2), dim=-1) if z2 is not None else _z1 + + # compute median depth overall (ignoring nans) + if quantile == 0.5: + shift_z = torch.nanmedian(_z, dim=-1).values + else: + shift_z = torch.nanquantile(_z, quantile, dim=-1) + return shift_z # (B,) + + +@torch.no_grad() +def get_joint_pointcloud_center_scale(pts1, pts2, valid_mask1=None, valid_mask2=None, z_only=False, center=True): + # set invalid points to NaN + _pts1 = invalid_to_nans(pts1, valid_mask1).reshape(len(pts1), -1, 3) + _pts2 = invalid_to_nans(pts2, valid_mask2).reshape(len(pts2), -1, 3) if pts2 is not None else None + _pts = torch.cat((_pts1, _pts2), dim=1) if pts2 is not None else _pts1 + + # compute median center + _center = torch.nanmedian(_pts, dim=1, keepdim=True).values # (B,1,3) + if z_only: + _center[..., :2] = 0 # do not center X and Y + + # compute median norm + _norm = ((_pts - _center) if center else _pts).norm(dim=-1) + scale = torch.nanmedian(_norm, dim=1).values + return _center[:, None, :, :], scale[:, None, None, None] + + +def find_reciprocal_matches(P1, P2): + """ + returns 3 values: + 1 - reciprocal_in_P2: a boolean array of size P2.shape[0], a "True" value indicates a match + 2 - nn2_in_P1: a int array of size P2.shape[0], it contains the indexes of the closest points in P1 + 3 - reciprocal_in_P2.sum(): the number of matches + """ + tree1 = KDTree(P1) + tree2 = KDTree(P2) + + _, nn1_in_P2 = tree2.query(P1, workers=8) + _, nn2_in_P1 = tree1.query(P2, workers=8) + + reciprocal_in_P1 = (nn2_in_P1[nn1_in_P2] == np.arange(len(nn1_in_P2))) + reciprocal_in_P2 = (nn1_in_P2[nn2_in_P1] == np.arange(len(nn2_in_P1))) + assert reciprocal_in_P1.sum() == reciprocal_in_P2.sum() + return reciprocal_in_P2, nn2_in_P1, reciprocal_in_P2.sum() + + +def get_med_dist_between_poses(poses): + from scipy.spatial.distance import pdist + return np.median(pdist([to_numpy(p[:3, 3]) for p in poses])) diff --git a/submodules/mast3r/dust3r/dust3r/utils/image.py b/submodules/mast3r/dust3r/dust3r/utils/image.py new file mode 100644 index 0000000000000000000000000000000000000000..6312a346df919ae6a0424504d824ef813fea250f --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/utils/image.py @@ -0,0 +1,126 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilitary functions about images (loading/converting...) +# -------------------------------------------------------- +import os +import torch +import numpy as np +import PIL.Image +from PIL.ImageOps import exif_transpose +import torchvision.transforms as tvf +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 # noqa + +try: + from pillow_heif import register_heif_opener # noqa + register_heif_opener() + heif_support_enabled = True +except ImportError: + heif_support_enabled = False + +ImgNorm = tvf.Compose([tvf.ToTensor(), tvf.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) + + +def img_to_arr( img ): + if isinstance(img, str): + img = imread_cv2(img) + return img + +def imread_cv2(path, options=cv2.IMREAD_COLOR): + """ Open an image or a depthmap with opencv-python. + """ + if path.endswith(('.exr', 'EXR')): + options = cv2.IMREAD_ANYDEPTH + img = cv2.imread(path, options) + if img is None: + raise IOError(f'Could not load image={path} with {options=}') + if img.ndim == 3: + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + return img + + +def rgb(ftensor, true_shape=None): + if isinstance(ftensor, list): + return [rgb(x, true_shape=true_shape) for x in ftensor] + if isinstance(ftensor, torch.Tensor): + ftensor = ftensor.detach().cpu().numpy() # H,W,3 + if ftensor.ndim == 3 and ftensor.shape[0] == 3: + ftensor = ftensor.transpose(1, 2, 0) + elif ftensor.ndim == 4 and ftensor.shape[1] == 3: + ftensor = ftensor.transpose(0, 2, 3, 1) + if true_shape is not None: + H, W = true_shape + ftensor = ftensor[:H, :W] + if ftensor.dtype == np.uint8: + img = np.float32(ftensor) / 255 + else: + img = (ftensor * 0.5) + 0.5 + return img.clip(min=0, max=1) + + +def _resize_pil_image(img, long_edge_size): + S = max(img.size) + if S > long_edge_size: + interp = PIL.Image.LANCZOS + elif S <= long_edge_size: + interp = PIL.Image.BICUBIC + new_size = tuple(int(round(x*long_edge_size/S)) for x in img.size) + return img.resize(new_size, interp) + + +def load_images(folder_or_list, size, square_ok=False, verbose=True): + """ open and convert all images in a list or folder to proper input format for DUSt3R + """ + if isinstance(folder_or_list, str): + if verbose: + print(f'>> Loading images from {folder_or_list}') + root, folder_content = folder_or_list, sorted(os.listdir(folder_or_list)) + + elif isinstance(folder_or_list, list): + if verbose: + print(f'>> Loading a list of {len(folder_or_list)} images') + root, folder_content = '', folder_or_list + + else: + raise ValueError(f'bad {folder_or_list=} ({type(folder_or_list)})') + + supported_images_extensions = ['.jpg', '.jpeg', '.png'] + if heif_support_enabled: + supported_images_extensions += ['.heic', '.heif'] + supported_images_extensions = tuple(supported_images_extensions) + + imgs = [] + for path in folder_content: + if not path.lower().endswith(supported_images_extensions): + continue + img = exif_transpose(PIL.Image.open(os.path.join(root, path))).convert('RGB') + W1, H1 = img.size + if size == 224: + # resize short side to 224 (then crop) + img = _resize_pil_image(img, round(size * max(W1/H1, H1/W1))) + else: + # resize long side to 512 + img = _resize_pil_image(img, size) + W, H = img.size + cx, cy = W//2, H//2 + if size == 224: + half = min(cx, cy) + img = img.crop((cx-half, cy-half, cx+half, cy+half)) + else: + halfw, halfh = ((2*cx)//16)*8, ((2*cy)//16)*8 + if not (square_ok) and W == H: + halfh = 3*halfw/4 + img = img.crop((cx-halfw, cy-halfh, cx+halfw, cy+halfh)) + + W2, H2 = img.size + if verbose: + print(f' - adding {path} with resolution {W1}x{H1} --> {W2}x{H2}') + imgs.append(dict(img=ImgNorm(img)[None], true_shape=np.int32( + [img.size[::-1]]), idx=len(imgs), instance=str(len(imgs)))) + + assert imgs, 'no images foud at '+root + if verbose: + print(f' (Found {len(imgs)} images)') + return imgs diff --git a/submodules/mast3r/dust3r/dust3r/utils/misc.py b/submodules/mast3r/dust3r/dust3r/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..88c4d2dab6d5c14021ed9ed6646c3159a3a4637b --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/utils/misc.py @@ -0,0 +1,121 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilitary functions for DUSt3R +# -------------------------------------------------------- +import torch + + +def fill_default_args(kwargs, func): + import inspect # a bit hacky but it works reliably + signature = inspect.signature(func) + + for k, v in signature.parameters.items(): + if v.default is inspect.Parameter.empty: + continue + kwargs.setdefault(k, v.default) + + return kwargs + + +def freeze_all_params(modules): + for module in modules: + try: + for n, param in module.named_parameters(): + param.requires_grad = False + except AttributeError: + # module is directly a parameter + module.requires_grad = False + + +def is_symmetrized(gt1, gt2): + x = gt1['instance'] + y = gt2['instance'] + if len(x) == len(y) and len(x) == 1: + return False # special case of batchsize 1 + ok = True + for i in range(0, len(x), 2): + ok = ok and (x[i] == y[i + 1]) and (x[i + 1] == y[i]) + return ok + + +def flip(tensor): + """ flip so that tensor[0::2] <=> tensor[1::2] """ + return torch.stack((tensor[1::2], tensor[0::2]), dim=1).flatten(0, 1) + + +def interleave(tensor1, tensor2): + res1 = torch.stack((tensor1, tensor2), dim=1).flatten(0, 1) + res2 = torch.stack((tensor2, tensor1), dim=1).flatten(0, 1) + return res1, res2 + + +def transpose_to_landscape(head, activate=True): + """ Predict in the correct aspect-ratio, + then transpose the result in landscape + and stack everything back together. + """ + def wrapper_no(decout, true_shape): + B = len(true_shape) + assert true_shape[0:1].allclose(true_shape), 'true_shape must be all identical' + H, W = true_shape[0].cpu().tolist() + res = head(decout, (H, W)) + return res + + def wrapper_yes(decout, true_shape): + B = len(true_shape) + # by definition, the batch is in landscape mode so W >= H + H, W = int(true_shape.min()), int(true_shape.max()) + + height, width = true_shape.T + is_landscape = (width >= height) + is_portrait = ~is_landscape + + # true_shape = true_shape.cpu() + if is_landscape.all(): + return head(decout, (H, W)) + if is_portrait.all(): + return transposed(head(decout, (W, H))) + + # batch is a mix of both portraint & landscape + def selout(ar): return [d[ar] for d in decout] + l_result = head(selout(is_landscape), (H, W)) + p_result = transposed(head(selout(is_portrait), (W, H))) + + # allocate full result + result = {} + for k in l_result | p_result: + x = l_result[k].new(B, *l_result[k].shape[1:]) + x[is_landscape] = l_result[k] + x[is_portrait] = p_result[k] + result[k] = x + + return result + + return wrapper_yes if activate else wrapper_no + + +def transposed(dic): + return {k: v.swapaxes(1, 2) for k, v in dic.items()} + + +def invalid_to_nans(arr, valid_mask, ndim=999): + if valid_mask is not None: + arr = arr.clone() + arr[~valid_mask] = float('nan') + if arr.ndim > ndim: + arr = arr.flatten(-2 - (arr.ndim - ndim), -2) + return arr + + +def invalid_to_zeros(arr, valid_mask, ndim=999): + if valid_mask is not None: + arr = arr.clone() + arr[~valid_mask] = 0 + nnz = valid_mask.view(len(valid_mask), -1).sum(1) + else: + nnz = arr.numel() // len(arr) if len(arr) else 0 # number of point per image + if arr.ndim > ndim: + arr = arr.flatten(-2 - (arr.ndim - ndim), -2) + return arr, nnz diff --git a/submodules/mast3r/dust3r/dust3r/utils/parallel.py b/submodules/mast3r/dust3r/dust3r/utils/parallel.py new file mode 100644 index 0000000000000000000000000000000000000000..06ae7fefdb9d2298929f0cbc20dfbc57eb7d7f7b --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/utils/parallel.py @@ -0,0 +1,79 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilitary functions for multiprocessing +# -------------------------------------------------------- +from tqdm import tqdm +from multiprocessing.dummy import Pool as ThreadPool +from multiprocessing import cpu_count + + +def parallel_threads(function, args, workers=0, star_args=False, kw_args=False, front_num=1, Pool=ThreadPool, **tqdm_kw): + """ tqdm but with parallel execution. + + Will essentially return + res = [ function(arg) # default + function(*arg) # if star_args is True + function(**arg) # if kw_args is True + for arg in args] + + Note: + the first elements of args will not be parallelized. + This can be useful for debugging. + """ + while workers <= 0: + workers += cpu_count() + if workers == 1: + front_num = float('inf') + + # convert into an iterable + try: + n_args_parallel = len(args) - front_num + except TypeError: + n_args_parallel = None + args = iter(args) + + # sequential execution first + front = [] + while len(front) < front_num: + try: + a = next(args) + except StopIteration: + return front # end of the iterable + front.append(function(*a) if star_args else function(**a) if kw_args else function(a)) + + # then parallel execution + out = [] + with Pool(workers) as pool: + # Pass the elements of args into function + if star_args: + futures = pool.imap(starcall, [(function, a) for a in args]) + elif kw_args: + futures = pool.imap(starstarcall, [(function, a) for a in args]) + else: + futures = pool.imap(function, args) + # Print out the progress as tasks complete + for f in tqdm(futures, total=n_args_parallel, **tqdm_kw): + out.append(f) + return front + out + + +def parallel_processes(*args, **kwargs): + """ Same as parallel_threads, with processes + """ + import multiprocessing as mp + kwargs['Pool'] = mp.Pool + return parallel_threads(*args, **kwargs) + + +def starcall(args): + """ convenient wrapper for Process.Pool """ + function, args = args + return function(*args) + + +def starstarcall(args): + """ convenient wrapper for Process.Pool """ + function, args = args + return function(**args) diff --git a/submodules/mast3r/dust3r/dust3r/utils/path_to_croco.py b/submodules/mast3r/dust3r/dust3r/utils/path_to_croco.py new file mode 100644 index 0000000000000000000000000000000000000000..39226ce6bc0e1993ba98a22096de32cb6fa916b4 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/utils/path_to_croco.py @@ -0,0 +1,19 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# CroCo submodule import +# -------------------------------------------------------- + +import sys +import os.path as path +HERE_PATH = path.normpath(path.dirname(__file__)) +CROCO_REPO_PATH = path.normpath(path.join(HERE_PATH, '../../croco')) +CROCO_MODELS_PATH = path.join(CROCO_REPO_PATH, 'models') +# check the presence of models directory in repo to be sure its cloned +if path.isdir(CROCO_MODELS_PATH): + # workaround for sibling import + sys.path.insert(0, CROCO_REPO_PATH) +else: + raise ImportError(f"croco is not initialized, could not find: {CROCO_MODELS_PATH}.\n " + "Did you forget to run 'git submodule update --init --recursive' ?") diff --git a/submodules/mast3r/dust3r/dust3r/viz.py b/submodules/mast3r/dust3r/dust3r/viz.py new file mode 100644 index 0000000000000000000000000000000000000000..9150e8b850d9f1e6bf9ddf6e865d34fc743e276a --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r/viz.py @@ -0,0 +1,381 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Visualization utilities using trimesh +# -------------------------------------------------------- +import PIL.Image +import numpy as np +from scipy.spatial.transform import Rotation +import torch + +from dust3r.utils.geometry import geotrf, get_med_dist_between_poses, depthmap_to_absolute_camera_coordinates +from dust3r.utils.device import to_numpy +from dust3r.utils.image import rgb, img_to_arr + +try: + import trimesh +except ImportError: + print('/!\\ module trimesh is not installed, cannot visualize results /!\\') + + + +def cat_3d(vecs): + if isinstance(vecs, (np.ndarray, torch.Tensor)): + vecs = [vecs] + return np.concatenate([p.reshape(-1, 3) for p in to_numpy(vecs)]) + + +def show_raw_pointcloud(pts3d, colors, point_size=2): + scene = trimesh.Scene() + + pct = trimesh.PointCloud(cat_3d(pts3d), colors=cat_3d(colors)) + scene.add_geometry(pct) + + scene.show(line_settings={'point_size': point_size}) + + +def pts3d_to_trimesh(img, pts3d, valid=None): + H, W, THREE = img.shape + assert THREE == 3 + assert img.shape == pts3d.shape + + vertices = pts3d.reshape(-1, 3) + + # make squares: each pixel == 2 triangles + idx = np.arange(len(vertices)).reshape(H, W) + idx1 = idx[:-1, :-1].ravel() # top-left corner + idx2 = idx[:-1, +1:].ravel() # right-left corner + idx3 = idx[+1:, :-1].ravel() # bottom-left corner + idx4 = idx[+1:, +1:].ravel() # bottom-right corner + faces = np.concatenate(( + np.c_[idx1, idx2, idx3], + np.c_[idx3, idx2, idx1], # same triangle, but backward (cheap solution to cancel face culling) + np.c_[idx2, idx3, idx4], + np.c_[idx4, idx3, idx2], # same triangle, but backward (cheap solution to cancel face culling) + ), axis=0) + + # prepare triangle colors + face_colors = np.concatenate(( + img[:-1, :-1].reshape(-1, 3), + img[:-1, :-1].reshape(-1, 3), + img[+1:, +1:].reshape(-1, 3), + img[+1:, +1:].reshape(-1, 3) + ), axis=0) + + # remove invalid faces + if valid is not None: + assert valid.shape == (H, W) + valid_idxs = valid.ravel() + valid_faces = valid_idxs[faces].all(axis=-1) + faces = faces[valid_faces] + face_colors = face_colors[valid_faces] + + assert len(faces) == len(face_colors) + return dict(vertices=vertices, face_colors=face_colors, faces=faces) + + +def cat_meshes(meshes): + vertices, faces, colors = zip(*[(m['vertices'], m['faces'], m['face_colors']) for m in meshes]) + n_vertices = np.cumsum([0]+[len(v) for v in vertices]) + for i in range(len(faces)): + faces[i][:] += n_vertices[i] + + vertices = np.concatenate(vertices) + colors = np.concatenate(colors) + faces = np.concatenate(faces) + return dict(vertices=vertices, face_colors=colors, faces=faces) + + +def show_duster_pairs(view1, view2, pred1, pred2): + import matplotlib.pyplot as pl + pl.ion() + + for e in range(len(view1['instance'])): + i = view1['idx'][e] + j = view2['idx'][e] + img1 = rgb(view1['img'][e]) + img2 = rgb(view2['img'][e]) + conf1 = pred1['conf'][e].squeeze() + conf2 = pred2['conf'][e].squeeze() + score = conf1.mean()*conf2.mean() + print(f">> Showing pair #{e} {i}-{j} {score=:g}") + pl.clf() + pl.subplot(221).imshow(img1) + pl.subplot(223).imshow(img2) + pl.subplot(222).imshow(conf1, vmin=1, vmax=30) + pl.subplot(224).imshow(conf2, vmin=1, vmax=30) + pts1 = pred1['pts3d'][e] + pts2 = pred2['pts3d_in_other_view'][e] + pl.subplots_adjust(0, 0, 1, 1, 0, 0) + if input('show pointcloud? (y/n) ') == 'y': + show_raw_pointcloud(cat(pts1, pts2), cat(img1, img2), point_size=5) + + +def auto_cam_size(im_poses): + return 0.1 * get_med_dist_between_poses(im_poses) + + +class SceneViz: + def __init__(self): + self.scene = trimesh.Scene() + + def add_rgbd(self, image, depth, intrinsics=None, cam2world=None, zfar=np.inf, mask=None): + image = img_to_arr(image) + + # make up some intrinsics + if intrinsics is None: + H, W, THREE = image.shape + focal = max(H, W) + intrinsics = np.float32([[focal, 0, W/2], [0, focal, H/2], [0, 0, 1]]) + + # compute 3d points + pts3d = depthmap_to_pts3d(depth, intrinsics, cam2world=cam2world) + + return self.add_pointcloud(pts3d, image, mask=(depth 150) + mask |= (hsv[:, :, 1] < 30) & (hsv[:, :, 2] > 180) + mask |= (hsv[:, :, 1] < 50) & (hsv[:, :, 2] > 220) + + # Morphological operations + kernel = np.ones((5, 5), np.uint8) + mask2 = ndimage.binary_opening(mask, structure=kernel) + + # keep only largest CC + _, labels, stats, _ = cv2.connectedComponentsWithStats(mask2.view(np.uint8), connectivity=8) + cc_sizes = stats[1:, cv2.CC_STAT_AREA] + order = cc_sizes.argsort()[::-1] # bigger first + i = 0 + selection = [] + while i < len(order) and cc_sizes[order[i]] > cc_sizes[order[0]] / 2: + selection.append(1 + order[i]) + i += 1 + mask3 = np.in1d(labels, selection).reshape(labels.shape) + + # Apply mask + return torch.from_numpy(mask3) diff --git a/submodules/mast3r/dust3r/dust3r_visloc/README.md b/submodules/mast3r/dust3r/dust3r_visloc/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6d0512ac1516ba2655a10d4ae3d10b51346e233c --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r_visloc/README.md @@ -0,0 +1,93 @@ +# Visual Localization with DUSt3R + +## Dataset preparation + +### CambridgeLandmarks + +Each subscene should look like this: + +``` +Cambridge_Landmarks +├─ mapping +│ ├─ GreatCourt +│ │ └─ colmap/reconstruction +│ │ ├─ cameras.txt +│ │ ├─ images.txt +│ │ └─ points3D.txt +├─ kapture +│ ├─ GreatCourt +│ │ └─ query # https://github.com/naver/kapture/blob/main/doc/datasets.adoc#cambridge-landmarks +│ ... +├─ GreatCourt +│ ├─ pairsfile/query +│ │ └─ AP-GeM-LM18_top50.txt # https://github.com/naver/deep-image-retrieval/blob/master/dirtorch/extract_kapture.py followed by https://github.com/naver/kapture-localization/blob/main/tools/kapture_compute_image_pairs.py +│ ├─ seq1 +│ ... +... +``` + +### 7Scenes +Each subscene should look like this: + +``` +7-scenes +├─ chess +│ ├─ mapping/ # https://github.com/naver/kapture/blob/main/doc/datasets.adoc#1-7-scenes +│ ├─ query/ # https://github.com/naver/kapture/blob/main/doc/datasets.adoc#1-7-scenes +│ └─ pairsfile/query/ +│ └─ APGeM-LM18_top20.txt # https://github.com/naver/deep-image-retrieval/blob/master/dirtorch/extract_kapture.py followed by https://github.com/naver/kapture-localization/blob/main/tools/kapture_compute_image_pairs.py +... +``` + +### Aachen-Day-Night + +``` +Aachen-Day-Night-v1.1 +├─ mapping +│ ├─ colmap/reconstruction +│ │ ├─ cameras.txt +│ │ ├─ images.txt +│ │ └─ points3D.txt +├─ kapture +│ └─ query # https://github.com/naver/kapture/blob/main/doc/datasets.adoc#2-aachen-day-night-v11 +├─ images +│ ├─ db +│ ├─ query +│ └─ sequences +└─ pairsfile/query + └─ fire_top50.txt # https://github.com/naver/fire/blob/main/kapture_compute_pairs.py +``` + +### InLoc + +``` +InLoc +├─ mapping # https://github.com/naver/kapture/blob/main/doc/datasets.adoc#6-inloc +├─ query # https://github.com/naver/kapture/blob/main/doc/datasets.adoc#6-inloc +└─ pairsfile/query + └─ pairs-query-netvlad40-temporal.txt # https://github.com/cvg/Hierarchical-Localization/blob/master/pairs/inloc/pairs-query-netvlad40-temporal.txt +``` + +## Example Commands + +With `visloc.py` you can run our visual localization experiments on Aachen-Day-Night, InLoc, Cambridge Landmarks and 7 Scenes. + +```bash +# Aachen-Day-Night-v1.1: +# scene in 'day' 'night' +# scene can also be 'all' +python3 visloc.py --model_name DUSt3R_ViTLarge_BaseDecoder_512_dpt --dataset "VislocAachenDayNight('/path/to/prepared/Aachen-Day-Night-v1.1/', subscene='${scene}', pairsfile='fire_top50', topk=20)" --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/Aachen-Day-Night-v1.1/${scene}/loc + +# InLoc +python3 visloc.py --model_name DUSt3R_ViTLarge_BaseDecoder_512_dpt --dataset "VislocInLoc('/path/to/prepared/InLoc/', pairsfile='pairs-query-netvlad40-temporal', topk=20)" --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/InLoc/loc + + +# 7-scenes: +# scene in 'chess' 'fire' 'heads' 'office' 'pumpkin' 'redkitchen' 'stairs' +python3 visloc.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt --dataset "VislocSevenScenes('/path/to/prepared/7-scenes/', subscene='${scene}', pairsfile='APGeM-LM18_top20', topk=1)" --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/7-scenes/${scene}/loc + +# Cambridge Landmarks: +# scene in 'ShopFacade' 'GreatCourt' 'KingsCollege' 'OldHospital' 'StMarysChurch' +python3 visloc.py --model_name DUSt3R_ViTLarge_BaseDecoder_512_dpt --dataset "VislocCambridgeLandmarks('/path/to/prepared/Cambridge_Landmarks/', subscene='${scene}', pairsfile='APGeM-LM18_top20', topk=1)" --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/Cambridge_Landmarks/${scene}/loc + +``` \ No newline at end of file diff --git a/submodules/mast3r/dust3r/dust3r_visloc/__init__.py b/submodules/mast3r/dust3r/dust3r_visloc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a32692113d830ddc4af4e6ed608f222fbe062e6e --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r_visloc/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). diff --git a/submodules/mast3r/dust3r/dust3r_visloc/datasets/__init__.py b/submodules/mast3r/dust3r/dust3r_visloc/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..566926b1e248e4b64fc5182031af634435bb8601 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r_visloc/datasets/__init__.py @@ -0,0 +1,6 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +from .sevenscenes import VislocSevenScenes +from .cambridge_landmarks import VislocCambridgeLandmarks +from .aachen_day_night import VislocAachenDayNight +from .inloc import VislocInLoc diff --git a/submodules/mast3r/dust3r/dust3r_visloc/datasets/aachen_day_night.py b/submodules/mast3r/dust3r/dust3r_visloc/datasets/aachen_day_night.py new file mode 100644 index 0000000000000000000000000000000000000000..159548e8b51a1b5872a2392cd9107ff96e40e801 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r_visloc/datasets/aachen_day_night.py @@ -0,0 +1,24 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# AachenDayNight dataloader +# -------------------------------------------------------- +import os +from dust3r_visloc.datasets.base_colmap import BaseVislocColmapDataset + + +class VislocAachenDayNight(BaseVislocColmapDataset): + def __init__(self, root, subscene, pairsfile, topk=1, cache_sfm=False): + assert subscene in [None, '', 'day', 'night', 'all'] + self.subscene = subscene + image_path = os.path.join(root, 'images') + map_path = os.path.join(root, 'mapping/colmap/reconstruction') + query_path = os.path.join(root, 'kapture', 'query') + pairsfile_path = os.path.join(root, 'pairsfile/query', pairsfile + '.txt') + super().__init__(image_path=image_path, map_path=map_path, + query_path=query_path, pairsfile_path=pairsfile_path, + topk=topk, cache_sfm=cache_sfm) + self.scenes = [filename for filename in self.scenes if filename in self.pairs] + if self.subscene == 'day' or self.subscene == 'night': + self.scenes = [filename for filename in self.scenes if self.subscene in filename] diff --git a/submodules/mast3r/dust3r/dust3r_visloc/datasets/base_colmap.py b/submodules/mast3r/dust3r/dust3r_visloc/datasets/base_colmap.py new file mode 100644 index 0000000000000000000000000000000000000000..5dc2d64f69fb0954a148f0f4170508fe2045e046 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r_visloc/datasets/base_colmap.py @@ -0,0 +1,282 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Base class for colmap / kapture +# -------------------------------------------------------- +import os +import numpy as np +from tqdm import tqdm +import collections +import pickle +import PIL.Image +import torch +from scipy.spatial.transform import Rotation +import torchvision.transforms as tvf + +from kapture.core import CameraType +from kapture.io.csv import kapture_from_dir +from kapture_localization.utils.pairsfile import get_ordered_pairs_from_file + +from dust3r_visloc.datasets.utils import cam_to_world_from_kapture, get_resize_function, rescale_points3d +from dust3r_visloc.datasets.base_dataset import BaseVislocDataset +from dust3r.datasets.utils.transforms import ImgNorm +from dust3r.utils.geometry import colmap_to_opencv_intrinsics + +KaptureSensor = collections.namedtuple('Sensor', 'sensor_params camera_params') + + +def kapture_to_opencv_intrinsics(sensor): + """ + Convert from Kapture to OpenCV parameters. + Warning: we assume that the camera and pixel coordinates follow Colmap conventions here. + Args: + sensor: Kapture sensor + """ + sensor_type = sensor.sensor_params[0] + if sensor_type == "SIMPLE_PINHOLE": + # Simple pinhole model. + # We still call OpenCV undistorsion however for code simplicity. + w, h, f, cx, cy = sensor.camera_params + k1 = 0 + k2 = 0 + p1 = 0 + p2 = 0 + fx = fy = f + elif sensor_type == "PINHOLE": + w, h, fx, fy, cx, cy = sensor.camera_params + k1 = 0 + k2 = 0 + p1 = 0 + p2 = 0 + elif sensor_type == "SIMPLE_RADIAL": + w, h, f, cx, cy, k1 = sensor.camera_params + k2 = 0 + p1 = 0 + p2 = 0 + fx = fy = f + elif sensor_type == "RADIAL": + w, h, f, cx, cy, k1, k2 = sensor.camera_params + p1 = 0 + p2 = 0 + fx = fy = f + elif sensor_type == "OPENCV": + w, h, fx, fy, cx, cy, k1, k2, p1, p2 = sensor.camera_params + else: + raise NotImplementedError(f"Sensor type {sensor_type} is not supported yet.") + + cameraMatrix = np.asarray([[fx, 0, cx], + [0, fy, cy], + [0, 0, 1]], dtype=np.float32) + + # We assume that Kapture data comes from Colmap: the origin is different. + cameraMatrix = colmap_to_opencv_intrinsics(cameraMatrix) + + distCoeffs = np.asarray([k1, k2, p1, p2], dtype=np.float32) + return cameraMatrix, distCoeffs, (w, h) + + +def K_from_colmap(elems): + sensor = KaptureSensor(elems, tuple(map(float, elems[1:]))) + cameraMatrix, distCoeffs, (w, h) = kapture_to_opencv_intrinsics(sensor) + res = dict(resolution=(w, h), + intrinsics=cameraMatrix, + distortion=distCoeffs) + return res + + +def pose_from_qwxyz_txyz(elems): + qw, qx, qy, qz, tx, ty, tz = map(float, elems) + pose = np.eye(4) + pose[:3, :3] = Rotation.from_quat((qx, qy, qz, qw)).as_matrix() + pose[:3, 3] = (tx, ty, tz) + return np.linalg.inv(pose) # returns cam2world + + +class BaseVislocColmapDataset(BaseVislocDataset): + def __init__(self, image_path, map_path, query_path, pairsfile_path, topk=1, cache_sfm=False): + super().__init__() + self.topk = topk + self.num_views = self.topk + 1 + self.image_path = image_path + self.cache_sfm = cache_sfm + + self._load_sfm(map_path) + + kdata_query = kapture_from_dir(query_path) + assert kdata_query.records_camera is not None and kdata_query.trajectories is not None + + kdata_query_searchindex = {kdata_query.records_camera[(timestamp, sensor_id)]: (timestamp, sensor_id) + for timestamp, sensor_id in kdata_query.records_camera.key_pairs()} + self.query_data = {'kdata': kdata_query, 'searchindex': kdata_query_searchindex} + + self.pairs = get_ordered_pairs_from_file(pairsfile_path) + self.scenes = kdata_query.records_camera.data_list() + + def _load_sfm(self, sfm_dir): + sfm_cache_path = os.path.join(sfm_dir, 'dust3r_cache.pkl') + if os.path.isfile(sfm_cache_path) and self.cache_sfm: + with open(sfm_cache_path, "rb") as f: + data = pickle.load(f) + self.img_infos = data['img_infos'] + self.points3D = data['points3D'] + return + + # load cameras + with open(os.path.join(sfm_dir, 'cameras.txt'), 'r') as f: + raw = f.read().splitlines()[3:] # skip header + + intrinsics = {} + for camera in tqdm(raw): + camera = camera.split(' ') + intrinsics[int(camera[0])] = K_from_colmap(camera[1:]) + + # load images + with open(os.path.join(sfm_dir, 'images.txt'), 'r') as f: + raw = f.read().splitlines() + raw = [line for line in raw if not line.startswith('#')] # skip header + + self.img_infos = {} + for image, points in tqdm(zip(raw[0::2], raw[1::2]), total=len(raw) // 2): + image = image.split(' ') + points = points.split(' ') + + img_name = image[-1] + current_points2D = {int(i): (float(x), float(y)) + for i, x, y in zip(points[2::3], points[0::3], points[1::3]) if i != '-1'} + self.img_infos[img_name] = dict(intrinsics[int(image[-2])], + path=img_name, + camera_pose=pose_from_qwxyz_txyz(image[1: -2]), + sparse_pts2d=current_points2D) + + # load 3D points + with open(os.path.join(sfm_dir, 'points3D.txt'), 'r') as f: + raw = f.read().splitlines() + raw = [line for line in raw if not line.startswith('#')] # skip header + + self.points3D = {} + for point in tqdm(raw): + point = point.split() + self.points3D[int(point[0])] = tuple(map(float, point[1:4])) + + if self.cache_sfm: + to_save = \ + { + 'img_infos': self.img_infos, + 'points3D': self.points3D + } + with open(sfm_cache_path, "wb") as f: + pickle.dump(to_save, f) + + def __len__(self): + return len(self.scenes) + + def _get_view_query(self, imgname): + kdata, searchindex = map(self.query_data.get, ['kdata', 'searchindex']) + + timestamp, camera_id = searchindex[imgname] + + camera_params = kdata.sensors[camera_id].camera_params + if kdata.sensors[camera_id].camera_type == CameraType.SIMPLE_PINHOLE: + W, H, f, cx, cy = camera_params + k1 = 0 + fx = fy = f + elif kdata.sensors[camera_id].camera_type == CameraType.SIMPLE_RADIAL: + W, H, f, cx, cy, k1 = camera_params + fx = fy = f + else: + raise NotImplementedError('not implemented') + + W, H = int(W), int(H) + intrinsics = np.float32([(fx, 0, cx), + (0, fy, cy), + (0, 0, 1)]) + intrinsics = colmap_to_opencv_intrinsics(intrinsics) + distortion = [k1, 0, 0, 0] + + if kdata.trajectories is not None and (timestamp, camera_id) in kdata.trajectories: + cam_to_world = cam_to_world_from_kapture(kdata, timestamp, camera_id) + else: + cam_to_world = np.eye(4, dtype=np.float32) + + # Load RGB image + rgb_image = PIL.Image.open(os.path.join(self.image_path, imgname)).convert('RGB') + rgb_image.load() + resize_func, _, to_orig = get_resize_function(self.maxdim, self.patch_size, H, W) + rgb_tensor = resize_func(ImgNorm(rgb_image)) + + view = { + 'intrinsics': intrinsics, + 'distortion': distortion, + 'cam_to_world': cam_to_world, + 'rgb': rgb_image, + 'rgb_rescaled': rgb_tensor, + 'to_orig': to_orig, + 'idx': 0, + 'image_name': imgname + } + return view + + def _get_view_map(self, imgname, idx): + infos = self.img_infos[imgname] + + rgb_image = PIL.Image.open(os.path.join(self.image_path, infos['path'])).convert('RGB') + rgb_image.load() + W, H = rgb_image.size + intrinsics = infos['intrinsics'] + intrinsics = colmap_to_opencv_intrinsics(intrinsics) + distortion_coefs = infos['distortion'] + + pts2d = infos['sparse_pts2d'] + sparse_pos2d = np.float32(list(pts2d.values())) # pts2d from colmap + sparse_pts3d = np.float32([self.points3D[i] for i in pts2d]) + + # store full resolution 2D->3D + sparse_pos2d_cv2 = sparse_pos2d.copy() + sparse_pos2d_cv2[:, 0] -= 0.5 + sparse_pos2d_cv2[:, 1] -= 0.5 + sparse_pos2d_int = sparse_pos2d_cv2.round().astype(np.int64) + valid = (sparse_pos2d_int[:, 0] >= 0) & (sparse_pos2d_int[:, 0] < W) & ( + sparse_pos2d_int[:, 1] >= 0) & (sparse_pos2d_int[:, 1] < H) + sparse_pos2d_int = sparse_pos2d_int[valid] + # nan => invalid + pts3d = np.full((H, W, 3), np.nan, dtype=np.float32) + pts3d[sparse_pos2d_int[:, 1], sparse_pos2d_int[:, 0]] = sparse_pts3d[valid] + pts3d = torch.from_numpy(pts3d) + + cam_to_world = infos['camera_pose'] # cam2world + + # also store resized resolution 2D->3D + resize_func, to_resize, to_orig = get_resize_function(self.maxdim, self.patch_size, H, W) + rgb_tensor = resize_func(ImgNorm(rgb_image)) + + HR, WR = rgb_tensor.shape[1:] + _, _, pts3d_rescaled, valid_rescaled = rescale_points3d(sparse_pos2d_cv2, sparse_pts3d, to_resize, HR, WR) + pts3d_rescaled = torch.from_numpy(pts3d_rescaled) + valid_rescaled = torch.from_numpy(valid_rescaled) + + view = { + 'intrinsics': intrinsics, + 'distortion': distortion_coefs, + 'cam_to_world': cam_to_world, + 'rgb': rgb_image, + "pts3d": pts3d, + "valid": pts3d.sum(dim=-1).isfinite(), + 'rgb_rescaled': rgb_tensor, + "pts3d_rescaled": pts3d_rescaled, + "valid_rescaled": valid_rescaled, + 'to_orig': to_orig, + 'idx': idx, + 'image_name': imgname + } + return view + + def __getitem__(self, idx): + assert self.maxdim is not None and self.patch_size is not None + query_image = self.scenes[idx] + map_images = [p[0] for p in self.pairs[query_image][:self.topk]] + views = [] + views.append(self._get_view_query(query_image)) + for idx, map_image in enumerate(map_images): + views.append(self._get_view_map(map_image, idx + 1)) + return views diff --git a/submodules/mast3r/dust3r/dust3r_visloc/datasets/base_dataset.py b/submodules/mast3r/dust3r/dust3r_visloc/datasets/base_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..cda3774c5ab5b668be5eecf89681abc96df5fe17 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r_visloc/datasets/base_dataset.py @@ -0,0 +1,19 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Base class +# -------------------------------------------------------- +class BaseVislocDataset: + def __init__(self): + pass + + def set_resolution(self, model): + self.maxdim = max(model.patch_embed.img_size) + self.patch_size = model.patch_embed.patch_size + + def __len__(self): + raise NotImplementedError() + + def __getitem__(self, idx): + raise NotImplementedError() \ No newline at end of file diff --git a/submodules/mast3r/dust3r/dust3r_visloc/datasets/cambridge_landmarks.py b/submodules/mast3r/dust3r/dust3r_visloc/datasets/cambridge_landmarks.py new file mode 100644 index 0000000000000000000000000000000000000000..ca3e131941bf444d86a709d23e518e7b93d3d0f6 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r_visloc/datasets/cambridge_landmarks.py @@ -0,0 +1,19 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Cambridge Landmarks dataloader +# -------------------------------------------------------- +import os +from dust3r_visloc.datasets.base_colmap import BaseVislocColmapDataset + + +class VislocCambridgeLandmarks (BaseVislocColmapDataset): + def __init__(self, root, subscene, pairsfile, topk=1, cache_sfm=False): + image_path = os.path.join(root, subscene) + map_path = os.path.join(root, 'mapping', subscene, 'colmap/reconstruction') + query_path = os.path.join(root, 'kapture', subscene, 'query') + pairsfile_path = os.path.join(root, subscene, 'pairsfile/query', pairsfile + '.txt') + super().__init__(image_path=image_path, map_path=map_path, + query_path=query_path, pairsfile_path=pairsfile_path, + topk=topk, cache_sfm=cache_sfm) \ No newline at end of file diff --git a/submodules/mast3r/dust3r/dust3r_visloc/datasets/inloc.py b/submodules/mast3r/dust3r/dust3r_visloc/datasets/inloc.py new file mode 100644 index 0000000000000000000000000000000000000000..99ed11f554203d353d0559d0589f40ec1ffbf66e --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r_visloc/datasets/inloc.py @@ -0,0 +1,167 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# InLoc dataloader +# -------------------------------------------------------- +import os +import numpy as np +import torch +import PIL.Image +import scipy.io + +import kapture +from kapture.io.csv import kapture_from_dir +from kapture_localization.utils.pairsfile import get_ordered_pairs_from_file + +from dust3r_visloc.datasets.utils import cam_to_world_from_kapture, get_resize_function, rescale_points3d +from dust3r_visloc.datasets.base_dataset import BaseVislocDataset +from dust3r.datasets.utils.transforms import ImgNorm +from dust3r.utils.geometry import xy_grid, geotrf + + +def read_alignments(path_to_alignment): + aligns = {} + with open(path_to_alignment, "r") as fid: + while True: + line = fid.readline() + if not line: + break + if len(line) == 4: + trans_nr = line[:-1] + while line != 'After general icp:\n': + line = fid.readline() + line = fid.readline() + p = [] + for i in range(4): + elems = line.split(' ') + line = fid.readline() + for e in elems: + if len(e) != 0: + p.append(float(e)) + P = np.array(p).reshape(4, 4) + aligns[trans_nr] = P + return aligns + + +class VislocInLoc(BaseVislocDataset): + def __init__(self, root, pairsfile, topk=1): + super().__init__() + self.root = root + self.topk = topk + self.num_views = self.topk + 1 + self.maxdim = None + self.patch_size = None + + query_path = os.path.join(self.root, 'query') + kdata_query = kapture_from_dir(query_path) + assert kdata_query.records_camera is not None + kdata_query_searchindex = {kdata_query.records_camera[(timestamp, sensor_id)]: (timestamp, sensor_id) + for timestamp, sensor_id in kdata_query.records_camera.key_pairs()} + self.query_data = {'path': query_path, 'kdata': kdata_query, 'searchindex': kdata_query_searchindex} + + map_path = os.path.join(self.root, 'mapping') + kdata_map = kapture_from_dir(map_path) + assert kdata_map.records_camera is not None and kdata_map.trajectories is not None + kdata_map_searchindex = {kdata_map.records_camera[(timestamp, sensor_id)]: (timestamp, sensor_id) + for timestamp, sensor_id in kdata_map.records_camera.key_pairs()} + self.map_data = {'path': map_path, 'kdata': kdata_map, 'searchindex': kdata_map_searchindex} + + try: + self.pairs = get_ordered_pairs_from_file(os.path.join(self.root, 'pairfiles/query', pairsfile + '.txt')) + except Exception as e: + # if using pairs from hloc + self.pairs = {} + with open(os.path.join(self.root, 'pairfiles/query', pairsfile + '.txt'), 'r') as fid: + lines = fid.readlines() + for line in lines: + splits = line.rstrip("\n\r").split(" ") + self.pairs.setdefault(splits[0].replace('query/', ''), []).append( + (splits[1].replace('database/cutouts/', ''), 1.0) + ) + + self.scenes = kdata_query.records_camera.data_list() + + self.aligns_DUC1 = read_alignments(os.path.join(self.root, 'mapping/DUC1_alignment/all_transformations.txt')) + self.aligns_DUC2 = read_alignments(os.path.join(self.root, 'mapping/DUC2_alignment/all_transformations.txt')) + + def __len__(self): + return len(self.scenes) + + def __getitem__(self, idx): + assert self.maxdim is not None and self.patch_size is not None + query_image = self.scenes[idx] + map_images = [p[0] for p in self.pairs[query_image][:self.topk]] + views = [] + dataarray = [(query_image, self.query_data, False)] + [(map_image, self.map_data, True) + for map_image in map_images] + for idx, (imgname, data, should_load_depth) in enumerate(dataarray): + imgpath, kdata, searchindex = map(data.get, ['path', 'kdata', 'searchindex']) + + timestamp, camera_id = searchindex[imgname] + + # for InLoc, SIMPLE_PINHOLE + camera_params = kdata.sensors[camera_id].camera_params + W, H, f, cx, cy = camera_params + distortion = [0, 0, 0, 0] + intrinsics = np.float32([(f, 0, cx), + (0, f, cy), + (0, 0, 1)]) + + if kdata.trajectories is not None and (timestamp, camera_id) in kdata.trajectories: + cam_to_world = cam_to_world_from_kapture(kdata, timestamp, camera_id) + else: + cam_to_world = np.eye(4, dtype=np.float32) + + # Load RGB image + rgb_image = PIL.Image.open(os.path.join(imgpath, 'sensors/records_data', imgname)).convert('RGB') + rgb_image.load() + + W, H = rgb_image.size + resize_func, to_resize, to_orig = get_resize_function(self.maxdim, self.patch_size, H, W) + + rgb_tensor = resize_func(ImgNorm(rgb_image)) + + view = { + 'intrinsics': intrinsics, + 'distortion': distortion, + 'cam_to_world': cam_to_world, + 'rgb': rgb_image, + 'rgb_rescaled': rgb_tensor, + 'to_orig': to_orig, + 'idx': idx, + 'image_name': imgname + } + + # Load depthmap + if should_load_depth: + depthmap_filename = os.path.join(imgpath, 'sensors/records_data', imgname + '.mat') + depthmap = scipy.io.loadmat(depthmap_filename) + + pt3d_cut = depthmap['XYZcut'] + scene_id = imgname.replace('\\', '/').split('/')[1] + if imgname.startswith('DUC1'): + pts3d_full = geotrf(self.aligns_DUC1[scene_id], pt3d_cut) + else: + pts3d_full = geotrf(self.aligns_DUC2[scene_id], pt3d_cut) + + pts3d_valid = np.isfinite(pts3d_full.sum(axis=-1)) + + pts3d = pts3d_full[pts3d_valid] + pts2d_int = xy_grid(W, H)[pts3d_valid] + pts2d = pts2d_int.astype(np.float64) + + # nan => invalid + pts3d_full[~pts3d_valid] = np.nan + pts3d_full = torch.from_numpy(pts3d_full) + view['pts3d'] = pts3d_full + view["valid"] = pts3d_full.sum(dim=-1).isfinite() + + HR, WR = rgb_tensor.shape[1:] + _, _, pts3d_rescaled, valid_rescaled = rescale_points3d(pts2d, pts3d, to_resize, HR, WR) + pts3d_rescaled = torch.from_numpy(pts3d_rescaled) + valid_rescaled = torch.from_numpy(valid_rescaled) + view['pts3d_rescaled'] = pts3d_rescaled + view["valid_rescaled"] = valid_rescaled + views.append(view) + return views diff --git a/submodules/mast3r/dust3r/dust3r_visloc/datasets/sevenscenes.py b/submodules/mast3r/dust3r/dust3r_visloc/datasets/sevenscenes.py new file mode 100644 index 0000000000000000000000000000000000000000..c15e851d262f0d7ba7071c933d8fe8f0a6b1c49d --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r_visloc/datasets/sevenscenes.py @@ -0,0 +1,123 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# 7 Scenes dataloader +# -------------------------------------------------------- +import os +import numpy as np +import torch +import PIL.Image + +import kapture +from kapture.io.csv import kapture_from_dir +from kapture_localization.utils.pairsfile import get_ordered_pairs_from_file +from kapture.io.records import depth_map_from_file + +from dust3r_visloc.datasets.utils import cam_to_world_from_kapture, get_resize_function, rescale_points3d +from dust3r_visloc.datasets.base_dataset import BaseVislocDataset +from dust3r.datasets.utils.transforms import ImgNorm +from dust3r.utils.geometry import depthmap_to_absolute_camera_coordinates, xy_grid, geotrf + + +class VislocSevenScenes(BaseVislocDataset): + def __init__(self, root, subscene, pairsfile, topk=1): + super().__init__() + self.root = root + self.subscene = subscene + self.topk = topk + self.num_views = self.topk + 1 + self.maxdim = None + self.patch_size = None + + query_path = os.path.join(self.root, subscene, 'query') + kdata_query = kapture_from_dir(query_path) + assert kdata_query.records_camera is not None and kdata_query.trajectories is not None and kdata_query.rigs is not None + kapture.rigs_remove_inplace(kdata_query.trajectories, kdata_query.rigs) + kdata_query_searchindex = {kdata_query.records_camera[(timestamp, sensor_id)]: (timestamp, sensor_id) + for timestamp, sensor_id in kdata_query.records_camera.key_pairs()} + self.query_data = {'path': query_path, 'kdata': kdata_query, 'searchindex': kdata_query_searchindex} + + map_path = os.path.join(self.root, subscene, 'mapping') + kdata_map = kapture_from_dir(map_path) + assert kdata_map.records_camera is not None and kdata_map.trajectories is not None and kdata_map.rigs is not None + kapture.rigs_remove_inplace(kdata_map.trajectories, kdata_map.rigs) + kdata_map_searchindex = {kdata_map.records_camera[(timestamp, sensor_id)]: (timestamp, sensor_id) + for timestamp, sensor_id in kdata_map.records_camera.key_pairs()} + self.map_data = {'path': map_path, 'kdata': kdata_map, 'searchindex': kdata_map_searchindex} + + self.pairs = get_ordered_pairs_from_file(os.path.join(self.root, subscene, + 'pairfiles/query', + pairsfile + '.txt')) + self.scenes = kdata_query.records_camera.data_list() + + def __len__(self): + return len(self.scenes) + + def __getitem__(self, idx): + assert self.maxdim is not None and self.patch_size is not None + query_image = self.scenes[idx] + map_images = [p[0] for p in self.pairs[query_image][:self.topk]] + views = [] + dataarray = [(query_image, self.query_data, False)] + [(map_image, self.map_data, True) + for map_image in map_images] + for idx, (imgname, data, should_load_depth) in enumerate(dataarray): + imgpath, kdata, searchindex = map(data.get, ['path', 'kdata', 'searchindex']) + + timestamp, camera_id = searchindex[imgname] + + # for 7scenes, SIMPLE_PINHOLE + camera_params = kdata.sensors[camera_id].camera_params + W, H, f, cx, cy = camera_params + distortion = [0, 0, 0, 0] + intrinsics = np.float32([(f, 0, cx), + (0, f, cy), + (0, 0, 1)]) + + cam_to_world = cam_to_world_from_kapture(kdata, timestamp, camera_id) + + # Load RGB image + rgb_image = PIL.Image.open(os.path.join(imgpath, 'sensors/records_data', imgname)).convert('RGB') + rgb_image.load() + + W, H = rgb_image.size + resize_func, to_resize, to_orig = get_resize_function(self.maxdim, self.patch_size, H, W) + + rgb_tensor = resize_func(ImgNorm(rgb_image)) + + view = { + 'intrinsics': intrinsics, + 'distortion': distortion, + 'cam_to_world': cam_to_world, + 'rgb': rgb_image, + 'rgb_rescaled': rgb_tensor, + 'to_orig': to_orig, + 'idx': idx, + 'image_name': imgname + } + + # Load depthmap + if should_load_depth: + depthmap_filename = os.path.join(imgpath, 'sensors/records_data', + imgname.replace('color.png', 'depth.reg')) + depthmap = depth_map_from_file(depthmap_filename, (int(W), int(H))).astype(np.float32) + pts3d_full, pts3d_valid = depthmap_to_absolute_camera_coordinates(depthmap, intrinsics, cam_to_world) + + pts3d = pts3d_full[pts3d_valid] + pts2d_int = xy_grid(W, H)[pts3d_valid] + pts2d = pts2d_int.astype(np.float64) + + # nan => invalid + pts3d_full[~pts3d_valid] = np.nan + pts3d_full = torch.from_numpy(pts3d_full) + view['pts3d'] = pts3d_full + view["valid"] = pts3d_full.sum(dim=-1).isfinite() + + HR, WR = rgb_tensor.shape[1:] + _, _, pts3d_rescaled, valid_rescaled = rescale_points3d(pts2d, pts3d, to_resize, HR, WR) + pts3d_rescaled = torch.from_numpy(pts3d_rescaled) + valid_rescaled = torch.from_numpy(valid_rescaled) + view['pts3d_rescaled'] = pts3d_rescaled + view["valid_rescaled"] = valid_rescaled + views.append(view) + return views diff --git a/submodules/mast3r/dust3r/dust3r_visloc/datasets/utils.py b/submodules/mast3r/dust3r/dust3r_visloc/datasets/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6053ae2e5ba6c0b0f5f014161b666623d6e0f3f5 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r_visloc/datasets/utils.py @@ -0,0 +1,118 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# dataset utilities +# -------------------------------------------------------- +import numpy as np +import quaternion +import torchvision.transforms as tvf +from dust3r.utils.geometry import geotrf + + +def cam_to_world_from_kapture(kdata, timestamp, camera_id): + camera_to_world = kdata.trajectories[timestamp, camera_id].inverse() + camera_pose = np.eye(4, dtype=np.float32) + camera_pose[:3, :3] = quaternion.as_rotation_matrix(camera_to_world.r) + camera_pose[:3, 3] = camera_to_world.t_raw + return camera_pose + + +ratios_resolutions = { + 224: {1.0: [224, 224]}, + 512: {4 / 3: [512, 384], 32 / 21: [512, 336], 16 / 9: [512, 288], 2 / 1: [512, 256], 16 / 5: [512, 160]} +} + + +def get_HW_resolution(H, W, maxdim, patchsize=16): + assert maxdim in ratios_resolutions, "Error, maxdim can only be 224 or 512 for now. Other maxdims not implemented yet." + ratios_resolutions_maxdim = ratios_resolutions[maxdim] + mindims = set([min(res) for res in ratios_resolutions_maxdim.values()]) + ratio = W / H + ref_ratios = np.array([*(ratios_resolutions_maxdim.keys())]) + islandscape = (W >= H) + if islandscape: + diff = np.abs(ratio - ref_ratios) + else: + diff = np.abs(ratio - (1 / ref_ratios)) + selkey = ref_ratios[np.argmin(diff)] + res = ratios_resolutions_maxdim[selkey] + # check patchsize and make sure output resolution is a multiple of patchsize + if isinstance(patchsize, tuple): + assert len(patchsize) == 2 and isinstance(patchsize[0], int) and isinstance( + patchsize[1], int), "What is your patchsize format? Expected a single int or a tuple of two ints." + assert patchsize[0] == patchsize[1], "Error, non square patches not managed" + patchsize = patchsize[0] + assert max(res) == maxdim + assert min(res) in mindims + return res[::-1] if islandscape else res # return HW + + +def get_resize_function(maxdim, patch_size, H, W, is_mask=False): + if [max(H, W), min(H, W)] in ratios_resolutions[maxdim].values(): + return lambda x: x, np.eye(3), np.eye(3) + else: + target_HW = get_HW_resolution(H, W, maxdim=maxdim, patchsize=patch_size) + + ratio = W / H + target_ratio = target_HW[1] / target_HW[0] + to_orig_crop = np.eye(3) + to_rescaled_crop = np.eye(3) + if abs(ratio - target_ratio) < np.finfo(np.float32).eps: + crop_W = W + crop_H = H + elif ratio - target_ratio < 0: + crop_W = W + crop_H = int(W / target_ratio) + to_orig_crop[1, 2] = (H - crop_H) / 2.0 + to_rescaled_crop[1, 2] = -(H - crop_H) / 2.0 + else: + crop_W = int(H * target_ratio) + crop_H = H + to_orig_crop[0, 2] = (W - crop_W) / 2.0 + to_rescaled_crop[0, 2] = - (W - crop_W) / 2.0 + + crop_op = tvf.CenterCrop([crop_H, crop_W]) + + if is_mask: + resize_op = tvf.Resize(size=target_HW, interpolation=tvf.InterpolationMode.NEAREST_EXACT) + else: + resize_op = tvf.Resize(size=target_HW) + to_orig_resize = np.array([[crop_W / target_HW[1], 0, 0], + [0, crop_H / target_HW[0], 0], + [0, 0, 1]]) + to_rescaled_resize = np.array([[target_HW[1] / crop_W, 0, 0], + [0, target_HW[0] / crop_H, 0], + [0, 0, 1]]) + + op = tvf.Compose([crop_op, resize_op]) + + return op, to_rescaled_resize @ to_rescaled_crop, to_orig_crop @ to_orig_resize + + +def rescale_points3d(pts2d, pts3d, to_resize, HR, WR): + # rescale pts2d as floats + # to colmap, so that the image is in [0, D] -> [0, NewD] + pts2d = pts2d.copy() + pts2d[:, 0] += 0.5 + pts2d[:, 1] += 0.5 + + pts2d_rescaled = geotrf(to_resize, pts2d, norm=True) + + pts2d_rescaled_int = pts2d_rescaled.copy() + # convert back to cv2 before round [-0.5, 0.5] -> pixel 0 + pts2d_rescaled_int[:, 0] -= 0.5 + pts2d_rescaled_int[:, 1] -= 0.5 + pts2d_rescaled_int = pts2d_rescaled_int.round().astype(np.int64) + + # update valid (remove cropped regions) + valid_rescaled = (pts2d_rescaled_int[:, 0] >= 0) & (pts2d_rescaled_int[:, 0] < WR) & ( + pts2d_rescaled_int[:, 1] >= 0) & (pts2d_rescaled_int[:, 1] < HR) + + pts2d_rescaled_int = pts2d_rescaled_int[valid_rescaled] + + # rebuild pts3d from rescaled ps2d poses + pts3d_rescaled = np.full((HR, WR, 3), np.nan, dtype=np.float32) # pts3d in 512 x something + pts3d_rescaled[pts2d_rescaled_int[:, 1], pts2d_rescaled_int[:, 0]] = pts3d[valid_rescaled] + + return pts2d_rescaled, pts2d_rescaled_int, pts3d_rescaled, np.isfinite(pts3d_rescaled.sum(axis=-1)) diff --git a/submodules/mast3r/dust3r/dust3r_visloc/evaluation.py b/submodules/mast3r/dust3r/dust3r_visloc/evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..027179f2b1007db558f57d3d67f48a6d7aa1ab9d --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r_visloc/evaluation.py @@ -0,0 +1,65 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# evaluation utilities +# -------------------------------------------------------- +import numpy as np +import quaternion +import torch +import roma +import collections +import os + + +def aggregate_stats(info_str, pose_errors, angular_errors): + stats = collections.Counter() + median_pos_error = np.median(pose_errors) + median_angular_error = np.median(angular_errors) + out_str = f'{info_str}: {len(pose_errors)} images - {median_pos_error=}, {median_angular_error=}' + + for trl_thr, ang_thr in [(0.1, 1), (0.25, 2), (0.5, 5), (5, 10)]: + for pose_error, angular_error in zip(pose_errors, angular_errors): + correct_for_this_threshold = (pose_error < trl_thr) and (angular_error < ang_thr) + stats[trl_thr, ang_thr] += correct_for_this_threshold + stats = {f'acc@{key[0]:g}m,{key[1]}deg': 100 * val / len(pose_errors) for key, val in stats.items()} + for metric, perf in stats.items(): + out_str += f' - {metric:12s}={float(perf):.3f}' + return out_str + + +def get_pose_error(pr_camtoworld, gt_cam_to_world): + abs_transl_error = torch.linalg.norm(torch.tensor(pr_camtoworld[:3, 3]) - torch.tensor(gt_cam_to_world[:3, 3])) + abs_angular_error = roma.rotmat_geodesic_distance(torch.tensor(pr_camtoworld[:3, :3]), + torch.tensor(gt_cam_to_world[:3, :3])) * 180 / np.pi + return abs_transl_error, abs_angular_error + + +def export_results(output_dir, xp_label, query_names, poses_pred): + if output_dir is not None: + os.makedirs(output_dir, exist_ok=True) + + lines = "" + lines_ltvl = "" + for query_name, pr_querycam_to_world in zip(query_names, poses_pred): + if pr_querycam_to_world is None: + pr_world_to_querycam = np.eye(4) + else: + pr_world_to_querycam = np.linalg.inv(pr_querycam_to_world) + query_shortname = os.path.basename(query_name) + pr_world_to_querycam_q = quaternion.from_rotation_matrix(pr_world_to_querycam[:3, :3]) + pr_world_to_querycam_t = pr_world_to_querycam[:3, 3] + + line_pose = quaternion.as_float_array(pr_world_to_querycam_q).tolist() + \ + pr_world_to_querycam_t.flatten().tolist() + + line_content = [query_name] + line_pose + lines += ' '.join(str(v) for v in line_content) + '\n' + + line_content_ltvl = [query_shortname] + line_pose + lines_ltvl += ' '.join(str(v) for v in line_content_ltvl) + '\n' + + with open(os.path.join(output_dir, xp_label + '_results.txt'), 'wt') as f: + f.write(lines) + with open(os.path.join(output_dir, xp_label + '_ltvl.txt'), 'wt') as f: + f.write(lines_ltvl) diff --git a/submodules/mast3r/dust3r/dust3r_visloc/localization.py b/submodules/mast3r/dust3r/dust3r_visloc/localization.py new file mode 100644 index 0000000000000000000000000000000000000000..ac8ae198dc3479f12a976bab0bda692328880710 --- /dev/null +++ b/submodules/mast3r/dust3r/dust3r_visloc/localization.py @@ -0,0 +1,140 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# main pnp code +# -------------------------------------------------------- +import numpy as np +import quaternion +import cv2 +from packaging import version + +from dust3r.utils.geometry import opencv_to_colmap_intrinsics + +try: + import poselib # noqa + HAS_POSELIB = True +except Exception as e: + HAS_POSELIB = False + +try: + import pycolmap # noqa + version_number = pycolmap.__version__ + if version.parse(version_number) < version.parse("0.5.0"): + HAS_PYCOLMAP = False + else: + HAS_PYCOLMAP = True +except Exception as e: + HAS_PYCOLMAP = False + +def run_pnp(pts2D, pts3D, K, distortion = None, mode='cv2', reprojectionError=5, img_size = None): + """ + use OPENCV model for distortion (4 values) + """ + assert mode in ['cv2', 'poselib', 'pycolmap'] + try: + if len(pts2D) > 4 and mode == "cv2": + confidence = 0.9999 + iterationsCount = 10_000 + if distortion is not None: + cv2_pts2ds = np.copy(pts2D) + cv2_pts2ds = cv2.undistortPoints(cv2_pts2ds, K, np.array(distortion), R=None, P=K) + pts2D = cv2_pts2ds.reshape((-1, 2)) + + success, r_pose, t_pose, _ = cv2.solvePnPRansac(pts3D, pts2D, K, None, flags=cv2.SOLVEPNP_SQPNP, + iterationsCount=iterationsCount, + reprojectionError=reprojectionError, + confidence=confidence) + if not success: + return False, None + r_pose = cv2.Rodrigues(r_pose)[0] # world2cam == world2cam2 + RT = np.r_[np.c_[r_pose, t_pose], [(0,0,0,1)]] # world2cam2 + return True, np.linalg.inv(RT) # cam2toworld + elif len(pts2D) > 4 and mode == "poselib": + assert HAS_POSELIB + confidence = 0.9999 + iterationsCount = 10_000 + # NOTE: `Camera` struct currently contains `width`/`height` fields, + # however these are not used anywhere in the code-base and are provided simply to be consistent with COLMAP. + # so we put garbage in there + colmap_intrinsics = opencv_to_colmap_intrinsics(K) + fx = colmap_intrinsics[0, 0] + fy = colmap_intrinsics[1, 1] + cx = colmap_intrinsics[0, 2] + cy = colmap_intrinsics[1, 2] + width = img_size[0] if img_size is not None else int(cx*2) + height = img_size[1] if img_size is not None else int(cy*2) + + if distortion is None: + camera = {'model': 'PINHOLE', 'width': width, 'height': height, 'params': [fx, fy, cx, cy]} + else: + camera = {'model': 'OPENCV', 'width': width, 'height': height, + 'params': [fx, fy, cx, cy] + distortion} + + pts2D = np.copy(pts2D) + pts2D[:, 0] += 0.5 + pts2D[:, 1] += 0.5 + pose, _ = poselib.estimate_absolute_pose(pts2D, pts3D, camera, + {'max_reproj_error': reprojectionError, + 'max_iterations': iterationsCount, + 'success_prob': confidence}, {}) + if pose is None: + return False, None + RT = pose.Rt # (3x4) + RT = np.r_[RT, [(0,0,0,1)]] # world2cam + return True, np.linalg.inv(RT) # cam2toworld + elif len(pts2D) > 4 and mode == "pycolmap": + assert HAS_PYCOLMAP + assert img_size is not None + + pts2D = np.copy(pts2D) + pts2D[:, 0] += 0.5 + pts2D[:, 1] += 0.5 + colmap_intrinsics = opencv_to_colmap_intrinsics(K) + fx = colmap_intrinsics[0, 0] + fy = colmap_intrinsics[1, 1] + cx = colmap_intrinsics[0, 2] + cy = colmap_intrinsics[1, 2] + width = img_size[0] + height = img_size[1] + if distortion is None: + camera_dict = {'model': 'PINHOLE', 'width': width, 'height': height, 'params': [fx, fy, cx, cy]} + else: + camera_dict = {'model': 'OPENCV', 'width': width, 'height': height, + 'params': [fx, fy, cx, cy] + distortion} + + pycolmap_camera = pycolmap.Camera( + model=camera_dict['model'], width=camera_dict['width'], height=camera_dict['height'], + params=camera_dict['params']) + + pycolmap_estimation_options = dict(ransac=dict(max_error=reprojectionError, min_inlier_ratio=0.01, + min_num_trials=1000, max_num_trials=100000, + confidence=0.9999)) + pycolmap_refinement_options=dict(refine_focal_length=False, refine_extra_params=False) + ret = pycolmap.absolute_pose_estimation(pts2D, pts3D, pycolmap_camera, + estimation_options=pycolmap_estimation_options, + refinement_options=pycolmap_refinement_options) + if ret is None: + ret = {'success': False} + else: + ret['success'] = True + if callable(ret['cam_from_world'].matrix): + retmat = ret['cam_from_world'].matrix() + else: + retmat = ret['cam_from_world'].matrix + ret['qvec'] = quaternion.from_rotation_matrix(retmat[:3, :3]) + ret['tvec'] = retmat[:3, 3] + + if not (ret['success'] and ret['num_inliers'] > 0): + success = False + pose = None + else: + success = True + pr_world_to_querycam = np.r_[ret['cam_from_world'].matrix(), [(0,0,0,1)]] + pose = np.linalg.inv(pr_world_to_querycam) + return success, pose + else: + return False, None + except Exception as e: + print(f'error during pnp: {e}') + return False, None \ No newline at end of file diff --git a/submodules/mast3r/dust3r/requirements.txt b/submodules/mast3r/dust3r/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f781884b76d339066f61804a8fe1b5da31c49593 --- /dev/null +++ b/submodules/mast3r/dust3r/requirements.txt @@ -0,0 +1,13 @@ +# torch +# torchvision +roma +gradio +matplotlib +tqdm +opencv-python +scipy +einops +trimesh +tensorboard +pyglet<2 +huggingface-hub[torch]>=0.22 \ No newline at end of file diff --git a/submodules/mast3r/dust3r/requirements_optional.txt b/submodules/mast3r/dust3r/requirements_optional.txt new file mode 100644 index 0000000000000000000000000000000000000000..d42662c0e87c6ce4ac990f2afedecc96cdea7f06 --- /dev/null +++ b/submodules/mast3r/dust3r/requirements_optional.txt @@ -0,0 +1,7 @@ +pillow-heif # add heif/heic image support +pyrender # for rendering depths in scannetpp +kapture # for visloc data loading +kapture-localization +numpy-quaternion +pycolmap # for pnp +poselib # for pnp diff --git a/submodules/mast3r/dust3r/train.py b/submodules/mast3r/dust3r/train.py new file mode 100644 index 0000000000000000000000000000000000000000..503e63572376c259e6b259850e19c3f6036aa535 --- /dev/null +++ b/submodules/mast3r/dust3r/train.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# training executable for DUSt3R +# -------------------------------------------------------- +from dust3r.training import get_args_parser, train + +if __name__ == '__main__': + args = get_args_parser() + args = args.parse_args() + train(args) diff --git a/submodules/mast3r/dust3r/visloc.py b/submodules/mast3r/dust3r/visloc.py new file mode 100644 index 0000000000000000000000000000000000000000..6411b3eaf96dea961f9524e887a12d92f2012c6b --- /dev/null +++ b/submodules/mast3r/dust3r/visloc.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Simple visloc script +# -------------------------------------------------------- +import numpy as np +import random +import argparse +from tqdm import tqdm +import math + +from dust3r.inference import inference +from dust3r.model import AsymmetricCroCo3DStereo +from dust3r.utils.geometry import find_reciprocal_matches, xy_grid, geotrf + +from dust3r_visloc.datasets import * +from dust3r_visloc.localization import run_pnp +from dust3r_visloc.evaluation import get_pose_error, aggregate_stats, export_results + + +def get_args_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=str, required=True, help="visloc dataset to eval") + parser_weights = parser.add_mutually_exclusive_group(required=True) + parser_weights.add_argument("--weights", type=str, help="path to the model weights", default=None) + parser_weights.add_argument("--model_name", type=str, help="name of the model weights", + choices=["DUSt3R_ViTLarge_BaseDecoder_512_dpt", + "DUSt3R_ViTLarge_BaseDecoder_512_linear", + "DUSt3R_ViTLarge_BaseDecoder_224_linear"]) + parser.add_argument("--confidence_threshold", type=float, default=3.0, + help="confidence values higher than threshold are invalid") + parser.add_argument("--device", type=str, default='cuda', help="pytorch device") + parser.add_argument("--pnp_mode", type=str, default="cv2", choices=['cv2', 'poselib', 'pycolmap'], + help="pnp lib to use") + parser_reproj = parser.add_mutually_exclusive_group() + parser_reproj.add_argument("--reprojection_error", type=float, default=5.0, help="pnp reprojection error") + parser_reproj.add_argument("--reprojection_error_diag_ratio", type=float, default=None, + help="pnp reprojection error as a ratio of the diagonal of the image") + + parser.add_argument("--pnp_max_points", type=int, default=100_000, help="pnp maximum number of points kept") + parser.add_argument("--viz_matches", type=int, default=0, help="debug matches") + + parser.add_argument("--output_dir", type=str, default=None, help="output path") + parser.add_argument("--output_label", type=str, default='', help="prefix for results files") + return parser + + +if __name__ == '__main__': + parser = get_args_parser() + args = parser.parse_args() + conf_thr = args.confidence_threshold + device = args.device + pnp_mode = args.pnp_mode + reprojection_error = args.reprojection_error + reprojection_error_diag_ratio = args.reprojection_error_diag_ratio + pnp_max_points = args.pnp_max_points + viz_matches = args.viz_matches + + if args.weights is not None: + weights_path = args.weights + else: + weights_path = "naver/" + args.model_name + model = AsymmetricCroCo3DStereo.from_pretrained(weights_path).to(args.device) + + dataset = eval(args.dataset) + dataset.set_resolution(model) + + query_names = [] + poses_pred = [] + pose_errors = [] + angular_errors = [] + for idx in tqdm(range(len(dataset))): + views = dataset[(idx)] # 0 is the query + query_view = views[0] + map_views = views[1:] + query_names.append(query_view['image_name']) + + query_pts2d = [] + query_pts3d = [] + for map_view in map_views: + # prepare batch + imgs = [] + for idx, img in enumerate([query_view['rgb_rescaled'], map_view['rgb_rescaled']]): + imgs.append(dict(img=img.unsqueeze(0), true_shape=np.int32([img.shape[1:]]), + idx=idx, instance=str(idx))) + output = inference([tuple(imgs)], model, device, batch_size=1, verbose=False) + pred1, pred2 = output['pred1'], output['pred2'] + confidence_masks = [pred1['conf'].squeeze(0) >= conf_thr, + (pred2['conf'].squeeze(0) >= conf_thr) & map_view['valid_rescaled']] + pts3d = [pred1['pts3d'].squeeze(0), pred2['pts3d_in_other_view'].squeeze(0)] + + # find 2D-2D matches between the two images + pts2d_list, pts3d_list = [], [] + for i in range(2): + conf_i = confidence_masks[i].cpu().numpy() + true_shape_i = imgs[i]['true_shape'][0] + pts2d_list.append(xy_grid(true_shape_i[1], true_shape_i[0])[conf_i]) + pts3d_list.append(pts3d[i].detach().cpu().numpy()[conf_i]) + + PQ, PM = pts3d_list[0], pts3d_list[1] + if len(PQ) == 0 or len(PM) == 0: + continue + reciprocal_in_PM, nnM_in_PQ, num_matches = find_reciprocal_matches(PQ, PM) + if viz_matches > 0: + print(f'found {num_matches} matches') + matches_im1 = pts2d_list[1][reciprocal_in_PM] + matches_im0 = pts2d_list[0][nnM_in_PQ][reciprocal_in_PM] + valid_pts3d = map_view['pts3d_rescaled'][matches_im1[:, 1], matches_im1[:, 0]] + + # from cv2 to colmap + matches_im0 = matches_im0.astype(np.float64) + matches_im1 = matches_im1.astype(np.float64) + matches_im0[:, 0] += 0.5 + matches_im0[:, 1] += 0.5 + matches_im1[:, 0] += 0.5 + matches_im1[:, 1] += 0.5 + # rescale coordinates + matches_im0 = geotrf(query_view['to_orig'], matches_im0, norm=True) + matches_im1 = geotrf(query_view['to_orig'], matches_im1, norm=True) + # from colmap back to cv2 + matches_im0[:, 0] -= 0.5 + matches_im0[:, 1] -= 0.5 + matches_im1[:, 0] -= 0.5 + matches_im1[:, 1] -= 0.5 + + # visualize a few matches + if viz_matches > 0: + viz_imgs = [np.array(query_view['rgb']), np.array(map_view['rgb'])] + from matplotlib import pyplot as pl + n_viz = viz_matches + match_idx_to_viz = np.round(np.linspace(0, num_matches - 1, n_viz)).astype(int) + viz_matches_im0, viz_matches_im1 = matches_im0[match_idx_to_viz], matches_im1[match_idx_to_viz] + + H0, W0, H1, W1 = *viz_imgs[0].shape[:2], *viz_imgs[1].shape[:2] + img0 = np.pad(viz_imgs[0], ((0, max(H1 - H0, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img1 = np.pad(viz_imgs[1], ((0, max(H0 - H1, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img = np.concatenate((img0, img1), axis=1) + pl.figure() + pl.imshow(img) + cmap = pl.get_cmap('jet') + for i in range(n_viz): + (x0, y0), (x1, y1) = viz_matches_im0[i].T, viz_matches_im1[i].T + pl.plot([x0, x1 + W0], [y0, y1], '-+', color=cmap(i / (n_viz - 1)), scalex=False, scaley=False) + pl.show(block=True) + + if len(valid_pts3d) == 0: + pass + else: + query_pts3d.append(valid_pts3d.cpu().numpy()) + query_pts2d.append(matches_im0) + + if len(query_pts2d) == 0: + success = False + pr_querycam_to_world = None + else: + query_pts2d = np.concatenate(query_pts2d, axis=0).astype(np.float32) + query_pts3d = np.concatenate(query_pts3d, axis=0) + if len(query_pts2d) > pnp_max_points: + idxs = random.sample(range(len(query_pts2d)), pnp_max_points) + query_pts3d = query_pts3d[idxs] + query_pts2d = query_pts2d[idxs] + + W, H = query_view['rgb'].size + if reprojection_error_diag_ratio is not None: + reprojection_error_img = reprojection_error_diag_ratio * math.sqrt(W**2 + H**2) + else: + reprojection_error_img = reprojection_error + success, pr_querycam_to_world = run_pnp(query_pts2d, query_pts3d, + query_view['intrinsics'], query_view['distortion'], + pnp_mode, reprojection_error_img, img_size=[W, H]) + + if not success: + abs_transl_error = float('inf') + abs_angular_error = float('inf') + else: + abs_transl_error, abs_angular_error = get_pose_error(pr_querycam_to_world, query_view['cam_to_world']) + + pose_errors.append(abs_transl_error) + angular_errors.append(abs_angular_error) + poses_pred.append(pr_querycam_to_world) + + xp_label = f'tol_conf_{conf_thr}' + if args.output_label: + xp_label = args.output_label + '_' + xp_label + if reprojection_error_diag_ratio is not None: + xp_label = xp_label + f'_reproj_diag_{reprojection_error_diag_ratio}' + else: + xp_label = xp_label + f'_reproj_err_{reprojection_error}' + export_results(args.output_dir, xp_label, query_names, poses_pred) + out_string = aggregate_stats(f'{args.dataset}', pose_errors, angular_errors) + print(out_string) diff --git a/submodules/mast3r/mast3r/__init__.py b/submodules/mast3r/mast3r/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7dd877d649ce4dbd749dd7195a8b34c0f91d4f0 --- /dev/null +++ b/submodules/mast3r/mast3r/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). \ No newline at end of file diff --git a/submodules/mast3r/mast3r/catmlp_dpt_head.py b/submodules/mast3r/mast3r/catmlp_dpt_head.py new file mode 100644 index 0000000000000000000000000000000000000000..ac4457908f97e7e25c4c59cc696fb059791fbff8 --- /dev/null +++ b/submodules/mast3r/mast3r/catmlp_dpt_head.py @@ -0,0 +1,123 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# MASt3R heads +# -------------------------------------------------------- +import torch +import torch.nn.functional as F + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.heads.postprocess import reg_dense_depth, reg_dense_conf # noqa +from dust3r.heads.dpt_head import PixelwiseTaskWithDPT # noqa +import dust3r.utils.path_to_croco # noqa +from models.blocks import Mlp # noqa + + +def reg_desc(desc, mode): + if 'norm' in mode: + desc = desc / desc.norm(dim=-1, keepdim=True) + else: + raise ValueError(f"Unknown desc mode {mode}") + return desc + + +def postprocess(out, depth_mode, conf_mode, desc_dim=None, desc_mode='norm', two_confs=False, desc_conf_mode=None): + if desc_conf_mode is None: + desc_conf_mode = conf_mode + fmap = out.permute(0, 2, 3, 1) # B,H,W,D + res = dict(pts3d=reg_dense_depth(fmap[..., 0:3], mode=depth_mode)) + if conf_mode is not None: + res['conf'] = reg_dense_conf(fmap[..., 3], mode=conf_mode) + if desc_dim is not None: + start = 3 + int(conf_mode is not None) + res['desc'] = reg_desc(fmap[..., start:start + desc_dim], mode=desc_mode) + if two_confs: + res['desc_conf'] = reg_dense_conf(fmap[..., start + desc_dim], mode=desc_conf_mode) + else: + res['desc_conf'] = res['conf'].clone() + return res + + +class Cat_MLP_LocalFeatures_DPT_Pts3d(PixelwiseTaskWithDPT): + """ Mixture between MLP and DPT head that outputs 3d points and local features (with MLP). + The input for both heads is a concatenation of Encoder and Decoder outputs + """ + + def __init__(self, net, has_conf=False, local_feat_dim=16, hidden_dim_factor=4., hooks_idx=None, dim_tokens=None, + num_channels=1, postprocess=None, feature_dim=256, last_dim=32, depth_mode=None, conf_mode=None, head_type="regression", **kwargs): + super().__init__(num_channels=num_channels, feature_dim=feature_dim, last_dim=last_dim, hooks_idx=hooks_idx, + dim_tokens=dim_tokens, depth_mode=depth_mode, postprocess=postprocess, conf_mode=conf_mode, head_type=head_type) + self.local_feat_dim = local_feat_dim + + patch_size = net.patch_embed.patch_size + if isinstance(patch_size, tuple): + assert len(patch_size) == 2 and isinstance(patch_size[0], int) and isinstance( + patch_size[1], int), "What is your patchsize format? Expected a single int or a tuple of two ints." + assert patch_size[0] == patch_size[1], "Error, non square patches not managed" + patch_size = patch_size[0] + self.patch_size = patch_size + + self.desc_mode = net.desc_mode + self.has_conf = has_conf + self.two_confs = net.two_confs # independent confs for 3D regr and descs + self.desc_conf_mode = net.desc_conf_mode + idim = net.enc_embed_dim + net.dec_embed_dim + + self.head_local_features = Mlp(in_features=idim, + hidden_features=int(hidden_dim_factor * idim), + out_features=(self.local_feat_dim + self.two_confs) * self.patch_size**2) + + def forward(self, decout, img_shape): + # pass through the heads + pts3d = self.dpt(decout, image_size=(img_shape[0], img_shape[1])) + + # recover encoder and decoder outputs + enc_output, dec_output = decout[0], decout[-1] + cat_output = torch.cat([enc_output, dec_output], dim=-1) # concatenate + H, W = img_shape + B, S, D = cat_output.shape + + # extract local_features + local_features = self.head_local_features(cat_output) # B,S,D + local_features = local_features.transpose(-1, -2).view(B, -1, H // self.patch_size, W // self.patch_size) + local_features = F.pixel_shuffle(local_features, self.patch_size) # B,d,H,W + + # post process 3D pts, descriptors and confidences + out = torch.cat([pts3d, local_features], dim=1) + if self.postprocess: + out = self.postprocess(out, + depth_mode=self.depth_mode, + conf_mode=self.conf_mode, + desc_dim=self.local_feat_dim, + desc_mode=self.desc_mode, + two_confs=self.two_confs, + desc_conf_mode=self.desc_conf_mode) + return out + + +def mast3r_head_factory(head_type, output_mode, net, has_conf=False): + """" build a prediction head for the decoder + """ + if head_type == 'catmlp+dpt' and output_mode.startswith('pts3d+desc'): + local_feat_dim = int(output_mode[10:]) + assert net.dec_depth > 9 + l2 = net.dec_depth + feature_dim = 256 + last_dim = feature_dim // 2 + out_nchan = 3 + ed = net.enc_embed_dim + dd = net.dec_embed_dim + return Cat_MLP_LocalFeatures_DPT_Pts3d(net, local_feat_dim=local_feat_dim, has_conf=has_conf, + num_channels=out_nchan + has_conf, + feature_dim=feature_dim, + last_dim=last_dim, + hooks_idx=[0, l2 * 2 // 4, l2 * 3 // 4, l2], + dim_tokens=[ed, dd, dd, dd], + postprocess=postprocess, + depth_mode=net.depth_mode, + conf_mode=net.conf_mode, + head_type='regression') + else: + raise NotImplementedError( + f"unexpected {head_type=} and {output_mode=}") diff --git a/submodules/mast3r/mast3r/cloud_opt/__init__.py b/submodules/mast3r/mast3r/cloud_opt/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7dd877d649ce4dbd749dd7195a8b34c0f91d4f0 --- /dev/null +++ b/submodules/mast3r/mast3r/cloud_opt/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). \ No newline at end of file diff --git a/submodules/mast3r/mast3r/cloud_opt/sparse_ga.py b/submodules/mast3r/mast3r/cloud_opt/sparse_ga.py new file mode 100644 index 0000000000000000000000000000000000000000..eb1eb6b4d264e458d4efdc4e50281f1d0c7c4012 --- /dev/null +++ b/submodules/mast3r/mast3r/cloud_opt/sparse_ga.py @@ -0,0 +1,1040 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# MASt3R Sparse Global Alignement +# -------------------------------------------------------- +from tqdm import tqdm +import roma +import torch +import torch.nn as nn +import torch.nn.functional as F +import numpy as np +import os +from collections import namedtuple +from functools import lru_cache +from scipy import sparse as sp +import copy + +from mast3r.utils.misc import mkdir_for, hash_md5 +from mast3r.cloud_opt.utils.losses import gamma_loss +from mast3r.cloud_opt.utils.schedules import linear_schedule, cosine_schedule +from mast3r.fast_nn import fast_reciprocal_NNs, merge_corres + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.utils.geometry import inv, geotrf # noqa +from dust3r.utils.device import to_cpu, to_numpy, todevice # noqa +from dust3r.post_process import estimate_focal_knowing_depth # noqa +from dust3r.optim_factory import adjust_learning_rate_by_lr # noqa +from dust3r.cloud_opt.base_opt import clean_pointcloud +from dust3r.viz import SceneViz + + +class SparseGA(): + def __init__(self, img_paths, pairs_in, res_fine, anchors, canonical_paths=None): + def fetch_img(im): + def torgb(x): return (x[0].permute(1, 2, 0).numpy() * .5 + .5).clip(min=0., max=1.) + for im1, im2 in pairs_in: + if im1['instance'] == im: + return torgb(im1['img']) + if im2['instance'] == im: + return torgb(im2['img']) + self.canonical_paths = canonical_paths + self.img_paths = img_paths + self.imgs = [fetch_img(img) for img in img_paths] + self.intrinsics = res_fine['intrinsics'] + self.cam2w = res_fine['cam2w'] + self.depthmaps = res_fine['depthmaps'] + self.pts3d = res_fine['pts3d'] + self.pts3d_colors = [] + self.working_device = self.cam2w.device + for i in range(len(self.imgs)): + im = self.imgs[i] + x, y = anchors[i][0][..., :2].detach().cpu().numpy().T + self.pts3d_colors.append(im[y, x]) + assert self.pts3d_colors[-1].shape == self.pts3d[i].shape + self.n_imgs = len(self.imgs) + + def get_focals(self): + return torch.tensor([ff[0, 0] for ff in self.intrinsics]).to(self.working_device) + + def get_principal_points(self): + return torch.stack([ff[:2, -1] for ff in self.intrinsics]).to(self.working_device) + + def get_im_poses(self): + return self.cam2w + + def get_sparse_pts3d(self): + return self.pts3d + + def get_dense_pts3d(self, clean_depth=True, subsample=8): + assert self.canonical_paths, 'cache_path is required for dense 3d points' + device = self.cam2w.device + confs = [] + base_focals = [] + anchors = {} + for i, canon_path in enumerate(self.canonical_paths): + (canon, canon2, conf), focal = torch.load(canon_path, map_location=device) + confs.append(conf) + base_focals.append(focal) + + H, W = conf.shape + pixels = torch.from_numpy(np.mgrid[:W, :H].T.reshape(-1, 2)).float().to(device) + idxs, offsets = anchor_depth_offsets(canon2, {i: (pixels, None)}, subsample=subsample) + anchors[i] = (pixels, idxs[i], offsets[i]) + + # densify sparse depthmaps + pts3d, depthmaps = make_pts3d(anchors, self.intrinsics, self.cam2w, [ + d.ravel() for d in self.depthmaps], base_focals=base_focals, ret_depth=True) + + if clean_depth: + confs = clean_pointcloud(confs, self.intrinsics, inv(self.cam2w), depthmaps, pts3d) + + return pts3d, depthmaps, confs + + def get_pts3d_colors(self): + return self.pts3d_colors + + def get_depthmaps(self): + return self.depthmaps + + def get_masks(self): + return [slice(None, None) for _ in range(len(self.imgs))] + + def show(self, show_cams=True): + pts3d, _, confs = self.get_dense_pts3d() + show_reconstruction(self.imgs, self.intrinsics if show_cams else None, self.cam2w, + [p.clip(min=-50, max=50) for p in pts3d], + masks=[c > 1 for c in confs]) + + +def convert_dust3r_pairs_naming(imgs, pairs_in): + for pair_id in range(len(pairs_in)): + for i in range(2): + pairs_in[pair_id][i]['instance'] = imgs[pairs_in[pair_id][i]['idx']] + return pairs_in + + +def sparse_global_alignment(imgs, pairs_in, cache_path, model, subsample=8, desc_conf='desc_conf', + device='cuda', dtype=torch.float32, shared_intrinsics=False, **kw): + """ Sparse alignment with MASt3R + imgs: list of image paths + cache_path: path where to dump temporary files (str) + + lr1, niter1: learning rate and #iterations for coarse global alignment (3D matching) + lr2, niter2: learning rate and #iterations for refinement (2D reproj error) + + lora_depth: smart dimensionality reduction with depthmaps + """ + # Convert pair naming convention from dust3r to mast3r + pairs_in = convert_dust3r_pairs_naming(imgs, pairs_in) + # forward pass + pairs, cache_path = forward_mast3r(pairs_in, model, + cache_path=cache_path, subsample=subsample, + desc_conf=desc_conf, device=device) + + # extract canonical pointmaps + tmp_pairs, pairwise_scores, canonical_views, canonical_paths, preds_21 = \ + prepare_canonical_data(imgs, pairs, subsample, cache_path=cache_path, mode='avg-angle', device=device) + + # compute minimal spanning tree + mst = compute_min_spanning_tree(pairwise_scores) + + # remove all edges not in the spanning tree? + # min_spanning_tree = {(imgs[i],imgs[j]) for i,j in mst[1]} + # tmp_pairs = {(a,b):v for (a,b),v in tmp_pairs.items() if {(a,b),(b,a)} & min_spanning_tree} + + # smartly combine all useful data + imsizes, pps, base_focals, core_depth, anchors, corres, corres2d, preds_21 = \ + condense_data(imgs, tmp_pairs, canonical_views, preds_21, dtype) + + imgs, res_coarse, res_fine = sparse_scene_optimizer( + imgs, subsample, imsizes, pps, base_focals, core_depth, anchors, corres, corres2d, preds_21, canonical_paths, mst, + shared_intrinsics=shared_intrinsics, cache_path=cache_path, device=device, dtype=dtype, **kw) + + return SparseGA(imgs, pairs_in, res_fine or res_coarse, anchors, canonical_paths) + + +def sparse_scene_optimizer(imgs, subsample, imsizes, pps, base_focals, core_depth, anchors, corres, corres2d, + preds_21, canonical_paths, mst, cache_path, + lr1=0.2, niter1=500, loss1=gamma_loss(1.1), + lr2=0.02, niter2=500, loss2=gamma_loss(0.4), + lossd=gamma_loss(1.1), + opt_pp=True, opt_depth=True, + schedule=cosine_schedule, depth_mode='add', exp_depth=False, + lora_depth=False, # dict(k=96, gamma=15, min_norm=.5), + shared_intrinsics=False, + init={}, device='cuda', dtype=torch.float32, + matching_conf_thr=5., loss_dust3r_w=0.01, + verbose=True, dbg=()): + init = copy.deepcopy(init) + # extrinsic parameters + vec0001 = torch.tensor((0, 0, 0, 1), dtype=dtype, device=device) + quats = [nn.Parameter(vec0001.clone()) for _ in range(len(imgs))] + trans = [nn.Parameter(torch.zeros(3, device=device, dtype=dtype)) for _ in range(len(imgs))] + + # initialize + ones = torch.ones((len(imgs), 1), device=device, dtype=dtype) + median_depths = torch.ones(len(imgs), device=device, dtype=dtype) + for img in imgs: + idx = imgs.index(img) + init_values = init.setdefault(img, {}) + if verbose and init_values: + print(f' >> initializing img=...{img[-25:]} [{idx}] for {set(init_values)}') + + K = init_values.get('intrinsics') + if K is not None: + K = K.detach() + focal = K[:2, :2].diag().mean() + pp = K[:2, 2] + base_focals[idx] = focal + pps[idx] = pp + pps[idx] /= imsizes[idx] # default principal_point would be (0.5, 0.5) + + depth = init_values.get('depthmap') + if depth is not None: + core_depth[idx] = depth.detach() + + median_depths[idx] = med_depth = core_depth[idx].median() + core_depth[idx] /= med_depth + + cam2w = init_values.get('cam2w') + if cam2w is not None: + rot = cam2w[:3, :3].detach() + cam_center = cam2w[:3, 3].detach() + quats[idx].data[:] = roma.rotmat_to_unitquat(rot) + trans_offset = med_depth * torch.cat((imsizes[idx] / base_focals[idx] * (0.5 - pps[idx]), ones[:1, 0])) + trans[idx].data[:] = cam_center + rot @ trans_offset + del rot + assert False, 'inverse kinematic chain not yet implemented' + + # intrinsics parameters + if shared_intrinsics: + # Optimize a single set of intrinsics for all cameras. Use averages as init. + confs = torch.stack([torch.load(pth)[0][2].mean() for pth in canonical_paths]).to(pps) + weighting = confs / confs.sum() + pp = nn.Parameter((weighting @ pps).to(dtype)) + pps = [pp for _ in range(len(imgs))] + focal_m = weighting @ base_focals + log_focal = nn.Parameter(focal_m.view(1).log().to(dtype)) + log_focals = [log_focal for _ in range(len(imgs))] + else: + pps = [nn.Parameter(pp.to(dtype)) for pp in pps] + log_focals = [nn.Parameter(f.view(1).log().to(dtype)) for f in base_focals] + + diags = imsizes.float().norm(dim=1) + min_focals = 0.25 * diags # diag = 1.2~1.4*max(W,H) => beta >= 1/(2*1.2*tan(fov/2)) ~= 0.26 + max_focals = 10 * diags + + assert len(mst[1]) == len(pps) - 1 + + def make_K_cam_depth(log_focals, pps, trans, quats, log_sizes, core_depth): + # make intrinsics + focals = torch.cat(log_focals).exp().clip(min=min_focals, max=max_focals) + pps = torch.stack(pps) + K = torch.eye(3, dtype=dtype, device=device)[None].expand(len(imgs), 3, 3).clone() + K[:, 0, 0] = K[:, 1, 1] = focals + K[:, 0:2, 2] = pps * imsizes + if trans is None: + return K + + # security! optimization is always trying to crush the scale down + sizes = torch.cat(log_sizes).exp() + global_scaling = 1 / sizes.min() + + # compute distance of camera to focal plane + # tan(fov) = W/2 / focal + z_cameras = sizes * median_depths * focals / base_focals + + # make extrinsic + rel_cam2cam = torch.eye(4, dtype=dtype, device=device)[None].expand(len(imgs), 4, 4).clone() + rel_cam2cam[:, :3, :3] = roma.unitquat_to_rotmat(F.normalize(torch.stack(quats), dim=1)) + rel_cam2cam[:, :3, 3] = torch.stack(trans) + + # camera are defined as a kinematic chain + tmp_cam2w = [None] * len(K) + tmp_cam2w[mst[0]] = rel_cam2cam[mst[0]] + for i, j in mst[1]: + # i is the cam_i_to_world reference, j is the relative pose = cam_j_to_cam_i + tmp_cam2w[j] = tmp_cam2w[i] @ rel_cam2cam[j] + tmp_cam2w = torch.stack(tmp_cam2w) + + # smart reparameterizaton of cameras + trans_offset = z_cameras.unsqueeze(1) * torch.cat((imsizes / focals.unsqueeze(1) * (0.5 - pps), ones), dim=-1) + new_trans = global_scaling * (tmp_cam2w[:, :3, 3:4] - tmp_cam2w[:, :3, :3] @ trans_offset.unsqueeze(-1)) + cam2w = torch.cat((torch.cat((tmp_cam2w[:, :3, :3], new_trans), dim=2), + vec0001.view(1, 1, 4).expand(len(K), 1, 4)), dim=1) + + depthmaps = [] + for i in range(len(imgs)): + core_depth_img = core_depth[i] + if exp_depth: + core_depth_img = core_depth_img.exp() + if lora_depth: # compute core_depth as a low-rank decomposition of 3d points + core_depth_img = lora_depth_proj[i] @ core_depth_img + if depth_mode == 'add': + core_depth_img = z_cameras[i] + (core_depth_img - 1) * (median_depths[i] * sizes[i]) + elif depth_mode == 'mul': + core_depth_img = z_cameras[i] * core_depth_img + else: + raise ValueError(f'Bad {depth_mode=}') + depthmaps.append(global_scaling * core_depth_img) + + return K, (inv(cam2w), cam2w), depthmaps + + K = make_K_cam_depth(log_focals, pps, None, None, None, None) + + if shared_intrinsics: + print('init focal (shared) = ', to_numpy(K[0, 0, 0]).round(2)) + else: + print('init focals =', to_numpy(K[:, 0, 0])) + + # spectral low-rank projection of depthmaps + if lora_depth: + core_depth, lora_depth_proj = spectral_projection_of_depthmaps( + imgs, K, core_depth, subsample, cache_path=cache_path, **lora_depth) + if exp_depth: + core_depth = [d.clip(min=1e-4).log() for d in core_depth] + core_depth = [nn.Parameter(d.ravel().to(dtype)) for d in core_depth] + log_sizes = [nn.Parameter(torch.zeros(1, dtype=dtype, device=device)) for _ in range(len(imgs))] + + # Fetch img slices + _, confs_sum, imgs_slices = corres + + # Define which pairs are fine to use with matching + def matching_check(x): return x.max() > matching_conf_thr + is_matching_ok = {} + for s in imgs_slices: + is_matching_ok[s.img1, s.img2] = matching_check(s.confs) + + # Prepare slices and corres for losses + dust3r_slices = [s for s in imgs_slices if not is_matching_ok[s.img1, s.img2]] + loss3d_slices = [s for s in imgs_slices if is_matching_ok[s.img1, s.img2]] + cleaned_corres2d = [] + for cci, (img1, pix1, confs, confsum, imgs_slices) in enumerate(corres2d): + cf_sum = 0 + pix1_filtered = [] + confs_filtered = [] + curstep = 0 + cleaned_slices = [] + for img2, slice2 in imgs_slices: + if is_matching_ok[img1, img2]: + tslice = slice(curstep, curstep + slice2.stop - slice2.start, slice2.step) + pix1_filtered.append(pix1[tslice]) + confs_filtered.append(confs[tslice]) + cleaned_slices.append((img2, slice2)) + curstep += slice2.stop - slice2.start + if pix1_filtered != []: + pix1_filtered = torch.cat(pix1_filtered) + confs_filtered = torch.cat(confs_filtered) + cf_sum = confs_filtered.sum() + cleaned_corres2d.append((img1, pix1_filtered, confs_filtered, cf_sum, cleaned_slices)) + + def loss_dust3r(cam2w, pts3d, pix_loss): + # In the case no correspondence could be established, fallback to DUSt3R GA regression loss formulation (sparsified) + loss = 0. + cf_sum = 0. + for s in dust3r_slices: + if init[imgs[s.img1]].get('freeze') and init[imgs[s.img2]].get('freeze'): + continue + # fallback to dust3r regression + tgt_pts, tgt_confs = preds_21[imgs[s.img2]][imgs[s.img1]] + tgt_pts = geotrf(cam2w[s.img2], tgt_pts) + cf_sum += tgt_confs.sum() + loss += tgt_confs @ pix_loss(pts3d[s.img1], tgt_pts) + return loss / cf_sum if cf_sum != 0. else 0. + + def loss_3d(K, w2cam, pts3d, pix_loss): + # For each correspondence, we have two 3D points (one for each image of the pair). + # For each 3D point, we have 2 reproj errors + if any(v.get('freeze') for v in init.values()): + pts3d_1 = [] + pts3d_2 = [] + confs = [] + for s in loss3d_slices: + if init[imgs[s.img1]].get('freeze') and init[imgs[s.img2]].get('freeze'): + continue + pts3d_1.append(pts3d[s.img1][s.slice1]) + pts3d_2.append(pts3d[s.img2][s.slice2]) + confs.append(s.confs) + else: + pts3d_1 = [pts3d[s.img1][s.slice1] for s in loss3d_slices] + pts3d_2 = [pts3d[s.img2][s.slice2] for s in loss3d_slices] + confs = [s.confs for s in loss3d_slices] + + if pts3d_1 != []: + confs = torch.cat(confs) + pts3d_1 = torch.cat(pts3d_1) + pts3d_2 = torch.cat(pts3d_2) + loss = confs @ pix_loss(pts3d_1, pts3d_2) + cf_sum = confs.sum() + else: + loss = 0. + cf_sum = 1. + + return loss / cf_sum + + def loss_2d(K, w2cam, pts3d, pix_loss): + # For each correspondence, we have two 3D points (one for each image of the pair). + # For each 3D point, we have 2 reproj errors + proj_matrix = K @ w2cam[:, :3] + loss = npix = 0 + for img1, pix1_filtered, confs_filtered, cf_sum, cleaned_slices in cleaned_corres2d: + if init[imgs[img1]].get('freeze', 0) >= 1: + continue # no need + pts3d_in_img1 = [pts3d[img2][slice2] for img2, slice2 in cleaned_slices] + if pts3d_in_img1 != []: + pts3d_in_img1 = torch.cat(pts3d_in_img1) + loss += confs_filtered @ pix_loss(pix1_filtered, reproj2d(proj_matrix[img1], pts3d_in_img1)) + npix += confs_filtered.sum() + + return loss / npix if npix != 0 else 0. + + def optimize_loop(loss_func, lr_base, niter, pix_loss, lr_end=0): + # create optimizer + params = pps + log_focals + quats + trans + log_sizes + core_depth + optimizer = torch.optim.Adam(params, lr=1, weight_decay=0, betas=(0.9, 0.9)) + ploss = pix_loss if 'meta' in repr(pix_loss) else (lambda a: pix_loss) + + with tqdm(total=niter) as bar: + for iter in range(niter or 1): + K, (w2cam, cam2w), depthmaps = make_K_cam_depth(log_focals, pps, trans, quats, log_sizes, core_depth) + pts3d = make_pts3d(anchors, K, cam2w, depthmaps, base_focals=base_focals) + if niter == 0: + break + + alpha = (iter / niter) + lr = schedule(alpha, lr_base, lr_end) + adjust_learning_rate_by_lr(optimizer, lr) + pix_loss = ploss(1 - alpha) + optimizer.zero_grad() + loss = loss_func(K, w2cam, pts3d, pix_loss) + loss_dust3r_w * loss_dust3r(cam2w, pts3d, lossd) + loss.backward() + optimizer.step() + + # make sure the pose remains well optimizable + for i in range(len(imgs)): + quats[i].data[:] /= quats[i].data.norm() + + loss = float(loss) + if loss != loss: + break # NaN loss + bar.set_postfix_str(f'{lr=:.4f}, {loss=:.3f}') + bar.update(1) + + if niter: + print(f'>> final loss = {loss}') + return dict(intrinsics=K.detach(), cam2w=cam2w.detach(), + depthmaps=[d.detach() for d in depthmaps], pts3d=[p.detach() for p in pts3d]) + + # at start, don't optimize 3d points + for i, img in enumerate(imgs): + trainable = not (init[img].get('freeze')) + pps[i].requires_grad_(False) + log_focals[i].requires_grad_(False) + quats[i].requires_grad_(trainable) + trans[i].requires_grad_(trainable) + log_sizes[i].requires_grad_(trainable) + core_depth[i].requires_grad_(False) + + res_coarse = optimize_loop(loss_3d, lr_base=lr1, niter=niter1, pix_loss=loss1) + + res_fine = None + if niter2: + # now we can optimize 3d points + for i, img in enumerate(imgs): + if init[img].get('freeze', 0) >= 1: + continue + pps[i].requires_grad_(bool(opt_pp)) + log_focals[i].requires_grad_(True) + core_depth[i].requires_grad_(opt_depth) + + # refinement with 2d reproj + res_fine = optimize_loop(loss_2d, lr_base=lr2, niter=niter2, pix_loss=loss2) + + K = make_K_cam_depth(log_focals, pps, None, None, None, None) + if shared_intrinsics: + print('Final focal (shared) = ', to_numpy(K[0, 0, 0]).round(2)) + else: + print('Final focals =', to_numpy(K[:, 0, 0])) + + return imgs, res_coarse, res_fine + + +@lru_cache +def mask110(device, dtype): + return torch.tensor((1, 1, 0), device=device, dtype=dtype) + + +def proj3d(inv_K, pixels, z): + if pixels.shape[-1] == 2: + pixels = torch.cat((pixels, torch.ones_like(pixels[..., :1])), dim=-1) + return z.unsqueeze(-1) * (pixels * inv_K.diag() + inv_K[:, 2] * mask110(z.device, z.dtype)) + + +def make_pts3d(anchors, K, cam2w, depthmaps, base_focals=None, ret_depth=False): + focals = K[:, 0, 0] + invK = inv(K) + all_pts3d = [] + depth_out = [] + + for img, (pixels, idxs, offsets) in anchors.items(): + # from depthmaps to 3d points + if base_focals is None: + pass + else: + # compensate for focal + # depth + depth * (offset - 1) * base_focal / focal + # = depth * (1 + (offset - 1) * (base_focal / focal)) + offsets = 1 + (offsets - 1) * (base_focals[img] / focals[img]) + + pts3d = proj3d(invK[img], pixels, depthmaps[img][idxs] * offsets) + if ret_depth: + depth_out.append(pts3d[..., 2]) # before camera rotation + + # rotate to world coordinate + pts3d = geotrf(cam2w[img], pts3d) + all_pts3d.append(pts3d) + + if ret_depth: + return all_pts3d, depth_out + return all_pts3d + + +def make_dense_pts3d(intrinsics, cam2w, depthmaps, canonical_paths, subsample, device='cuda'): + base_focals = [] + anchors = {} + confs = [] + for i, canon_path in enumerate(canonical_paths): + (canon, canon2, conf), focal = torch.load(canon_path, map_location=device) + confs.append(conf) + base_focals.append(focal) + H, W = conf.shape + pixels = torch.from_numpy(np.mgrid[:W, :H].T.reshape(-1, 2)).float().to(device) + idxs, offsets = anchor_depth_offsets(canon2, {i: (pixels, None)}, subsample=subsample) + anchors[i] = (pixels, idxs[i], offsets[i]) + + # densify sparse depthmaps + pts3d, depthmaps_out = make_pts3d(anchors, intrinsics, cam2w, [ + d.ravel() for d in depthmaps], base_focals=base_focals, ret_depth=True) + + return pts3d, depthmaps_out, confs + + +@torch.no_grad() +def forward_mast3r(pairs, model, cache_path, desc_conf='desc_conf', + device='cuda', subsample=8, **matching_kw): + res_paths = {} + + for img1, img2 in tqdm(pairs): + idx1 = hash_md5(img1['instance']) + idx2 = hash_md5(img2['instance']) + + path1 = cache_path + f'/forward/{idx1}/{idx2}.pth' + path2 = cache_path + f'/forward/{idx2}/{idx1}.pth' + path_corres = cache_path + f'/corres_conf={desc_conf}_{subsample=}/{idx1}-{idx2}.pth' + path_corres2 = cache_path + f'/corres_conf={desc_conf}_{subsample=}/{idx2}-{idx1}.pth' + + if os.path.isfile(path_corres2) and not os.path.isfile(path_corres): + score, (xy1, xy2, confs) = torch.load(path_corres2) + torch.save((score, (xy2, xy1, confs)), path_corres) + + if not all(os.path.isfile(p) for p in (path1, path2, path_corres)): + if model is None: + continue + res = symmetric_inference(model, img1, img2, device=device) + X11, X21, X22, X12 = [r['pts3d'][0] for r in res] + C11, C21, C22, C12 = [r['conf'][0] for r in res] + descs = [r['desc'][0] for r in res] + qonfs = [r[desc_conf][0] for r in res] + + # save + torch.save(to_cpu((X11, C11, X21, C21)), mkdir_for(path1)) + torch.save(to_cpu((X22, C22, X12, C12)), mkdir_for(path2)) + + # perform reciprocal matching + corres = extract_correspondences(descs, qonfs, device=device, subsample=subsample) + + conf_score = (C11.mean() * C12.mean() * C21.mean() * C22.mean()).sqrt().sqrt() + matching_score = (float(conf_score), float(corres[2].sum()), len(corres[2])) + if cache_path is not None: + torch.save((matching_score, corres), mkdir_for(path_corres)) + + res_paths[img1['instance'], img2['instance']] = (path1, path2), path_corres + + del model + torch.cuda.empty_cache() + + return res_paths, cache_path + + +def symmetric_inference(model, img1, img2, device): + shape1 = torch.from_numpy(img1['true_shape']).to(device, non_blocking=True) + shape2 = torch.from_numpy(img2['true_shape']).to(device, non_blocking=True) + img1 = img1['img'].to(device, non_blocking=True) + img2 = img2['img'].to(device, non_blocking=True) + + # compute encoder only once + feat1, feat2, pos1, pos2 = model._encode_image_pairs(img1, img2, shape1, shape2) + + def decoder(feat1, feat2, pos1, pos2, shape1, shape2): + dec1, dec2 = model._decoder(feat1, pos1, feat2, pos2) + with torch.cuda.amp.autocast(enabled=False): + res1 = model._downstream_head(1, [tok.float() for tok in dec1], shape1) + res2 = model._downstream_head(2, [tok.float() for tok in dec2], shape2) + return res1, res2 + + # decoder 1-2 + res11, res21 = decoder(feat1, feat2, pos1, pos2, shape1, shape2) + # decoder 2-1 + res22, res12 = decoder(feat2, feat1, pos2, pos1, shape2, shape1) + + return (res11, res21, res22, res12) + + +def extract_correspondences(feats, qonfs, subsample=8, device=None, ptmap_key='pred_desc'): + feat11, feat21, feat22, feat12 = feats + qonf11, qonf21, qonf22, qonf12 = qonfs + assert feat11.shape[:2] == feat12.shape[:2] == qonf11.shape == qonf12.shape + assert feat21.shape[:2] == feat22.shape[:2] == qonf21.shape == qonf22.shape + + if '3d' in ptmap_key: + opt = dict(device='cpu', workers=32) + else: + opt = dict(device=device, dist='dot', block_size=2**13) + + # matching the two pairs + idx1 = [] + idx2 = [] + qonf1 = [] + qonf2 = [] + # TODO add non symmetric / pixel_tol options + for A, B, QA, QB in [(feat11, feat21, qonf11.cpu(), qonf21.cpu()), + (feat12, feat22, qonf12.cpu(), qonf22.cpu())]: + nn1to2 = fast_reciprocal_NNs(A, B, subsample_or_initxy1=subsample, ret_xy=False, **opt) + nn2to1 = fast_reciprocal_NNs(B, A, subsample_or_initxy1=subsample, ret_xy=False, **opt) + + idx1.append(np.r_[nn1to2[0], nn2to1[1]]) + idx2.append(np.r_[nn1to2[1], nn2to1[0]]) + qonf1.append(QA.ravel()[idx1[-1]]) + qonf2.append(QB.ravel()[idx2[-1]]) + + # merge corres from opposite pairs + H1, W1 = feat11.shape[:2] + H2, W2 = feat22.shape[:2] + cat = np.concatenate + + xy1, xy2, idx = merge_corres(cat(idx1), cat(idx2), (H1, W1), (H2, W2), ret_xy=True, ret_index=True) + corres = (xy1.copy(), xy2.copy(), np.sqrt(cat(qonf1)[idx] * cat(qonf2)[idx])) + + return todevice(corres, device) + + +@torch.no_grad() +def prepare_canonical_data(imgs, tmp_pairs, subsample, order_imgs=False, min_conf_thr=0, + cache_path=None, device='cuda', **kw): + canonical_views = {} + pairwise_scores = torch.zeros((len(imgs), len(imgs)), device=device) + canonical_paths = [] + preds_21 = {} + + for img in tqdm(imgs): + if cache_path: + cache = os.path.join(cache_path, 'canon_views', hash_md5(img) + f'_{subsample=}_{kw=}.pth') + canonical_paths.append(cache) + try: + (canon, canon2, cconf), focal = torch.load(cache, map_location=device) + except IOError: + # cache does not exist yet, we create it! + canon = focal = None + + # collect all pred1 + n_pairs = sum((img in pair) for pair in tmp_pairs) + + ptmaps11 = None + pixels = {} + n = 0 + for (img1, img2), ((path1, path2), path_corres) in tmp_pairs.items(): + score = None + if img == img1: + X, C, X2, C2 = torch.load(path1, map_location=device) + score, (xy1, xy2, confs) = load_corres(path_corres, device, min_conf_thr) + pixels[img2] = xy1, confs + if img not in preds_21: + preds_21[img] = {} + # Subsample preds_21 + preds_21[img][img2] = X2[::subsample, ::subsample].reshape(-1, 3), C2[::subsample, ::subsample].ravel() + + if img == img2: + X, C, X2, C2 = torch.load(path2, map_location=device) + score, (xy1, xy2, confs) = load_corres(path_corres, device, min_conf_thr) + pixels[img1] = xy2, confs + if img not in preds_21: + preds_21[img] = {} + preds_21[img][img1] = X2[::subsample, ::subsample].reshape(-1, 3), C2[::subsample, ::subsample].ravel() + + if score is not None: + i, j = imgs.index(img1), imgs.index(img2) + # score = score[0] + # score = np.log1p(score[2]) + score = score[2] + pairwise_scores[i, j] = score + pairwise_scores[j, i] = score + + if canon is not None: + continue + if ptmaps11 is None: + H, W = C.shape + ptmaps11 = torch.empty((n_pairs, H, W, 3), device=device) + confs11 = torch.empty((n_pairs, H, W), device=device) + + ptmaps11[n] = X + confs11[n] = C + n += 1 + + if canon is None: + canon, canon2, cconf = canonical_view(ptmaps11, confs11, subsample, **kw) + del ptmaps11 + del confs11 + + # compute focals + H, W = canon.shape[:2] + pp = torch.tensor([W / 2, H / 2], device=device) + if focal is None: + focal = estimate_focal_knowing_depth(canon[None], pp, focal_mode='weiszfeld', min_focal=0.5, max_focal=3.5) + if cache: + torch.save(to_cpu(((canon, canon2, cconf), focal)), mkdir_for(cache)) + + # extract depth offsets with correspondences + core_depth = canon[subsample // 2::subsample, subsample // 2::subsample, 2] + idxs, offsets = anchor_depth_offsets(canon2, pixels, subsample=subsample) + + canonical_views[img] = (pp, (H, W), focal.view(1), core_depth, pixels, idxs, offsets) + + return tmp_pairs, pairwise_scores, canonical_views, canonical_paths, preds_21 + + +def load_corres(path_corres, device, min_conf_thr): + score, (xy1, xy2, confs) = torch.load(path_corres, map_location=device) + valid = confs > min_conf_thr if min_conf_thr else slice(None) + # valid = (xy1 > 0).all(dim=1) & (xy2 > 0).all(dim=1) & (xy1 < 512).all(dim=1) & (xy2 < 512).all(dim=1) + # print(f'keeping {valid.sum()} / {len(valid)} correspondences') + return score, (xy1[valid], xy2[valid], confs[valid]) + + +PairOfSlices = namedtuple( + 'ImgPair', 'img1, slice1, pix1, anchor_idxs1, img2, slice2, pix2, anchor_idxs2, confs, confs_sum') + + +def condense_data(imgs, tmp_paths, canonical_views, preds_21, dtype=torch.float32): + # aggregate all data properly + set_imgs = set(imgs) + + principal_points = [] + shapes = [] + focals = [] + core_depth = [] + img_anchors = {} + tmp_pixels = {} + + for idx1, img1 in enumerate(imgs): + # load stuff + pp, shape, focal, anchors, pixels_confs, idxs, offsets = canonical_views[img1] + + principal_points.append(pp) + shapes.append(shape) + focals.append(focal) + core_depth.append(anchors) + + img_uv1 = [] + img_idxs = [] + img_offs = [] + cur_n = [0] + + for img2, (pixels, match_confs) in pixels_confs.items(): + if img2 not in set_imgs: + continue + assert len(pixels) == len(idxs[img2]) == len(offsets[img2]) + img_uv1.append(torch.cat((pixels, torch.ones_like(pixels[:, :1])), dim=-1)) + img_idxs.append(idxs[img2]) + img_offs.append(offsets[img2]) + cur_n.append(cur_n[-1] + len(pixels)) + # store the position of 3d points + tmp_pixels[img1, img2] = pixels.to(dtype), match_confs.to(dtype), slice(*cur_n[-2:]) + img_anchors[idx1] = (torch.cat(img_uv1), torch.cat(img_idxs), torch.cat(img_offs)) + + all_confs = [] + imgs_slices = [] + corres2d = {img: [] for img in range(len(imgs))} + + for img1, img2 in tmp_paths: + try: + pix1, confs1, slice1 = tmp_pixels[img1, img2] + pix2, confs2, slice2 = tmp_pixels[img2, img1] + except KeyError: + continue + img1 = imgs.index(img1) + img2 = imgs.index(img2) + confs = (confs1 * confs2).sqrt() + + # prepare for loss_3d + all_confs.append(confs) + anchor_idxs1 = canonical_views[imgs[img1]][5][imgs[img2]] + anchor_idxs2 = canonical_views[imgs[img2]][5][imgs[img1]] + imgs_slices.append(PairOfSlices(img1, slice1, pix1, anchor_idxs1, + img2, slice2, pix2, anchor_idxs2, + confs, float(confs.sum()))) + + # prepare for loss_2d + corres2d[img1].append((pix1, confs, img2, slice2)) + corres2d[img2].append((pix2, confs, img1, slice1)) + + all_confs = torch.cat(all_confs) + corres = (all_confs, float(all_confs.sum()), imgs_slices) + + def aggreg_matches(img1, list_matches): + pix1, confs, img2, slice2 = zip(*list_matches) + all_pix1 = torch.cat(pix1).to(dtype) + all_confs = torch.cat(confs).to(dtype) + return img1, all_pix1, all_confs, float(all_confs.sum()), [(j, sl2) for j, sl2 in zip(img2, slice2)] + corres2d = [aggreg_matches(img, m) for img, m in corres2d.items()] + + imsizes = torch.tensor([(W, H) for H, W in shapes], device=pp.device) # (W,H) + principal_points = torch.stack(principal_points) + focals = torch.cat(focals) + + # Subsample preds_21 + subsamp_preds_21 = {} + for imk, imv in preds_21.items(): + subsamp_preds_21[imk] = {} + for im2k, (pred, conf) in preds_21[imk].items(): + idxs = img_anchors[imgs.index(im2k)][1] + subsamp_preds_21[imk][im2k] = (pred[idxs], conf[idxs]) # anchors subsample + + return imsizes, principal_points, focals, core_depth, img_anchors, corres, corres2d, subsamp_preds_21 + + +def canonical_view(ptmaps11, confs11, subsample, mode='avg-angle'): + assert len(ptmaps11) == len(confs11) > 0, 'not a single view1 for img={i}' + + # canonical pointmap is just a weighted average + confs11 = confs11.unsqueeze(-1) - 0.999 + canon = (confs11 * ptmaps11).sum(0) / confs11.sum(0) + + canon_depth = ptmaps11[..., 2].unsqueeze(1) + S = slice(subsample // 2, None, subsample) + center_depth = canon_depth[:, :, S, S] + center_depth = torch.clip(center_depth, min=torch.finfo(center_depth.dtype).eps) + + stacked_depth = F.pixel_unshuffle(canon_depth, subsample) + stacked_confs = F.pixel_unshuffle(confs11[:, None, :, :, 0], subsample) + + if mode == 'avg-reldepth': + rel_depth = stacked_depth / center_depth + stacked_canon = (stacked_confs * rel_depth).sum(dim=0) / stacked_confs.sum(dim=0) + canon2 = F.pixel_shuffle(stacked_canon.unsqueeze(0), subsample).squeeze() + + elif mode == 'avg-angle': + xy = ptmaps11[..., 0:2].permute(0, 3, 1, 2) + stacked_xy = F.pixel_unshuffle(xy, subsample) + B, _, H, W = stacked_xy.shape + stacked_radius = (stacked_xy.view(B, 2, -1, H, W) - xy[:, :, None, S, S]).norm(dim=1) + stacked_radius.clip_(min=1e-8) + + stacked_angle = torch.arctan((stacked_depth - center_depth) / stacked_radius) + avg_angle = (stacked_confs * stacked_angle).sum(dim=0) / stacked_confs.sum(dim=0) + + # back to depth + stacked_depth = stacked_radius.mean(dim=0) * torch.tan(avg_angle) + + canon2 = F.pixel_shuffle((1 + stacked_depth / canon[S, S, 2]).unsqueeze(0), subsample).squeeze() + else: + raise ValueError(f'bad {mode=}') + + confs = (confs11.square().sum(dim=0) / confs11.sum(dim=0)).squeeze() + return canon, canon2, confs + + +def anchor_depth_offsets(canon_depth, pixels, subsample=8): + device = canon_depth.device + + # create a 2D grid of anchor 3D points + H1, W1 = canon_depth.shape + yx = np.mgrid[subsample // 2:H1:subsample, subsample // 2:W1:subsample] + H2, W2 = yx.shape[1:] + cy, cx = yx.reshape(2, -1) + core_depth = canon_depth[cy, cx] + assert (core_depth > 0).all() + + # slave 3d points (attached to core 3d points) + core_idxs = {} # core_idxs[img2] = {corr_idx:core_idx} + core_offs = {} # core_offs[img2] = {corr_idx:3d_offset} + + for img2, (xy1, _confs) in pixels.items(): + px, py = xy1.long().T + + # find nearest anchor == block quantization + core_idx = (py // subsample) * W2 + (px // subsample) + core_idxs[img2] = core_idx.to(device) + + # compute relative depth offsets w.r.t. anchors + ref_z = core_depth[core_idx] + pts_z = canon_depth[py, px] + offset = pts_z / ref_z + core_offs[img2] = offset.detach().to(device) + + return core_idxs, core_offs + + +def spectral_clustering(graph, k=None, normalized_cuts=False): + graph.fill_diagonal_(0) + + # graph laplacian + degrees = graph.sum(dim=-1) + laplacian = torch.diag(degrees) - graph + if normalized_cuts: + i_inv = torch.diag(degrees.sqrt().reciprocal()) + laplacian = i_inv @ laplacian @ i_inv + + # compute eigenvectors! + eigval, eigvec = torch.linalg.eigh(laplacian) + return eigval[:k], eigvec[:, :k] + + +def sim_func(p1, p2, gamma): + diff = (p1 - p2).norm(dim=-1) + avg_depth = (p1[:, :, 2] + p2[:, :, 2]) + rel_distance = diff / avg_depth + sim = torch.exp(-gamma * rel_distance.square()) + return sim + + +def backproj(K, depthmap, subsample): + H, W = depthmap.shape + uv = np.mgrid[subsample // 2:subsample * W:subsample, subsample // 2:subsample * H:subsample].T.reshape(H, W, 2) + xyz = depthmap.unsqueeze(-1) * geotrf(inv(K), todevice(uv, K.device), ncol=3) + return xyz + + +def spectral_projection_depth(K, depthmap, subsample, k=64, cache_path='', + normalized_cuts=True, gamma=7, min_norm=5): + try: + if cache_path: + cache_path = cache_path + f'_{k=}_norm={normalized_cuts}_{gamma=}.pth' + lora_proj = torch.load(cache_path, map_location=K.device) + + except IOError: + # reconstruct 3d points in camera coordinates + xyz = backproj(K, depthmap, subsample) + + # compute all distances + xyz = xyz.reshape(-1, 3) + graph = sim_func(xyz[:, None], xyz[None, :], gamma=gamma) + _, lora_proj = spectral_clustering(graph, k, normalized_cuts=normalized_cuts) + + if cache_path: + torch.save(lora_proj.cpu(), mkdir_for(cache_path)) + + lora_proj, coeffs = lora_encode_normed(lora_proj, depthmap.ravel(), min_norm=min_norm) + + # depthmap ~= lora_proj @ coeffs + return coeffs, lora_proj + + +def lora_encode_normed(lora_proj, x, min_norm, global_norm=False): + # encode the pointmap + coeffs = torch.linalg.pinv(lora_proj) @ x + + # rectify the norm of basis vector to be ~ equal + if coeffs.ndim == 1: + coeffs = coeffs[:, None] + if global_norm: + lora_proj *= coeffs[1:].norm() * min_norm / coeffs.shape[1] + elif min_norm: + lora_proj *= coeffs.norm(dim=1).clip(min=min_norm) + # can have rounding errors here! + coeffs = (torch.linalg.pinv(lora_proj.double()) @ x.double()).float() + + return lora_proj.detach(), coeffs.detach() + + +@torch.no_grad() +def spectral_projection_of_depthmaps(imgs, intrinsics, depthmaps, subsample, cache_path=None, **kw): + # recover 3d points + core_depth = [] + lora_proj = [] + + for i, img in enumerate(tqdm(imgs)): + cache = os.path.join(cache_path, 'lora_depth', hash_md5(img)) if cache_path else None + depth, proj = spectral_projection_depth(intrinsics[i], depthmaps[i], subsample, + cache_path=cache, **kw) + core_depth.append(depth) + lora_proj.append(proj) + + return core_depth, lora_proj + + +def reproj2d(Trf, pts3d): + res = (pts3d @ Trf[:3, :3].transpose(-1, -2)) + Trf[:3, 3] + clipped_z = res[:, 2:3].clip(min=1e-3) # make sure we don't have nans! + uv = res[:, 0:2] / clipped_z + return uv.clip(min=-1000, max=2000) + + +def bfs(tree, start_node): + order, predecessors = sp.csgraph.breadth_first_order(tree, start_node, directed=False) + ranks = np.arange(len(order)) + ranks[order] = ranks.copy() + return ranks, predecessors + + +def compute_min_spanning_tree(pws): + sparse_graph = sp.dok_array(pws.shape) + for i, j in pws.nonzero().cpu().tolist(): + sparse_graph[i, j] = -float(pws[i, j]) + msp = sp.csgraph.minimum_spanning_tree(sparse_graph) + + # now reorder the oriented edges, starting from the central point + ranks1, _ = bfs(msp, 0) + ranks2, _ = bfs(msp, ranks1.argmax()) + ranks1, _ = bfs(msp, ranks2.argmax()) + # this is the point farther from any leaf + root = np.minimum(ranks1, ranks2).argmax() + + # find the ordered list of edges that describe the tree + order, predecessors = sp.csgraph.breadth_first_order(msp, root, directed=False) + order = order[1:] # root not do not have a predecessor + edges = [(predecessors[i], i) for i in order] + + return root, edges + + +def show_reconstruction(shapes_or_imgs, K, cam2w, pts3d, gt_cam2w=None, gt_K=None, cam_size=None, masks=None, **kw): + viz = SceneViz() + + cc = cam2w[:, :3, 3] + cs = cam_size or float(torch.cdist(cc, cc).fill_diagonal_(np.inf).min(dim=0).values.median()) + colors = 64 + np.random.randint(255 - 64, size=(len(cam2w), 3)) + + if isinstance(shapes_or_imgs, np.ndarray) and shapes_or_imgs.ndim == 2: + cam_kws = dict(imsizes=shapes_or_imgs[:, ::-1], cam_size=cs) + else: + imgs = shapes_or_imgs + cam_kws = dict(images=imgs, cam_size=cs) + if K is not None: + viz.add_cameras(to_numpy(cam2w), to_numpy(K), colors=colors, **cam_kws) + + if gt_cam2w is not None: + if gt_K is None: + gt_K = K + viz.add_cameras(to_numpy(gt_cam2w), to_numpy(gt_K), colors=colors, marker='o', **cam_kws) + + if pts3d is not None: + for i, p in enumerate(pts3d): + if not len(p): + continue + if masks is None: + viz.add_pointcloud(to_numpy(p), color=tuple(colors[i].tolist())) + else: + viz.add_pointcloud(to_numpy(p), mask=masks[i], color=imgs[i]) + viz.show(**kw) diff --git a/submodules/mast3r/mast3r/cloud_opt/triangulation.py b/submodules/mast3r/mast3r/cloud_opt/triangulation.py new file mode 100644 index 0000000000000000000000000000000000000000..2af88df37bfd360161b4e96b93b0fd28a0ecf183 --- /dev/null +++ b/submodules/mast3r/mast3r/cloud_opt/triangulation.py @@ -0,0 +1,80 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Matches Triangulation Utils +# -------------------------------------------------------- + +import numpy as np +import torch + +# Batched Matches Triangulation +def batched_triangulate(pts2d, # [B, Ncams, Npts, 2] + proj_mats): # [B, Ncams, 3, 4] I@E projection matrix + B, Ncams, Npts, two = pts2d.shape + assert two==2 + assert proj_mats.shape == (B, Ncams, 3, 4) + # P - xP + x = proj_mats[...,0,:][...,None,:] - torch.einsum('bij,bik->bijk', pts2d[...,0], proj_mats[...,2,:]) # [B, Ncams, Npts, 4] + y = proj_mats[...,1,:][...,None,:] - torch.einsum('bij,bik->bijk', pts2d[...,1], proj_mats[...,2,:]) # [B, Ncams, Npts, 4] + eq = torch.cat([x, y], dim=1).transpose(1, 2) # [B, Npts, 2xNcams, 4] + return torch.linalg.lstsq(eq[...,:3], -eq[...,3]).solution + +def matches_to_depths(intrinsics, # input camera intrinsics [B, Ncams, 3, 3] + extrinsics, # input camera extrinsics [B, Ncams, 3, 4] + matches, # input correspondences [B, Ncams, Npts, 2] + batchsize=16, # bs for batched processing + min_num_valids_ratio=.3 # at least this ratio of image pairs need to predict a match for a given pixel of img1 + ): + B, Nv, H, W, five = matches.shape + min_num_valids = np.floor(Nv*min_num_valids_ratio) + out_aggregated_points, out_depths, out_confs = [], [], [] + for b in range(B//batchsize+1): # batched processing + start, stop = b*batchsize,min(B,(b+1)*batchsize) + sub_batch=slice(start,stop) + sub_batchsize = stop-start + if sub_batchsize==0:continue + points1, points2, confs = matches[sub_batch, ..., :2], matches[sub_batch, ..., 2:4], matches[sub_batch, ..., -1] + allpoints = torch.cat([points1.view([sub_batchsize*Nv,1,H*W,2]), points2.view([sub_batchsize*Nv,1,H*W,2])],dim=1) # [BxNv, 2, HxW, 2] + + allcam_Ps = intrinsics[sub_batch] @ extrinsics[sub_batch,:,:3,:] + cam_Ps1, cam_Ps2 = allcam_Ps[:,[0]].repeat([1,Nv,1,1]), allcam_Ps[:,1:] # [B, Nv, 3, 4] + formatted_camPs = torch.cat([cam_Ps1.reshape([sub_batchsize*Nv,1,3,4]), cam_Ps2.reshape([sub_batchsize*Nv,1,3,4])],dim=1) # [BxNv, 2, 3, 4] + + # Triangulate matches to 3D + points_3d_world = batched_triangulate(allpoints, formatted_camPs) # [BxNv, HxW, three] + + # Aggregate pairwise predictions + points_3d_world = points_3d_world.view([sub_batchsize,Nv,H,W,3]) + valids = points_3d_world.isfinite() + valids_sum = valids.sum(dim=-1) + validsuni=valids_sum.unique() + assert torch.all(torch.logical_or(validsuni == 0 , validsuni == 3)), "Error, can only be nan for none or all XYZ values, not a subset" + confs[valids_sum==0] = 0. + points_3d_world = points_3d_world*confs[...,None] + + # Take care of NaNs + normalization = confs.sum(dim=1)[:,None].repeat(1,Nv,1,1) + normalization[normalization <= 1e-5] = 1. + points_3d_world[valids] /= normalization[valids_sum==3][:,None].repeat(1,3).view(-1) + points_3d_world[~valids] = 0. + aggregated_points = points_3d_world.sum(dim=1) # weighted average (by confidence value) ignoring nans + + # Reset invalid values to nans, with a min visibility threshold + aggregated_points[valids_sum.sum(dim=1)/3 <= min_num_valids] = torch.nan + + # From 3D to depths + refcamE = extrinsics[sub_batch, 0] + points_3d_camera = (refcamE[:,:3, :3] @ aggregated_points.view(sub_batchsize,-1,3).transpose(-2,-1) + refcamE[:,:3,[3]]).transpose(-2,-1) # [B,HxW,3] + depths = points_3d_camera.view(sub_batchsize,H,W,3)[..., 2] # [B,H,W] + + # Cat results + out_aggregated_points.append(aggregated_points.cpu()) + out_depths.append(depths.cpu()) + out_confs.append(confs.sum(dim=1).cpu()) + + out_aggregated_points = torch.cat(out_aggregated_points,dim=0) + out_depths = torch.cat(out_depths,dim=0) + out_confs = torch.cat(out_confs,dim=0) + + return out_aggregated_points, out_depths, out_confs diff --git a/submodules/mast3r/mast3r/cloud_opt/tsdf_optimizer.py b/submodules/mast3r/mast3r/cloud_opt/tsdf_optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..69f138c0301e4ad3cd4804d265f241b923e1b2b8 --- /dev/null +++ b/submodules/mast3r/mast3r/cloud_opt/tsdf_optimizer.py @@ -0,0 +1,273 @@ +import torch +from torch import nn +import numpy as np +from tqdm import tqdm +from matplotlib import pyplot as pl + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.utils.geometry import depthmap_to_pts3d, geotrf, inv +from dust3r.cloud_opt.base_opt import clean_pointcloud + + +class TSDFPostProcess: + """ Optimizes a signed distance-function to improve depthmaps. + """ + + def __init__(self, optimizer, subsample=8, TSDF_thresh=0., TSDF_batchsize=int(1e7)): + self.TSDF_thresh = TSDF_thresh # None -> no TSDF + self.TSDF_batchsize = TSDF_batchsize + self.optimizer = optimizer + + pts3d, depthmaps, confs = optimizer.get_dense_pts3d(clean_depth=False, subsample=subsample) + pts3d, depthmaps = self._TSDF_postprocess_or_not(pts3d, depthmaps, confs) + self.pts3d = pts3d + self.depthmaps = depthmaps + self.confs = confs + + def _get_depthmaps(self, TSDF_filtering_thresh=None): + if TSDF_filtering_thresh: + self._refine_depths_with_TSDF(self.optimizer, TSDF_filtering_thresh) # compute refined depths if needed + dms = self.TSDF_im_depthmaps if TSDF_filtering_thresh else self.im_depthmaps + return [d.exp() for d in dms] + + @torch.no_grad() + def _refine_depths_with_TSDF(self, TSDF_filtering_thresh, niter=1, nsamples=1000): + """ + Leverage TSDF to post-process estimated depths + for each pixel, find zero level of TSDF along ray (or closest to 0) + """ + print("Post-Processing Depths with TSDF fusion.") + self.TSDF_im_depthmaps = [] + alldepths, allposes, allfocals, allpps, allimshapes = self._get_depthmaps(), self.optimizer.get_im_poses( + ), self.optimizer.get_focals(), self.optimizer.get_principal_points(), self.imshapes + for vi in tqdm(range(self.optimizer.n_imgs)): + dm, pose, focal, pp, imshape = alldepths[vi], allposes[vi], allfocals[vi], allpps[vi], allimshapes[vi] + minvals = torch.full(dm.shape, 1e20) + + for it in range(niter): + H, W = dm.shape + curthresh = (niter - it) * TSDF_filtering_thresh + dm_offsets = (torch.randn(H, W, nsamples).to(dm) - 1.) * \ + curthresh # decreasing search std along with iterations + newdm = dm[..., None] + dm_offsets # [H,W,Nsamp] + curproj = self._backproj_pts3d(in_depths=[newdm], in_im_poses=pose[None], in_focals=focal[None], in_pps=pp[None], in_imshapes=[ + imshape])[0] # [H,W,Nsamp,3] + # Batched TSDF eval + curproj = curproj.view(-1, 3) + tsdf_vals = [] + valids = [] + for batch in range(0, len(curproj), self.TSDF_batchsize): + values, valid = self._TSDF_query( + curproj[batch:min(batch + self.TSDF_batchsize, len(curproj))], curthresh) + tsdf_vals.append(values) + valids.append(valid) + tsdf_vals = torch.cat(tsdf_vals, dim=0) + valids = torch.cat(valids, dim=0) + + tsdf_vals = tsdf_vals.view([H, W, nsamples]) + valids = valids.view([H, W, nsamples]) + + # keep depth value that got us the closest to 0 + tsdf_vals[~valids] = torch.inf # ignore invalid values + tsdf_vals = tsdf_vals.abs() + mins = torch.argmin(tsdf_vals, dim=-1, keepdim=True) + # when all samples live on a very flat zone, do nothing + allbad = (tsdf_vals == curthresh).sum(dim=-1) == nsamples + dm[~allbad] = torch.gather(newdm, -1, mins)[..., 0][~allbad] + + # Save refined depth map + self.TSDF_im_depthmaps.append(dm.log()) + + def _TSDF_query(self, qpoints, TSDF_filtering_thresh, weighted=True): + """ + TSDF query call: returns the weighted TSDF value for each query point [N, 3] + """ + N, three = qpoints.shape + assert three == 3 + qpoints = qpoints[None].repeat(self.optimizer.n_imgs, 1, 1) # [B,N,3] + # get projection coordinates and depths onto images + coords_and_depth = self._proj_pts3d(pts3d=qpoints, cam2worlds=self.optimizer.get_im_poses( + ), focals=self.optimizer.get_focals(), pps=self.optimizer.get_principal_points()) + image_coords = coords_and_depth[..., :2].round().to(int) # for now, there's no interpolation... + proj_depths = coords_and_depth[..., -1] + # recover depth values after scene optim + pred_depths, pred_confs, valids = self._get_pixel_depths(image_coords) + # Gather TSDF scores + all_SDF_scores = pred_depths - proj_depths # SDF + unseen = all_SDF_scores < -TSDF_filtering_thresh # handle visibility + # all_TSDF_scores = all_SDF_scores.clip(-TSDF_filtering_thresh,TSDF_filtering_thresh) # SDF -> TSDF + all_TSDF_scores = all_SDF_scores.clip(-TSDF_filtering_thresh, 1e20) # SDF -> TSDF + # Gather TSDF confidences and ignore points that are unseen, either OOB during reproj or too far behind seen depth + all_TSDF_weights = (~unseen).float() * valids.float() + if weighted: + all_TSDF_weights = pred_confs.exp() * all_TSDF_weights + # Aggregate all votes, ignoring zeros + TSDF_weights = all_TSDF_weights.sum(dim=0) + valids = TSDF_weights != 0. + TSDF_wsum = (all_TSDF_weights * all_TSDF_scores).sum(dim=0) + TSDF_wsum[valids] /= TSDF_weights[valids] + return TSDF_wsum, valids + + def _get_pixel_depths(self, image_coords, TSDF_filtering_thresh=None, with_normals_conf=False): + """ Recover depth value for each input pixel coordinate, along with OOB validity mask + """ + B, N, two = image_coords.shape + assert B == self.optimizer.n_imgs and two == 2 + depths = torch.zeros([B, N], device=image_coords.device) + valids = torch.zeros([B, N], dtype=bool, device=image_coords.device) + confs = torch.zeros([B, N], device=image_coords.device) + curconfs = self._get_confs_with_normals() if with_normals_conf else self.im_conf + for ni, (imc, depth, conf) in enumerate(zip(image_coords, self._get_depthmaps(TSDF_filtering_thresh), curconfs)): + H, W = depth.shape + valids[ni] = torch.logical_and(0 <= imc[:, 1], imc[:, 1] < + H) & torch.logical_and(0 <= imc[:, 0], imc[:, 0] < W) + imc[~valids[ni]] = 0 + depths[ni] = depth[imc[:, 1], imc[:, 0]] + confs[ni] = conf.cuda()[imc[:, 1], imc[:, 0]] + return depths, confs, valids + + def _get_confs_with_normals(self): + outconfs = [] + # Confidence basedf on depth gradient + + class Sobel(nn.Module): + def __init__(self): + super().__init__() + self.filter = nn.Conv2d(in_channels=1, out_channels=2, kernel_size=3, stride=1, padding=1, bias=False) + Gx = torch.tensor([[2.0, 0.0, -2.0], [4.0, 0.0, -4.0], [2.0, 0.0, -2.0]]) + Gy = torch.tensor([[2.0, 4.0, 2.0], [0.0, 0.0, 0.0], [-2.0, -4.0, -2.0]]) + G = torch.cat([Gx.unsqueeze(0), Gy.unsqueeze(0)], 0) + G = G.unsqueeze(1) + self.filter.weight = nn.Parameter(G, requires_grad=False) + + def forward(self, img): + x = self.filter(img) + x = torch.mul(x, x) + x = torch.sum(x, dim=1, keepdim=True) + x = torch.sqrt(x) + return x + + grad_op = Sobel().to(self.im_depthmaps[0].device) + for conf, depth in zip(self.im_conf, self.im_depthmaps): + grad_confs = (1. - grad_op(depth[None, None])[0, 0]).clip(0) + if not 'dbg show': + pl.imshow(grad_confs.cpu()) + pl.show() + outconfs.append(conf * grad_confs.to(conf)) + return outconfs + + def _proj_pts3d(self, pts3d, cam2worlds, focals, pps): + """ + Projection operation: from 3D points to 2D coordinates + depths + """ + B = pts3d.shape[0] + assert pts3d.shape[0] == cam2worlds.shape[0] + # prepare Extrinsincs + R, t = cam2worlds[:, :3, :3], cam2worlds[:, :3, -1] + Rinv = R.transpose(-2, -1) + tinv = -Rinv @ t[..., None] + + # prepare intrinsics + intrinsics = torch.eye(3).to(cam2worlds)[None].repeat(focals.shape[0], 1, 1) + if len(focals.shape) == 1: + focals = torch.stack([focals, focals], dim=-1) + intrinsics[:, 0, 0] = focals[:, 0] + intrinsics[:, 1, 1] = focals[:, 1] + intrinsics[:, :2, -1] = pps + # Project + projpts = intrinsics @ (Rinv @ pts3d.transpose(-2, -1) + tinv) # I(RX+t) : [B,3,N] + projpts = projpts.transpose(-2, -1) # [B,N,3] + projpts[..., :2] /= projpts[..., [-1]] # [B,N,3] (X/Z , Y/Z, Z) + return projpts + + def _backproj_pts3d(self, in_depths=None, in_im_poses=None, + in_focals=None, in_pps=None, in_imshapes=None): + """ + Backprojection operation: from image depths to 3D points + """ + # Get depths and projection params if not provided + focals = self.optimizer.get_focals() if in_focals is None else in_focals + im_poses = self.optimizer.get_im_poses() if in_im_poses is None else in_im_poses + depth = self._get_depthmaps() if in_depths is None else in_depths + pp = self.optimizer.get_principal_points() if in_pps is None else in_pps + imshapes = self.imshapes if in_imshapes is None else in_imshapes + def focal_ex(i): return focals[i][..., None, None].expand(1, *focals[i].shape, *imshapes[i]) + dm_to_3d = [depthmap_to_pts3d(depth[i][None], focal_ex(i), pp=pp[[i]]) for i in range(im_poses.shape[0])] + + def autoprocess(x): + x = x[0] + return x.transpose(-2, -1) if len(x.shape) == 4 else x + return [geotrf(pose, autoprocess(pt)) for pose, pt in zip(im_poses, dm_to_3d)] + + def _pts3d_to_depth(self, pts3d, cam2worlds, focals, pps): + """ + Projection operation: from 3D points to 2D coordinates + depths + """ + B = pts3d.shape[0] + assert pts3d.shape[0] == cam2worlds.shape[0] + # prepare Extrinsincs + R, t = cam2worlds[:, :3, :3], cam2worlds[:, :3, -1] + Rinv = R.transpose(-2, -1) + tinv = -Rinv @ t[..., None] + + # prepare intrinsics + intrinsics = torch.eye(3).to(cam2worlds)[None].repeat(self.optimizer.n_imgs, 1, 1) + if len(focals.shape) == 1: + focals = torch.stack([focals, focals], dim=-1) + intrinsics[:, 0, 0] = focals[:, 0] + intrinsics[:, 1, 1] = focals[:, 1] + intrinsics[:, :2, -1] = pps + # Project + projpts = intrinsics @ (Rinv @ pts3d.transpose(-2, -1) + tinv) # I(RX+t) : [B,3,N] + projpts = projpts.transpose(-2, -1) # [B,N,3] + projpts[..., :2] /= projpts[..., [-1]] # [B,N,3] (X/Z , Y/Z, Z) + return projpts + + def _depth_to_pts3d(self, in_depths=None, in_im_poses=None, in_focals=None, in_pps=None, in_imshapes=None): + """ + Backprojection operation: from image depths to 3D points + """ + # Get depths and projection params if not provided + focals = self.optimizer.get_focals() if in_focals is None else in_focals + im_poses = self.optimizer.get_im_poses() if in_im_poses is None else in_im_poses + depth = self._get_depthmaps() if in_depths is None else in_depths + pp = self.optimizer.get_principal_points() if in_pps is None else in_pps + imshapes = self.imshapes if in_imshapes is None else in_imshapes + + def focal_ex(i): return focals[i][..., None, None].expand(1, *focals[i].shape, *imshapes[i]) + + dm_to_3d = [depthmap_to_pts3d(depth[i][None], focal_ex(i), pp=pp[i:i + 1]) for i in range(im_poses.shape[0])] + + def autoprocess(x): + x = x[0] + H, W, three = x.shape[:3] + return x.transpose(-2, -1) if len(x.shape) == 4 else x + return [geotrf(pp, autoprocess(pt)) for pp, pt in zip(im_poses, dm_to_3d)] + + def _get_pts3d(self, TSDF_filtering_thresh=None, **kw): + """ + return 3D points (possibly filtering depths with TSDF) + """ + return self._backproj_pts3d(in_depths=self._get_depthmaps(TSDF_filtering_thresh=TSDF_filtering_thresh), **kw) + + def _TSDF_postprocess_or_not(self, pts3d, depthmaps, confs, niter=1): + # Setup inner variables + self.imshapes = [im.shape[:2] for im in self.optimizer.imgs] + self.im_depthmaps = [dd.log().view(imshape) for dd, imshape in zip(depthmaps, self.imshapes)] + self.im_conf = confs + + if self.TSDF_thresh > 0.: + # Create or update self.TSDF_im_depthmaps that contain logdepths filtered with TSDF + self._refine_depths_with_TSDF(self.TSDF_thresh, niter=niter) + depthmaps = [dd.exp() for dd in self.TSDF_im_depthmaps] + # Turn them into 3D points + pts3d = self._backproj_pts3d(in_depths=depthmaps) + depthmaps = [dd.flatten() for dd in depthmaps] + pts3d = [pp.view(-1, 3) for pp in pts3d] + return pts3d, depthmaps + + def get_dense_pts3d(self, clean_depth=True): + if clean_depth: + confs = clean_pointcloud(self.confs, self.optimizer.intrinsics, inv(self.optimizer.cam2w), + self.depthmaps, self.pts3d) + return self.pts3d, self.depthmaps, confs diff --git a/submodules/mast3r/mast3r/cloud_opt/utils/__init__.py b/submodules/mast3r/mast3r/cloud_opt/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7dd877d649ce4dbd749dd7195a8b34c0f91d4f0 --- /dev/null +++ b/submodules/mast3r/mast3r/cloud_opt/utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). \ No newline at end of file diff --git a/submodules/mast3r/mast3r/cloud_opt/utils/losses.py b/submodules/mast3r/mast3r/cloud_opt/utils/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..e1dd36afd6862592b8d00c499988136a972bd6e6 --- /dev/null +++ b/submodules/mast3r/mast3r/cloud_opt/utils/losses.py @@ -0,0 +1,32 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# losses for sparse ga +# -------------------------------------------------------- +import torch +import numpy as np + + +def l05_loss(x, y): + return torch.linalg.norm(x - y, dim=-1).sqrt() + + +def l1_loss(x, y): + return torch.linalg.norm(x - y, dim=-1) + + +def gamma_loss(gamma, mul=1, offset=None, clip=np.inf): + if offset is None: + if gamma == 1: + return l1_loss + # d(x**p)/dx = 1 ==> p * x**(p-1) == 1 ==> x = (1/p)**(1/(p-1)) + offset = (1 / gamma)**(1 / (gamma - 1)) + + def loss_func(x, y): + return (mul * l1_loss(x, y).clip(max=clip) + offset) ** gamma - offset ** gamma + return loss_func + + +def meta_gamma_loss(): + return lambda alpha: gamma_loss(alpha) diff --git a/submodules/mast3r/mast3r/cloud_opt/utils/schedules.py b/submodules/mast3r/mast3r/cloud_opt/utils/schedules.py new file mode 100644 index 0000000000000000000000000000000000000000..d96253b4348d2f089c10142c5991e5afb8a9b683 --- /dev/null +++ b/submodules/mast3r/mast3r/cloud_opt/utils/schedules.py @@ -0,0 +1,17 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# lr schedules for sparse ga +# -------------------------------------------------------- +import numpy as np + + +def linear_schedule(alpha, lr_base, lr_end=0): + lr = (1 - alpha) * lr_base + alpha * lr_end + return lr + + +def cosine_schedule(alpha, lr_base, lr_end=0): + lr = lr_end + (lr_base - lr_end) * (1 + np.cos(alpha * np.pi)) / 2 + return lr diff --git a/submodules/mast3r/mast3r/colmap/__init__.py b/submodules/mast3r/mast3r/colmap/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7dd877d649ce4dbd749dd7195a8b34c0f91d4f0 --- /dev/null +++ b/submodules/mast3r/mast3r/colmap/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). \ No newline at end of file diff --git a/submodules/mast3r/mast3r/colmap/database.py b/submodules/mast3r/mast3r/colmap/database.py new file mode 100644 index 0000000000000000000000000000000000000000..5de83a35664d4038a99713de7f397e83940e5421 --- /dev/null +++ b/submodules/mast3r/mast3r/colmap/database.py @@ -0,0 +1,383 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# MASt3R to colmap export functions +# -------------------------------------------------------- +import os +import torch +import copy +import numpy as np +import torchvision +import numpy as np +from tqdm import tqdm +from scipy.cluster.hierarchy import DisjointSet +from scipy.spatial.transform import Rotation as R + +from mast3r.utils.misc import hash_md5 + +from mast3r.fast_nn import extract_correspondences_nonsym, bruteforce_reciprocal_nns + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.utils.geometry import find_reciprocal_matches, xy_grid, geotrf # noqa + + +def convert_im_matches_pairs(img0, img1, image_to_colmap, im_keypoints, matches_im0, matches_im1, viz): + if viz: + from matplotlib import pyplot as pl + + image_mean = torch.as_tensor( + [0.5, 0.5, 0.5], device='cpu').reshape(1, 3, 1, 1) + image_std = torch.as_tensor( + [0.5, 0.5, 0.5], device='cpu').reshape(1, 3, 1, 1) + rgb0 = img0['img'] * image_std + image_mean + rgb0 = torchvision.transforms.functional.to_pil_image(rgb0[0]) + rgb0 = np.array(rgb0) + + rgb1 = img1['img'] * image_std + image_mean + rgb1 = torchvision.transforms.functional.to_pil_image(rgb1[0]) + rgb1 = np.array(rgb1) + + imgs = [rgb0, rgb1] + # visualize a few matches + n_viz = 100 + num_matches = matches_im0.shape[0] + match_idx_to_viz = np.round(np.linspace( + 0, num_matches - 1, n_viz)).astype(int) + viz_matches_im0, viz_matches_im1 = matches_im0[match_idx_to_viz], matches_im1[match_idx_to_viz] + + H0, W0, H1, W1 = *imgs[0].shape[:2], *imgs[1].shape[:2] + rgb0 = np.pad(imgs[0], ((0, max(H1 - H0, 0)), + (0, 0), (0, 0)), 'constant', constant_values=0) + rgb1 = np.pad(imgs[1], ((0, max(H0 - H1, 0)), + (0, 0), (0, 0)), 'constant', constant_values=0) + img = np.concatenate((rgb0, rgb1), axis=1) + pl.figure() + pl.imshow(img) + cmap = pl.get_cmap('jet') + for ii in range(n_viz): + (x0, y0), (x1, + y1) = viz_matches_im0[ii].T, viz_matches_im1[ii].T + pl.plot([x0, x1 + W0], [y0, y1], '-+', color=cmap(ii / + (n_viz - 1)), scalex=False, scaley=False) + pl.show(block=True) + + matches = [matches_im0.astype(np.float64), matches_im1.astype(np.float64)] + imgs = [img0, img1] + imidx0 = img0['idx'] + imidx1 = img1['idx'] + ravel_matches = [] + for j in range(2): + H, W = imgs[j]['true_shape'][0] + with np.errstate(invalid='ignore'): + qx, qy = matches[j].round().astype(np.int32).T + ravel_matches_j = qx.clip(min=0, max=W - 1, out=qx) + W * qy.clip(min=0, max=H - 1, out=qy) + ravel_matches.append(ravel_matches_j) + imidxj = imgs[j]['idx'] + for m in ravel_matches_j: + if m not in im_keypoints[imidxj]: + im_keypoints[imidxj][m] = 0 + im_keypoints[imidxj][m] += 1 + imid0 = copy.deepcopy(image_to_colmap[imidx0]['colmap_imid']) + imid1 = copy.deepcopy(image_to_colmap[imidx1]['colmap_imid']) + if imid0 > imid1: + colmap_matches = np.stack([ravel_matches[1], ravel_matches[0]], axis=-1) + imid0, imid1 = imid1, imid0 + imidx0, imidx1 = imidx1, imidx0 + else: + colmap_matches = np.stack([ravel_matches[0], ravel_matches[1]], axis=-1) + colmap_matches = np.unique(colmap_matches, axis=0) + return imidx0, imidx1, colmap_matches + + +def get_im_matches(pred1, pred2, pairs, image_to_colmap, im_keypoints, conf_thr, + is_sparse=True, subsample=8, pixel_tol=0, viz=False, device='cuda'): + im_matches = {} + for i in range(len(pred1['pts3d'])): + imidx0 = pairs[i][0]['idx'] + imidx1 = pairs[i][1]['idx'] + if 'desc' in pred1: # mast3r + descs = [pred1['desc'][i], pred2['desc'][i]] + confidences = [pred1['desc_conf'][i], pred2['desc_conf'][i]] + desc_dim = descs[0].shape[-1] + + if is_sparse: + corres = extract_correspondences_nonsym(descs[0], descs[1], confidences[0], confidences[1], + device=device, subsample=subsample, pixel_tol=pixel_tol) + conf = corres[2] + mask = conf >= conf_thr + matches_im0 = corres[0][mask].cpu().numpy() + matches_im1 = corres[1][mask].cpu().numpy() + else: + confidence_masks = [confidences[0] >= + conf_thr, confidences[1] >= conf_thr] + pts2d_list, desc_list = [], [] + for j in range(2): + conf_j = confidence_masks[j].cpu().numpy().flatten() + true_shape_j = pairs[i][j]['true_shape'][0] + pts2d_j = xy_grid( + true_shape_j[1], true_shape_j[0]).reshape(-1, 2)[conf_j] + desc_j = descs[j].detach().cpu( + ).numpy().reshape(-1, desc_dim)[conf_j] + pts2d_list.append(pts2d_j) + desc_list.append(desc_j) + if len(desc_list[0]) == 0 or len(desc_list[1]) == 0: + continue + + nn0, nn1 = bruteforce_reciprocal_nns(desc_list[0], desc_list[1], + device=device, dist='dot', block_size=2**13) + reciprocal_in_P0 = (nn1[nn0] == np.arange(len(nn0))) + + matches_im1 = pts2d_list[1][nn0][reciprocal_in_P0] + matches_im0 = pts2d_list[0][reciprocal_in_P0] + else: + pts3d = [pred1['pts3d'][i], pred2['pts3d_in_other_view'][i]] + confidences = [pred1['conf'][i], pred2['conf'][i]] + + if is_sparse: + corres = extract_correspondences_nonsym(pts3d[0], pts3d[1], confidences[0], confidences[1], + device=device, subsample=subsample, pixel_tol=pixel_tol, + ptmap_key='3d') + conf = corres[2] + mask = conf >= conf_thr + matches_im0 = corres[0][mask].cpu().numpy() + matches_im1 = corres[1][mask].cpu().numpy() + else: + confidence_masks = [confidences[0] >= + conf_thr, confidences[1] >= conf_thr] + # find 2D-2D matches between the two images + pts2d_list, pts3d_list = [], [] + for j in range(2): + conf_j = confidence_masks[j].cpu().numpy().flatten() + true_shape_j = pairs[i][j]['true_shape'][0] + pts2d_j = xy_grid(true_shape_j[1], true_shape_j[0]).reshape(-1, 2)[conf_j] + pts3d_j = pts3d[j].detach().cpu().numpy().reshape(-1, 3)[conf_j] + pts2d_list.append(pts2d_j) + pts3d_list.append(pts3d_j) + + PQ, PM = pts3d_list[0], pts3d_list[1] + if len(PQ) == 0 or len(PM) == 0: + continue + reciprocal_in_PM, nnM_in_PQ, num_matches = find_reciprocal_matches( + PQ, PM) + + matches_im1 = pts2d_list[1][reciprocal_in_PM] + matches_im0 = pts2d_list[0][nnM_in_PQ][reciprocal_in_PM] + + if len(matches_im0) == 0: + continue + imidx0, imidx1, colmap_matches = convert_im_matches_pairs(pairs[i][0], pairs[i][1], + image_to_colmap, im_keypoints, + matches_im0, matches_im1, viz) + im_matches[(imidx0, imidx1)] = colmap_matches + return im_matches + + +def get_im_matches_from_cache(pairs, cache_path, desc_conf, subsample, + image_to_colmap, im_keypoints, conf_thr, + viz=False, device='cuda'): + im_matches = {} + for i in range(len(pairs)): + imidx0 = pairs[i][0]['idx'] + imidx1 = pairs[i][1]['idx'] + + corres_idx1 = hash_md5(pairs[i][0]['instance']) + corres_idx2 = hash_md5(pairs[i][1]['instance']) + + path_corres = cache_path + f'/corres_conf={desc_conf}_{subsample=}/{corres_idx1}-{corres_idx2}.pth' + if os.path.isfile(path_corres): + score, (xy1, xy2, confs) = torch.load(path_corres, map_location=device) + else: + path_corres = cache_path + f'/corres_conf={desc_conf}_{subsample=}/{corres_idx2}-{corres_idx1}.pth' + score, (xy2, xy1, confs) = torch.load(path_corres, map_location=device) + mask = confs >= conf_thr + matches_im0 = xy1[mask].cpu().numpy() + matches_im1 = xy2[mask].cpu().numpy() + + if len(matches_im0) == 0: + continue + imidx0, imidx1, colmap_matches = convert_im_matches_pairs(pairs[i][0], pairs[i][1], + image_to_colmap, im_keypoints, + matches_im0, matches_im1, viz) + im_matches[(imidx0, imidx1)] = colmap_matches + return im_matches + + +def export_images(db, images, image_paths, focals, ga_world_to_cam, camera_model): + # add cameras/images to the db + # with the output of ga as prior + image_to_colmap = {} + im_keypoints = {} + for idx in range(len(image_paths)): + im_keypoints[idx] = {} + H, W = images[idx]["orig_shape"] + if focals is None: + focal_x = focal_y = 1.2 * max(W, H) + prior_focal_length = False + cx = W / 2.0 + cy = H / 2.0 + elif isinstance(focals[idx], np.ndarray) and len(focals[idx].shape) == 2: + # intrinsics + focal_x = focals[idx][0, 0] + focal_y = focals[idx][1, 1] + cx = focals[idx][0, 2] * images[idx]["to_orig"][0, 0] + cy = focals[idx][1, 2] * images[idx]["to_orig"][1, 1] + prior_focal_length = True + else: + focal_x = focal_y = float(focals[idx]) + prior_focal_length = True + cx = W / 2.0 + cy = H / 2.0 + focal_x = focal_x * images[idx]["to_orig"][0, 0] + focal_y = focal_y * images[idx]["to_orig"][1, 1] + + if camera_model == "SIMPLE_PINHOLE": + model_id = 0 + focal = (focal_x + focal_y) / 2.0 + params = np.asarray([focal, cx, cy], np.float64) + elif camera_model == "PINHOLE": + model_id = 1 + params = np.asarray([focal_x, focal_y, cx, cy], np.float64) + elif camera_model == "SIMPLE_RADIAL": + model_id = 2 + focal = (focal_x + focal_y) / 2.0 + params = np.asarray([focal, cx, cy, 0.0], np.float64) + elif camera_model == "OPENCV": + model_id = 4 + params = np.asarray([focal_x, focal_y, cx, cy, 0.0, 0.0, 0.0, 0.0], np.float64) + else: + raise ValueError(f"invalid camera model {camera_model}") + + H, W = int(H), int(W) + # OPENCV camera model + camid = db.add_camera( + model_id, W, H, params, prior_focal_length=prior_focal_length) + if ga_world_to_cam is None: + prior_t = np.zeros(3) + prior_q = np.zeros(4) + else: + q = R.from_matrix(ga_world_to_cam[idx][:3, :3]).as_quat() + prior_t = ga_world_to_cam[idx][:3, 3] + prior_q = np.array([q[-1], q[0], q[1], q[2]]) + imid = db.add_image( + image_paths[idx], camid, prior_q=prior_q, prior_t=prior_t) + image_to_colmap[idx] = { + 'colmap_imid': imid, + 'colmap_camid': camid + } + return image_to_colmap, im_keypoints + + +def export_matches(db, images, image_to_colmap, im_keypoints, im_matches, min_len_track, skip_geometric_verification): + colmap_image_pairs = [] + # 2D-2D are quite dense + # we want to remove the very small tracks + # and export only kpt for which we have values + # build tracks + print("building tracks") + keypoints_to_track_id = {} + track_id_to_kpt_list = [] + to_merge = [] + for (imidx0, imidx1), colmap_matches in tqdm(im_matches.items()): + if imidx0 not in keypoints_to_track_id: + keypoints_to_track_id[imidx0] = {} + if imidx1 not in keypoints_to_track_id: + keypoints_to_track_id[imidx1] = {} + + for m in colmap_matches: + if m[0] not in keypoints_to_track_id[imidx0] and m[1] not in keypoints_to_track_id[imidx1]: + # new pair of kpts never seen before + track_idx = len(track_id_to_kpt_list) + keypoints_to_track_id[imidx0][m[0]] = track_idx + keypoints_to_track_id[imidx1][m[1]] = track_idx + track_id_to_kpt_list.append( + [(imidx0, m[0]), (imidx1, m[1])]) + elif m[1] not in keypoints_to_track_id[imidx1]: + # 0 has a track, not 1 + track_idx = keypoints_to_track_id[imidx0][m[0]] + keypoints_to_track_id[imidx1][m[1]] = track_idx + track_id_to_kpt_list[track_idx].append((imidx1, m[1])) + elif m[0] not in keypoints_to_track_id[imidx0]: + # 1 has a track, not 0 + track_idx = keypoints_to_track_id[imidx1][m[1]] + keypoints_to_track_id[imidx0][m[0]] = track_idx + track_id_to_kpt_list[track_idx].append((imidx0, m[0])) + else: + # both have tracks, merge them + track_idx0 = keypoints_to_track_id[imidx0][m[0]] + track_idx1 = keypoints_to_track_id[imidx1][m[1]] + if track_idx0 != track_idx1: + # let's deal with them later + to_merge.append((track_idx0, track_idx1)) + + # regroup merge targets + print("merging tracks") + unique = np.unique(to_merge) + tree = DisjointSet(unique) + for track_idx0, track_idx1 in tqdm(to_merge): + tree.merge(track_idx0, track_idx1) + + subsets = tree.subsets() + print("applying merge") + for setvals in tqdm(subsets): + new_trackid = len(track_id_to_kpt_list) + kpt_list = [] + for track_idx in setvals: + kpt_list.extend(track_id_to_kpt_list[track_idx]) + for imidx, kpid in track_id_to_kpt_list[track_idx]: + keypoints_to_track_id[imidx][kpid] = new_trackid + track_id_to_kpt_list.append(kpt_list) + + # binc = np.bincount([len(v) for v in track_id_to_kpt_list]) + # nonzero = np.nonzero(binc) + # nonzerobinc = binc[nonzero[0]] + # print(nonzero[0].tolist()) + # print(nonzerobinc) + num_valid_tracks = sum( + [1 for v in track_id_to_kpt_list if len(v) >= min_len_track]) + + keypoints_to_idx = {} + print(f"squashing keypoints - {num_valid_tracks} valid tracks") + for imidx, keypoints_imid in tqdm(im_keypoints.items()): + imid = image_to_colmap[imidx]['colmap_imid'] + keypoints_kept = [] + keypoints_to_idx[imidx] = {} + for kp in keypoints_imid.keys(): + if kp not in keypoints_to_track_id[imidx]: + continue + track_idx = keypoints_to_track_id[imidx][kp] + track_length = len(track_id_to_kpt_list[track_idx]) + if track_length < min_len_track: + continue + keypoints_to_idx[imidx][kp] = len(keypoints_kept) + keypoints_kept.append(kp) + if len(keypoints_kept) == 0: + continue + keypoints_kept = np.array(keypoints_kept) + keypoints_kept = np.unravel_index(keypoints_kept, images[imidx]['true_shape'][0])[ + 0].base[:, ::-1].copy().astype(np.float32) + # rescale coordinates + keypoints_kept[:, 0] += 0.5 + keypoints_kept[:, 1] += 0.5 + keypoints_kept = geotrf(images[imidx]['to_orig'], keypoints_kept, norm=True) + + H, W = images[imidx]['orig_shape'] + keypoints_kept[:, 0] = keypoints_kept[:, 0].clip(min=0, max=W - 0.01) + keypoints_kept[:, 1] = keypoints_kept[:, 1].clip(min=0, max=H - 0.01) + + db.add_keypoints(imid, keypoints_kept) + + print("exporting im_matches") + for (imidx0, imidx1), colmap_matches in im_matches.items(): + imid0, imid1 = image_to_colmap[imidx0]['colmap_imid'], image_to_colmap[imidx1]['colmap_imid'] + assert imid0 < imid1 + final_matches = np.array([[keypoints_to_idx[imidx0][m[0]], keypoints_to_idx[imidx1][m[1]]] + for m in colmap_matches + if m[0] in keypoints_to_idx[imidx0] and m[1] in keypoints_to_idx[imidx1]]) + if len(final_matches) > 0: + colmap_image_pairs.append( + (images[imidx0]['instance'], images[imidx1]['instance'])) + db.add_matches(imid0, imid1, final_matches) + if skip_geometric_verification: + db.add_two_view_geometry(imid0, imid1, final_matches) + return colmap_image_pairs diff --git a/submodules/mast3r/mast3r/datasets/__init__.py b/submodules/mast3r/mast3r/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c625aca0a773c105ed229ff87364721b4755bc8d --- /dev/null +++ b/submodules/mast3r/mast3r/datasets/__init__.py @@ -0,0 +1,62 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +from .base.mast3r_base_stereo_view_dataset import MASt3RBaseStereoViewDataset + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.datasets.arkitscenes import ARKitScenes as DUSt3R_ARKitScenes # noqa +from dust3r.datasets.blendedmvs import BlendedMVS as DUSt3R_BlendedMVS # noqa +from dust3r.datasets.co3d import Co3d as DUSt3R_Co3d # noqa +from dust3r.datasets.megadepth import MegaDepth as DUSt3R_MegaDepth # noqa +from dust3r.datasets.scannetpp import ScanNetpp as DUSt3R_ScanNetpp # noqa +from dust3r.datasets.staticthings3d import StaticThings3D as DUSt3R_StaticThings3D # noqa +from dust3r.datasets.waymo import Waymo as DUSt3R_Waymo # noqa +from dust3r.datasets.wildrgbd import WildRGBD as DUSt3R_WildRGBD # noqa + + +class ARKitScenes(DUSt3R_ARKitScenes, MASt3RBaseStereoViewDataset): + def __init__(self, *args, split, ROOT, **kwargs): + super().__init__(*args, split=split, ROOT=ROOT, **kwargs) + self.is_metric_scale = True + + +class BlendedMVS(DUSt3R_BlendedMVS, MASt3RBaseStereoViewDataset): + def __init__(self, *args, ROOT, split=None, **kwargs): + super().__init__(*args, ROOT=ROOT, split=split, **kwargs) + self.is_metric_scale = False + + +class Co3d(DUSt3R_Co3d, MASt3RBaseStereoViewDataset): + def __init__(self, mask_bg=True, *args, ROOT, **kwargs): + super().__init__(mask_bg, *args, ROOT=ROOT, **kwargs) + self.is_metric_scale = False + + +class MegaDepth(DUSt3R_MegaDepth, MASt3RBaseStereoViewDataset): + def __init__(self, *args, split, ROOT, **kwargs): + super().__init__(*args, split=split, ROOT=ROOT, **kwargs) + self.is_metric_scale = False + + +class ScanNetpp(DUSt3R_ScanNetpp, MASt3RBaseStereoViewDataset): + def __init__(self, *args, ROOT, **kwargs): + super().__init__(*args, ROOT=ROOT, **kwargs) + self.is_metric_scale = True + + +class StaticThings3D(DUSt3R_StaticThings3D, MASt3RBaseStereoViewDataset): + def __init__(self, ROOT, *args, mask_bg='rand', **kwargs): + super().__init__(ROOT, *args, mask_bg=mask_bg, **kwargs) + self.is_metric_scale = False + + +class Waymo(DUSt3R_Waymo, MASt3RBaseStereoViewDataset): + def __init__(self, *args, ROOT, **kwargs): + super().__init__(*args, ROOT=ROOT, **kwargs) + self.is_metric_scale = True + + +class WildRGBD(DUSt3R_WildRGBD, MASt3RBaseStereoViewDataset): + def __init__(self, mask_bg=True, *args, ROOT, **kwargs): + super().__init__(mask_bg, *args, ROOT=ROOT, **kwargs) + self.is_metric_scale = True diff --git a/submodules/mast3r/mast3r/datasets/base/__init__.py b/submodules/mast3r/mast3r/datasets/base/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7dd877d649ce4dbd749dd7195a8b34c0f91d4f0 --- /dev/null +++ b/submodules/mast3r/mast3r/datasets/base/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). \ No newline at end of file diff --git a/submodules/mast3r/mast3r/datasets/base/mast3r_base_stereo_view_dataset.py b/submodules/mast3r/mast3r/datasets/base/mast3r_base_stereo_view_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..3ced0ef0dc6b1d6225781af55d3e924e133fdeaf --- /dev/null +++ b/submodules/mast3r/mast3r/datasets/base/mast3r_base_stereo_view_dataset.py @@ -0,0 +1,355 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# base class for implementing datasets +# -------------------------------------------------------- +import PIL.Image +import PIL.Image as Image +import numpy as np +import torch +import copy + +from mast3r.datasets.utils.cropping import (extract_correspondences_from_pts3d, + gen_random_crops, in2d_rect, crop_to_homography) + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset, view_name, is_good_type # noqa +from dust3r.datasets.utils.transforms import ImgNorm +from dust3r.utils.geometry import depthmap_to_absolute_camera_coordinates, geotrf, depthmap_to_camera_coordinates +import dust3r.datasets.utils.cropping as cropping + + +class MASt3RBaseStereoViewDataset(BaseStereoViewDataset): + def __init__(self, *, # only keyword arguments + split=None, + resolution=None, # square_size or (width, height) or list of [(width,height), ...] + transform=ImgNorm, + aug_crop=False, + aug_swap=False, + aug_monocular=False, + aug_portrait_or_landscape=True, # automatic choice between landscape/portrait when possible + aug_rot90=False, + n_corres=0, + nneg=0, + n_tentative_crops=4, + seed=None): + super().__init__(split=split, resolution=resolution, transform=transform, aug_crop=aug_crop, seed=seed) + self.is_metric_scale = False # by default a dataset is not metric scale, subclasses can overwrite this + + self.aug_swap = aug_swap + self.aug_monocular = aug_monocular + self.aug_portrait_or_landscape = aug_portrait_or_landscape + self.aug_rot90 = aug_rot90 + + self.n_corres = n_corres + self.nneg = nneg + assert self.n_corres == 'all' or isinstance(self.n_corres, int) or (isinstance(self.n_corres, list) and len( + self.n_corres) == self.num_views), f"Error, n_corres should either be 'all', a single integer or a list of length {self.num_views}" + assert self.nneg == 0 or self.n_corres != 'all' + self.n_tentative_crops = n_tentative_crops + + def _swap_view_aug(self, views): + if self._rng.random() < 0.5: + views.reverse() + + def _crop_resize_if_necessary(self, image, depthmap, intrinsics, resolution, rng=None, info=None): + """ This function: + - first downsizes the image with LANCZOS inteprolation, + which is better than bilinear interpolation in + """ + if not isinstance(image, PIL.Image.Image): + image = PIL.Image.fromarray(image) + + # transpose the resolution if necessary + W, H = image.size # new size + assert resolution[0] >= resolution[1] + if H > 1.1 * W: + # image is portrait mode + resolution = resolution[::-1] + elif 0.9 < H / W < 1.1 and resolution[0] != resolution[1]: + # image is square, so we chose (portrait, landscape) randomly + if rng.integers(2) and self.aug_portrait_or_landscape: + resolution = resolution[::-1] + + # high-quality Lanczos down-scaling + target_resolution = np.array(resolution) + image, depthmap, intrinsics = cropping.rescale_image_depthmap(image, depthmap, intrinsics, target_resolution) + + # actual cropping (if necessary) with bilinear interpolation + offset_factor = 0.5 + intrinsics2 = cropping.camera_matrix_of_crop(intrinsics, image.size, resolution, offset_factor=offset_factor) + crop_bbox = cropping.bbox_from_intrinsics_in_out(intrinsics, intrinsics2, resolution) + image, depthmap, intrinsics2 = cropping.crop_image_depthmap(image, depthmap, intrinsics, crop_bbox) + + return image, depthmap, intrinsics2 + + def generate_crops_from_pair(self, view1, view2, resolution, aug_crop_arg, n_crops=4, rng=np.random): + views = [view1, view2] + + if aug_crop_arg is False: + # compatibility + for i in range(2): + view = views[i] + view['img'], view['depthmap'], view['camera_intrinsics'] = self._crop_resize_if_necessary(view['img'], + view['depthmap'], + view['camera_intrinsics'], + resolution, + rng=rng) + view['pts3d'], view['valid_mask'] = depthmap_to_absolute_camera_coordinates(view['depthmap'], + view['camera_intrinsics'], + view['camera_pose']) + return + + # extract correspondences + corres = extract_correspondences_from_pts3d(*views, target_n_corres=None, rng=rng) + + # generate 4 random crops in each view + view_crops = [] + crops_resolution = [] + corres_msks = [] + for i in range(2): + + if aug_crop_arg == 'auto': + S = min(views[i]['img'].size) + R = min(resolution) + aug_crop = S * (S - R) // R + aug_crop = max(.1 * S, aug_crop) # for cropping: augment scale of at least 10%, and more if possible + else: + aug_crop = aug_crop_arg + + # tranpose the target resolution if necessary + assert resolution[0] >= resolution[1] + W, H = imsize = views[i]['img'].size + crop_resolution = resolution + if H > 1.1 * W: + # image is portrait mode + crop_resolution = resolution[::-1] + elif 0.9 < H / W < 1.1 and resolution[0] != resolution[1]: + # image is square, so we chose (portrait, landscape) randomly + if rng.integers(2): + crop_resolution = resolution[::-1] + + crops = gen_random_crops(imsize, n_crops, crop_resolution, aug_crop=aug_crop, rng=rng) + view_crops.append(crops) + crops_resolution.append(crop_resolution) + + # compute correspondences + corres_msks.append(in2d_rect(corres[i], crops)) + + # compute IoU for each + intersection = np.float32(corres_msks[0]).T @ np.float32(corres_msks[1]) + # select best pair of crops + best = np.unravel_index(intersection.argmax(), (n_crops, n_crops)) + crops = [view_crops[i][c] for i, c in enumerate(best)] + + # crop with the homography + for i in range(2): + view = views[i] + imsize, K_new, R, H = crop_to_homography(view['camera_intrinsics'], crops[i], crops_resolution[i]) + # imsize, K_new, H = upscale_homography(imsize, resolution, K_new, H) + + # update camera params + K_old = view['camera_intrinsics'] + view['camera_intrinsics'] = K_new + view['camera_pose'] = view['camera_pose'].copy() + view['camera_pose'][:3, :3] = view['camera_pose'][:3, :3] @ R + + # apply homography to image and depthmap + homo8 = (H / H[2, 2]).ravel().tolist()[:8] + view['img'] = view['img'].transform(imsize, Image.Transform.PERSPECTIVE, + homo8, + resample=Image.Resampling.BICUBIC) + + depthmap2 = depthmap_to_camera_coordinates(view['depthmap'], K_old)[0] @ R[:, 2] + view['depthmap'] = np.array(Image.fromarray(depthmap2).transform( + imsize, Image.Transform.PERSPECTIVE, homo8)) + + if 'track_labels' in view: + # convert from uint64 --> uint32, because PIL.Image cannot handle uint64 + mapping, track_labels = np.unique(view['track_labels'], return_inverse=True) + track_labels = track_labels.astype(np.uint32).reshape(view['track_labels'].shape) + + # homography transformation + res = np.array(Image.fromarray(track_labels).transform(imsize, Image.Transform.PERSPECTIVE, homo8)) + view['track_labels'] = mapping[res] # mapping back to uint64 + + # recompute 3d points from scratch + view['pts3d'], view['valid_mask'] = depthmap_to_absolute_camera_coordinates(view['depthmap'], + view['camera_intrinsics'], + view['camera_pose']) + + def __getitem__(self, idx): + if isinstance(idx, tuple): + # the idx is specifying the aspect-ratio + idx, ar_idx = idx + else: + assert len(self._resolutions) == 1 + ar_idx = 0 + + # set-up the rng + if self.seed: # reseed for each __getitem__ + self._rng = np.random.default_rng(seed=self.seed + idx) + elif not hasattr(self, '_rng'): + seed = torch.initial_seed() # this is different for each dataloader process + self._rng = np.random.default_rng(seed=seed) + + # over-loaded code + resolution = self._resolutions[ar_idx] # DO NOT CHANGE THIS (compatible with BatchedRandomSampler) + views = self._get_views(idx, resolution, self._rng) + assert len(views) == self.num_views + + for v, view in enumerate(views): + assert 'pts3d' not in view, f"pts3d should not be there, they will be computed afterwards based on intrinsics+depthmap for view {view_name(view)}" + view['idx'] = (idx, ar_idx, v) + view['is_metric_scale'] = self.is_metric_scale + + assert 'camera_intrinsics' in view + if 'camera_pose' not in view: + view['camera_pose'] = np.full((4, 4), np.nan, dtype=np.float32) + else: + assert np.isfinite(view['camera_pose']).all(), f'NaN in camera pose for view {view_name(view)}' + assert 'pts3d' not in view + assert 'valid_mask' not in view + assert np.isfinite(view['depthmap']).all(), f'NaN in depthmap for view {view_name(view)}' + + pts3d, valid_mask = depthmap_to_absolute_camera_coordinates(**view) + + view['pts3d'] = pts3d + view['valid_mask'] = valid_mask & np.isfinite(pts3d).all(axis=-1) + + self.generate_crops_from_pair(views[0], views[1], resolution=resolution, + aug_crop_arg=self.aug_crop, + n_crops=self.n_tentative_crops, + rng=self._rng) + for v, view in enumerate(views): + # encode the image + width, height = view['img'].size + view['true_shape'] = np.int32((height, width)) + view['img'] = self.transform(view['img']) + # Pixels for which depth is fundamentally undefined + view['sky_mask'] = (view['depthmap'] < 0) + + if self.aug_swap: + self._swap_view_aug(views) + + if self.aug_monocular: + if self._rng.random() < self.aug_monocular: + views = [copy.deepcopy(views[0]) for _ in range(len(views))] + + # automatic extraction of correspondences from pts3d + pose + if self.n_corres > 0 and ('corres' not in view): + corres1, corres2, valid = extract_correspondences_from_pts3d(*views, self.n_corres, + self._rng, nneg=self.nneg) + views[0]['corres'] = corres1 + views[1]['corres'] = corres2 + views[0]['valid_corres'] = valid + views[1]['valid_corres'] = valid + + if self.aug_rot90 is False: + pass + elif self.aug_rot90 == 'same': + rotate_90(views, k=self._rng.choice(4)) + elif self.aug_rot90 == 'diff': + rotate_90(views[:1], k=self._rng.choice(4)) + rotate_90(views[1:], k=self._rng.choice(4)) + else: + raise ValueError(f'Bad value for {self.aug_rot90=}') + + # check data-types metric_scale + for v, view in enumerate(views): + if 'corres' not in view: + view['corres'] = np.full((self.n_corres, 2), np.nan, dtype=np.float32) + + # check all datatypes + for key, val in view.items(): + res, err_msg = is_good_type(key, val) + assert res, f"{err_msg} with {key}={val} for view {view_name(view)}" + K = view['camera_intrinsics'] + + # check shapes + assert view['depthmap'].shape == view['img'].shape[1:] + assert view['depthmap'].shape == view['pts3d'].shape[:2] + assert view['depthmap'].shape == view['valid_mask'].shape + + # last thing done! + for view in views: + # transpose to make sure all views are the same size + transpose_to_landscape(view) + # this allows to check whether the RNG is is the same state each time + view['rng'] = int.from_bytes(self._rng.bytes(4), 'big') + + return views + + +def transpose_to_landscape(view, revert=False): + height, width = view['true_shape'] + + if width < height: + if revert: + height, width = width, height + + # rectify portrait to landscape + assert view['img'].shape == (3, height, width) + view['img'] = view['img'].swapaxes(1, 2) + + assert view['valid_mask'].shape == (height, width) + view['valid_mask'] = view['valid_mask'].swapaxes(0, 1) + + assert view['sky_mask'].shape == (height, width) + view['sky_mask'] = view['sky_mask'].swapaxes(0, 1) + + assert view['depthmap'].shape == (height, width) + view['depthmap'] = view['depthmap'].swapaxes(0, 1) + + assert view['pts3d'].shape == (height, width, 3) + view['pts3d'] = view['pts3d'].swapaxes(0, 1) + + # transpose x and y pixels + view['camera_intrinsics'] = view['camera_intrinsics'][[1, 0, 2]] + + # transpose correspondences x and y + view['corres'] = view['corres'][:, [1, 0]] + + +def rotate_90(views, k=1): + from scipy.spatial.transform import Rotation + # print('rotation =', k) + + RT = np.eye(4, dtype=np.float32) + RT[:3, :3] = Rotation.from_euler('z', 90 * k, degrees=True).as_matrix() + + for view in views: + view['img'] = torch.rot90(view['img'], k=k, dims=(-2, -1)) # WARNING!! dims=(-1,-2) != dims=(-2,-1) + view['depthmap'] = np.rot90(view['depthmap'], k=k).copy() + view['camera_pose'] = view['camera_pose'] @ RT + + RT2 = np.eye(3, dtype=np.float32) + RT2[:2, :2] = RT[:2, :2] * ((1, -1), (-1, 1)) + H, W = view['depthmap'].shape + if k % 4 == 0: + pass + elif k % 4 == 1: + # top-left (0,0) pixel becomes (0,H-1) + RT2[:2, 2] = (0, H - 1) + elif k % 4 == 2: + # top-left (0,0) pixel becomes (W-1,H-1) + RT2[:2, 2] = (W - 1, H - 1) + elif k % 4 == 3: + # top-left (0,0) pixel becomes (W-1,0) + RT2[:2, 2] = (W - 1, 0) + else: + raise ValueError(f'Bad value for {k=}') + + view['camera_intrinsics'][:2, 2] = geotrf(RT2, view['camera_intrinsics'][:2, 2]) + if k % 2 == 1: + K = view['camera_intrinsics'] + np.fill_diagonal(K, K.diagonal()[[1, 0, 2]]) + + pts3d, valid_mask = depthmap_to_absolute_camera_coordinates(**view) + view['pts3d'] = pts3d + view['valid_mask'] = np.rot90(view['valid_mask'], k=k).copy() + view['sky_mask'] = np.rot90(view['sky_mask'], k=k).copy() + + view['corres'] = geotrf(RT2, view['corres']).round().astype(view['corres'].dtype) + view['true_shape'] = np.int32((H, W)) diff --git a/submodules/mast3r/mast3r/datasets/utils/__init__.py b/submodules/mast3r/mast3r/datasets/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a32692113d830ddc4af4e6ed608f222fbe062e6e --- /dev/null +++ b/submodules/mast3r/mast3r/datasets/utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). diff --git a/submodules/mast3r/mast3r/datasets/utils/cropping.py b/submodules/mast3r/mast3r/datasets/utils/cropping.py new file mode 100644 index 0000000000000000000000000000000000000000..57f4d84b019eaac9cf0c308a94f2cb8e2ec1a6ba --- /dev/null +++ b/submodules/mast3r/mast3r/datasets/utils/cropping.py @@ -0,0 +1,219 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# cropping/match extraction +# -------------------------------------------------------- +import numpy as np +import mast3r.utils.path_to_dust3r # noqa +from dust3r.utils.device import to_numpy +from dust3r.utils.geometry import inv, geotrf + + +def reciprocal_1d(corres_1_to_2, corres_2_to_1, ret_recip=False): + is_reciprocal1 = (corres_2_to_1[corres_1_to_2] == np.arange(len(corres_1_to_2))) + pos1 = is_reciprocal1.nonzero()[0] + pos2 = corres_1_to_2[pos1] + if ret_recip: + return is_reciprocal1, pos1, pos2 + return pos1, pos2 + + +def extract_correspondences_from_pts3d(view1, view2, target_n_corres, rng=np.random, ret_xy=True, nneg=0): + view1, view2 = to_numpy((view1, view2)) + # project pixels from image1 --> 3d points --> image2 pixels + shape1, corres1_to_2 = reproject_view(view1['pts3d'], view2) + shape2, corres2_to_1 = reproject_view(view2['pts3d'], view1) + + # compute reciprocal correspondences: + # pos1 == valid pixels (correspondences) in image1 + is_reciprocal1, pos1, pos2 = reciprocal_1d(corres1_to_2, corres2_to_1, ret_recip=True) + is_reciprocal2 = (corres1_to_2[corres2_to_1] == np.arange(len(corres2_to_1))) + + if target_n_corres is None: + if ret_xy: + pos1 = unravel_xy(pos1, shape1) + pos2 = unravel_xy(pos2, shape2) + return pos1, pos2 + + available_negatives = min((~is_reciprocal1).sum(), (~is_reciprocal2).sum()) + target_n_positives = int(target_n_corres * (1 - nneg)) + n_positives = min(len(pos1), target_n_positives) + n_negatives = min(target_n_corres - n_positives, available_negatives) + + if n_negatives + n_positives != target_n_corres: + # should be really rare => when there are not enough negatives + # in that case, break nneg and add a few more positives ? + n_positives = target_n_corres - n_negatives + assert n_positives <= len(pos1) + + assert n_positives <= len(pos1) + assert n_positives <= len(pos2) + assert n_negatives <= (~is_reciprocal1).sum() + assert n_negatives <= (~is_reciprocal2).sum() + assert n_positives + n_negatives == target_n_corres + + valid = np.ones(n_positives, dtype=bool) + if n_positives < len(pos1): + # random sub-sampling of valid correspondences + perm = rng.permutation(len(pos1))[:n_positives] + pos1 = pos1[perm] + pos2 = pos2[perm] + + if n_negatives > 0: + # add false correspondences if not enough + def norm(p): return p / p.sum() + pos1 = np.r_[pos1, rng.choice(shape1[0] * shape1[1], size=n_negatives, replace=False, p=norm(~is_reciprocal1))] + pos2 = np.r_[pos2, rng.choice(shape2[0] * shape2[1], size=n_negatives, replace=False, p=norm(~is_reciprocal2))] + valid = np.r_[valid, np.zeros(n_negatives, dtype=bool)] + + # convert (x+W*y) back to 2d (x,y) coordinates + if ret_xy: + pos1 = unravel_xy(pos1, shape1) + pos2 = unravel_xy(pos2, shape2) + return pos1, pos2, valid + + +def reproject_view(pts3d, view2): + shape = view2['pts3d'].shape[:2] + return reproject(pts3d, view2['camera_intrinsics'], inv(view2['camera_pose']), shape) + + +def reproject(pts3d, K, world2cam, shape): + H, W, THREE = pts3d.shape + assert THREE == 3 + + # reproject in camera2 space + with np.errstate(divide='ignore', invalid='ignore'): + pos = geotrf(K @ world2cam[:3], pts3d, norm=1, ncol=2) + + # quantize to pixel positions + return (H, W), ravel_xy(pos, shape) + + +def ravel_xy(pos, shape): + H, W = shape + with np.errstate(invalid='ignore'): + qx, qy = pos.reshape(-1, 2).round().astype(np.int32).T + quantized_pos = qx.clip(min=0, max=W - 1, out=qx) + W * qy.clip(min=0, max=H - 1, out=qy) + return quantized_pos + + +def unravel_xy(pos, shape): + # convert (x+W*y) back to 2d (x,y) coordinates + return np.unravel_index(pos, shape)[0].base[:, ::-1].copy() + + +def _rotation_origin_to_pt(target): + """ Align the origin (0,0,1) with the target point (x,y,1) in projective space. + Method: rotate z to put target on (x'+,0,1), then rotate on Y to get (0,0,1) and un-rotate z. + """ + from scipy.spatial.transform import Rotation + x, y = target + rot_z = np.arctan2(y, x) + rot_y = np.arctan(np.linalg.norm(target)) + R = Rotation.from_euler('ZYZ', [rot_z, rot_y, -rot_z]).as_matrix() + return R + + +def _dotmv(Trf, pts, ncol=None, norm=False): + assert Trf.ndim >= 2 + ncol = ncol or pts.shape[-1] + + # adapt shape if necessary + output_reshape = pts.shape[:-1] + if Trf.ndim >= 3: + n = Trf.ndim - 2 + assert Trf.shape[:n] == pts.shape[:n], 'batch size does not match' + Trf = Trf.reshape(-1, Trf.shape[-2], Trf.shape[-1]) + + if pts.ndim > Trf.ndim: + # Trf == (B,d,d) & pts == (B,H,W,d) --> (B, H*W, d) + pts = pts.reshape(Trf.shape[0], -1, pts.shape[-1]) + elif pts.ndim == 2: + # Trf == (B,d,d) & pts == (B,d) --> (B, 1, d) + pts = pts[:, None, :] + + if pts.shape[-1] + 1 == Trf.shape[-1]: + Trf = Trf.swapaxes(-1, -2) # transpose Trf + pts = pts @ Trf[..., :-1, :] + Trf[..., -1:, :] + + elif pts.shape[-1] == Trf.shape[-1]: + Trf = Trf.swapaxes(-1, -2) # transpose Trf + pts = pts @ Trf + else: + pts = Trf @ pts.T + if pts.ndim >= 2: + pts = pts.swapaxes(-1, -2) + + if norm: + pts = pts / pts[..., -1:] # DONT DO /= BECAUSE OF WEIRD PYTORCH BUG + if norm != 1: + pts *= norm + + res = pts[..., :ncol].reshape(*output_reshape, ncol) + return res + + +def crop_to_homography(K, crop, target_size=None): + """ Given an image and its intrinsics, + we want to replicate a rectangular crop with an homography, + so that the principal point of the new 'crop' is centered. + """ + # build intrinsics for the crop + crop = np.round(crop) + crop_size = crop[2:] - crop[:2] + K2 = K.copy() # same focal + K2[:2, 2] = crop_size / 2 # new principal point is perfectly centered + + # find which corner is the most far-away from current principal point + # so that the final homography does not go over the image borders + corners = crop.reshape(-1, 2) + corner_idx = np.abs(corners - K[:2, 2]).argmax(0) + corner = corners[corner_idx, [0, 1]] + # align with the corresponding corner from the target view + corner2 = np.c_[[0, 0], crop_size][[0, 1], corner_idx] + + old_pt = _dotmv(np.linalg.inv(K), corner, norm=1) + new_pt = _dotmv(np.linalg.inv(K2), corner2, norm=1) + R = _rotation_origin_to_pt(old_pt) @ np.linalg.inv(_rotation_origin_to_pt(new_pt)) + + if target_size is not None: + imsize = target_size + target_size = np.asarray(target_size) + scaling = min(target_size / crop_size) + K2[:2] *= scaling + K2[:2, 2] = target_size / 2 + else: + imsize = tuple(np.int32(crop_size).tolist()) + + return imsize, K2, R, K @ R @ np.linalg.inv(K2) + + +def gen_random_crops(imsize, n_crops, resolution, aug_crop, rng=np.random): + """ Generate random crops of size=resolution, + for an input image upscaled to (imsize + randint(0 , aug_crop)) + """ + resolution_crop = np.array(resolution) * min(np.array(imsize) / resolution) + + # (virtually) upscale the input image + # scaling = rng.uniform(1, 1+(aug_crop+1)/min(imsize)) + scaling = np.exp(rng.uniform(0, np.log(1 + aug_crop / min(imsize)))) + imsize2 = np.int32(np.array(imsize) * scaling) + + # generate some random crops + topleft = rng.random((n_crops, 2)) * (imsize2 - resolution_crop) + crops = np.c_[topleft, topleft + resolution_crop] + # print(f"{scaling=}, {topleft=}") + # reduce the resolution to come back to original size + crops /= scaling + return crops + + +def in2d_rect(corres, crops): + # corres = (N,2) + # crops = (M,4) + # output = (N, M) + is_sup = (corres[:, None] >= crops[None, :, 0:2]) + is_inf = (corres[:, None] < crops[None, :, 2:4]) + return (is_sup & is_inf).all(axis=-1) diff --git a/submodules/mast3r/mast3r/demo.py b/submodules/mast3r/mast3r/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..22b6a66c24666776a7197844a0463d7821ed53ce --- /dev/null +++ b/submodules/mast3r/mast3r/demo.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# sparse gradio demo functions +# -------------------------------------------------------- +import math +import gradio +import os +import numpy as np +import functools +import trimesh +import copy +from scipy.spatial.transform import Rotation +import tempfile +import shutil + +from mast3r.cloud_opt.sparse_ga import sparse_global_alignment +from mast3r.cloud_opt.tsdf_optimizer import TSDFPostProcess + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.image_pairs import make_pairs +from dust3r.utils.image import load_images +from dust3r.utils.device import to_numpy +from dust3r.viz import add_scene_cam, CAM_COLORS, OPENGL, pts3d_to_trimesh, cat_meshes +from dust3r.demo import get_args_parser as dust3r_get_args_parser + +import matplotlib.pyplot as pl + + +class SparseGAState(): + def __init__(self, sparse_ga, should_delete=False, cache_dir=None, outfile_name=None): + self.sparse_ga = sparse_ga + self.cache_dir = cache_dir + self.outfile_name = outfile_name + self.should_delete = should_delete + + def __del__(self): + if not self.should_delete: + return + if self.cache_dir is not None and os.path.isdir(self.cache_dir): + shutil.rmtree(self.cache_dir) + self.cache_dir = None + if self.outfile_name is not None and os.path.isfile(self.outfile_name): + os.remove(self.outfile_name) + self.outfile_name = None + + +def get_args_parser(): + parser = dust3r_get_args_parser() + parser.add_argument('--share', action='store_true') + parser.add_argument('--gradio_delete_cache', default=None, type=int, + help='age/frequency at which gradio removes the file. If >0, matching cache is purged') + + actions = parser._actions + for action in actions: + if action.dest == 'model_name': + action.choices = ["MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric"] + # change defaults + parser.prog = 'mast3r demo' + return parser + + +def _convert_scene_output_to_glb(outfile, imgs, pts3d, mask, focals, cams2world, cam_size=0.05, + cam_color=None, as_pointcloud=False, + transparent_cams=False, silent=False): + assert len(pts3d) == len(mask) <= len(imgs) <= len(cams2world) == len(focals) + pts3d = to_numpy(pts3d) + imgs = to_numpy(imgs) + focals = to_numpy(focals) + cams2world = to_numpy(cams2world) + + scene = trimesh.Scene() + + # full pointcloud + if as_pointcloud: + pts = np.concatenate([p[m.ravel()] for p, m in zip(pts3d, mask)]).reshape(-1, 3) + col = np.concatenate([p[m] for p, m in zip(imgs, mask)]).reshape(-1, 3) + valid_msk = np.isfinite(pts.sum(axis=1)) + pct = trimesh.PointCloud(pts[valid_msk], colors=col[valid_msk]) + scene.add_geometry(pct) + else: + meshes = [] + for i in range(len(imgs)): + pts3d_i = pts3d[i].reshape(imgs[i].shape) + msk_i = mask[i] & np.isfinite(pts3d_i.sum(axis=-1)) + meshes.append(pts3d_to_trimesh(imgs[i], pts3d_i, msk_i)) + mesh = trimesh.Trimesh(**cat_meshes(meshes)) + scene.add_geometry(mesh) + + # add each camera + for i, pose_c2w in enumerate(cams2world): + if isinstance(cam_color, list): + camera_edge_color = cam_color[i] + else: + camera_edge_color = cam_color or CAM_COLORS[i % len(CAM_COLORS)] + add_scene_cam(scene, pose_c2w, camera_edge_color, + None if transparent_cams else imgs[i], focals[i], + imsize=imgs[i].shape[1::-1], screen_width=cam_size) + + rot = np.eye(4) + rot[:3, :3] = Rotation.from_euler('y', np.deg2rad(180)).as_matrix() + scene.apply_transform(np.linalg.inv(cams2world[0] @ OPENGL @ rot)) + if not silent: + print('(exporting 3D scene to', outfile, ')') + scene.export(file_obj=outfile) + return outfile + + +def get_3D_model_from_scene(silent, scene_state, min_conf_thr=2, as_pointcloud=False, mask_sky=False, + clean_depth=False, transparent_cams=False, cam_size=0.05, TSDF_thresh=0): + """ + extract 3D_model (glb file) from a reconstructed scene + """ + if scene_state is None: + return None + outfile = scene_state.outfile_name + if outfile is None: + return None + + # get optimized values from scene + scene = scene_state.sparse_ga + rgbimg = scene.imgs + focals = scene.get_focals().cpu() + cams2world = scene.get_im_poses().cpu() + + # 3D pointcloud from depthmap, poses and intrinsics + if TSDF_thresh > 0: + tsdf = TSDFPostProcess(scene, TSDF_thresh=TSDF_thresh) + pts3d, _, confs = to_numpy(tsdf.get_dense_pts3d(clean_depth=clean_depth)) + else: + pts3d, _, confs = to_numpy(scene.get_dense_pts3d(clean_depth=clean_depth)) + msk = to_numpy([c > min_conf_thr for c in confs]) + return _convert_scene_output_to_glb(outfile, rgbimg, pts3d, msk, focals, cams2world, as_pointcloud=as_pointcloud, + transparent_cams=transparent_cams, cam_size=cam_size, silent=silent) + + +def get_reconstructed_scene(outdir, gradio_delete_cache, model, device, silent, image_size, current_scene_state, + filelist, optim_level, lr1, niter1, lr2, niter2, min_conf_thr, matching_conf_thr, + as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size, scenegraph_type, winsize, + win_cyclic, refid, TSDF_thresh, shared_intrinsics, **kw): + """ + from a list of images, run mast3r inference, sparse global aligner. + then run get_3D_model_from_scene + """ + imgs = load_images(filelist, size=image_size, verbose=not silent) + if len(imgs) == 1: + imgs = [imgs[0], copy.deepcopy(imgs[0])] + imgs[1]['idx'] = 1 + filelist = [filelist[0], filelist[0] + '_2'] + + scene_graph_params = [scenegraph_type] + if scenegraph_type in ["swin", "logwin"]: + scene_graph_params.append(str(winsize)) + elif scenegraph_type == "oneref": + scene_graph_params.append(str(refid)) + if scenegraph_type in ["swin", "logwin"] and not win_cyclic: + scene_graph_params.append('noncyclic') + scene_graph = '-'.join(scene_graph_params) + pairs = make_pairs(imgs, scene_graph=scene_graph, prefilter=None, symmetrize=True) + if optim_level == 'coarse': + niter2 = 0 + # Sparse GA (forward mast3r -> matching -> 3D optim -> 2D refinement -> triangulation) + if current_scene_state is not None and \ + not current_scene_state.should_delete and \ + current_scene_state.cache_dir is not None: + cache_dir = current_scene_state.cache_dir + elif gradio_delete_cache: + cache_dir = tempfile.mkdtemp(suffix='_cache', dir=outdir) + else: + cache_dir = os.path.join(outdir, 'cache') + os.makedirs(cache_dir, exist_ok=True) + scene = sparse_global_alignment(filelist, pairs, cache_dir, + model, lr1=lr1, niter1=niter1, lr2=lr2, niter2=niter2, device=device, + opt_depth='depth' in optim_level, shared_intrinsics=shared_intrinsics, + matching_conf_thr=matching_conf_thr, **kw) + if current_scene_state is not None and \ + not current_scene_state.should_delete and \ + current_scene_state.outfile_name is not None: + outfile_name = current_scene_state.outfile_name + else: + outfile_name = tempfile.mktemp(suffix='_scene.glb', dir=outdir) + + scene_state = SparseGAState(scene, gradio_delete_cache, cache_dir, outfile_name) + outfile = get_3D_model_from_scene(silent, scene_state, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh) + return scene_state, outfile + + +def set_scenegraph_options(inputfiles, win_cyclic, refid, scenegraph_type): + num_files = len(inputfiles) if inputfiles is not None else 1 + show_win_controls = scenegraph_type in ["swin", "logwin"] + show_winsize = scenegraph_type in ["swin", "logwin"] + show_cyclic = scenegraph_type in ["swin", "logwin"] + max_winsize, min_winsize = 1, 1 + if scenegraph_type == "swin": + if win_cyclic: + max_winsize = max(1, math.ceil((num_files - 1) / 2)) + else: + max_winsize = num_files - 1 + elif scenegraph_type == "logwin": + if win_cyclic: + half_size = math.ceil((num_files - 1) / 2) + max_winsize = max(1, math.ceil(math.log(half_size, 2))) + else: + max_winsize = max(1, math.ceil(math.log(num_files, 2))) + winsize = gradio.Slider(label="Scene Graph: Window Size", value=max_winsize, + minimum=min_winsize, maximum=max_winsize, step=1, visible=show_winsize) + win_cyclic = gradio.Checkbox(value=win_cyclic, label="Cyclic sequence", visible=show_cyclic) + win_col = gradio.Column(visible=show_win_controls) + refid = gradio.Slider(label="Scene Graph: Id", value=0, minimum=0, + maximum=num_files - 1, step=1, visible=scenegraph_type == 'oneref') + return win_col, winsize, win_cyclic, refid + + +def main_demo(tmpdirname, model, device, image_size, server_name, server_port, silent=False, + share=False, gradio_delete_cache=False): + if not silent: + print('Outputing stuff in', tmpdirname) + + recon_fun = functools.partial(get_reconstructed_scene, tmpdirname, gradio_delete_cache, model, device, + silent, image_size) + model_from_scene_fun = functools.partial(get_3D_model_from_scene, silent) + + def get_context(delete_cache): + css = """.gradio-container {margin: 0 !important; min-width: 100%};""" + title = "MASt3R Demo" + if delete_cache: + return gradio.Blocks(css=css, title=title, delete_cache=(delete_cache, delete_cache)) + else: + return gradio.Blocks(css=css, title="MASt3R Demo") # for compatibility with older versions + + with get_context(gradio_delete_cache) as demo: + # scene state is save so that you can change conf_thr, cam_size... without rerunning the inference + scene = gradio.State(None) + gradio.HTML('

MASt3R Demo

') + with gradio.Column(): + inputfiles = gradio.File(file_count="multiple") + with gradio.Row(): + with gradio.Column(): + with gradio.Row(): + lr1 = gradio.Slider(label="Coarse LR", value=0.07, minimum=0.01, maximum=0.2, step=0.01) + niter1 = gradio.Number(value=500, precision=0, minimum=0, maximum=10_000, + label="num_iterations", info="For coarse alignment!") + lr2 = gradio.Slider(label="Fine LR", value=0.014, minimum=0.005, maximum=0.05, step=0.001) + niter2 = gradio.Number(value=200, precision=0, minimum=0, maximum=100_000, + label="num_iterations", info="For refinement!") + optim_level = gradio.Dropdown(["coarse", "refine", "refine+depth"], + value='refine+depth', label="OptLevel", + info="Optimization level") + with gradio.Row(): + matching_conf_thr = gradio.Slider(label="Matching Confidence Thr", value=5., + minimum=0., maximum=30., step=0.1, + info="Before Fallback to Regr3D!") + shared_intrinsics = gradio.Checkbox(value=False, label="Shared intrinsics", + info="Only optimize one set of intrinsics for all views") + scenegraph_type = gradio.Dropdown([("complete: all possible image pairs", "complete"), + ("swin: sliding window", "swin"), + ("logwin: sliding window with long range", "logwin"), + ("oneref: match one image with all", "oneref")], + value='complete', label="Scenegraph", + info="Define how to make pairs", + interactive=True) + with gradio.Column(visible=False) as win_col: + winsize = gradio.Slider(label="Scene Graph: Window Size", value=1, + minimum=1, maximum=1, step=1) + win_cyclic = gradio.Checkbox(value=False, label="Cyclic sequence") + refid = gradio.Slider(label="Scene Graph: Id", value=0, + minimum=0, maximum=0, step=1, visible=False) + run_btn = gradio.Button("Run") + + with gradio.Row(): + # adjust the confidence threshold + min_conf_thr = gradio.Slider(label="min_conf_thr", value=1.5, minimum=0.0, maximum=10, step=0.1) + # adjust the camera size in the output pointcloud + cam_size = gradio.Slider(label="cam_size", value=0.2, minimum=0.001, maximum=1.0, step=0.001) + TSDF_thresh = gradio.Slider(label="TSDF Threshold", value=0., minimum=0., maximum=1., step=0.01) + with gradio.Row(): + as_pointcloud = gradio.Checkbox(value=True, label="As pointcloud") + # two post process implemented + mask_sky = gradio.Checkbox(value=False, label="Mask sky") + clean_depth = gradio.Checkbox(value=True, label="Clean-up depthmaps") + transparent_cams = gradio.Checkbox(value=False, label="Transparent cameras") + + outmodel = gradio.Model3D() + + # events + scenegraph_type.change(set_scenegraph_options, + inputs=[inputfiles, win_cyclic, refid, scenegraph_type], + outputs=[win_col, winsize, win_cyclic, refid]) + inputfiles.change(set_scenegraph_options, + inputs=[inputfiles, win_cyclic, refid, scenegraph_type], + outputs=[win_col, winsize, win_cyclic, refid]) + win_cyclic.change(set_scenegraph_options, + inputs=[inputfiles, win_cyclic, refid, scenegraph_type], + outputs=[win_col, winsize, win_cyclic, refid]) + run_btn.click(fn=recon_fun, + inputs=[scene, inputfiles, optim_level, lr1, niter1, lr2, niter2, min_conf_thr, matching_conf_thr, + as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size, + scenegraph_type, winsize, win_cyclic, refid, TSDF_thresh, shared_intrinsics], + outputs=[scene, outmodel]) + min_conf_thr.release(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh], + outputs=outmodel) + cam_size.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh], + outputs=outmodel) + TSDF_thresh.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh], + outputs=outmodel) + as_pointcloud.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh], + outputs=outmodel) + mask_sky.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh], + outputs=outmodel) + clean_depth.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh], + outputs=outmodel) + transparent_cams.change(model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh], + outputs=outmodel) + demo.launch(share=share, server_name=server_name, server_port=server_port) diff --git a/submodules/mast3r/mast3r/fast_nn.py b/submodules/mast3r/mast3r/fast_nn.py new file mode 100644 index 0000000000000000000000000000000000000000..05537f43c1be10b3733e80def8295c2ff5b5b8c0 --- /dev/null +++ b/submodules/mast3r/mast3r/fast_nn.py @@ -0,0 +1,223 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# MASt3R Fast Nearest Neighbor +# -------------------------------------------------------- +import torch +import numpy as np +import math +from scipy.spatial import KDTree + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.utils.device import to_numpy, todevice # noqa + + +@torch.no_grad() +def bruteforce_reciprocal_nns(A, B, device='cuda', block_size=None, dist='l2'): + if isinstance(A, np.ndarray): + A = torch.from_numpy(A).to(device) + if isinstance(B, np.ndarray): + B = torch.from_numpy(B).to(device) + + A = A.to(device) + B = B.to(device) + + if dist == 'l2': + dist_func = torch.cdist + argmin = torch.min + elif dist == 'dot': + def dist_func(A, B): + return A @ B.T + + def argmin(X, dim): + sim, nn = torch.max(X, dim=dim) + return sim.neg_(), nn + else: + raise ValueError(f'Unknown {dist=}') + + if block_size is None or len(A) * len(B) <= block_size**2: + dists = dist_func(A, B) + _, nn_A = argmin(dists, dim=1) + _, nn_B = argmin(dists, dim=0) + else: + dis_A = torch.full((A.shape[0],), float('inf'), device=device, dtype=A.dtype) + dis_B = torch.full((B.shape[0],), float('inf'), device=device, dtype=B.dtype) + nn_A = torch.full((A.shape[0],), -1, device=device, dtype=torch.int64) + nn_B = torch.full((B.shape[0],), -1, device=device, dtype=torch.int64) + number_of_iteration_A = math.ceil(A.shape[0] / block_size) + number_of_iteration_B = math.ceil(B.shape[0] / block_size) + + for i in range(number_of_iteration_A): + A_i = A[i * block_size:(i + 1) * block_size] + for j in range(number_of_iteration_B): + B_j = B[j * block_size:(j + 1) * block_size] + dists_blk = dist_func(A_i, B_j) # A, B, 1 + # dists_blk = dists[i * block_size:(i+1)*block_size, j * block_size:(j+1)*block_size] + min_A_i, argmin_A_i = argmin(dists_blk, dim=1) + min_B_j, argmin_B_j = argmin(dists_blk, dim=0) + + col_mask = min_A_i < dis_A[i * block_size:(i + 1) * block_size] + line_mask = min_B_j < dis_B[j * block_size:(j + 1) * block_size] + + dis_A[i * block_size:(i + 1) * block_size][col_mask] = min_A_i[col_mask] + dis_B[j * block_size:(j + 1) * block_size][line_mask] = min_B_j[line_mask] + + nn_A[i * block_size:(i + 1) * block_size][col_mask] = argmin_A_i[col_mask] + (j * block_size) + nn_B[j * block_size:(j + 1) * block_size][line_mask] = argmin_B_j[line_mask] + (i * block_size) + nn_A = nn_A.cpu().numpy() + nn_B = nn_B.cpu().numpy() + return nn_A, nn_B + + +class cdistMatcher: + def __init__(self, db_pts, device='cuda'): + self.db_pts = db_pts.to(device) + self.device = device + + def query(self, queries, k=1, **kw): + assert k == 1 + if queries.numel() == 0: + return None, [] + nnA, nnB = bruteforce_reciprocal_nns(queries, self.db_pts, device=self.device, **kw) + dis = None + return dis, nnA + + +def merge_corres(idx1, idx2, shape1=None, shape2=None, ret_xy=True, ret_index=False): + assert idx1.dtype == idx2.dtype == np.int32 + + # unique and sort along idx1 + corres = np.unique(np.c_[idx2, idx1].view(np.int64), return_index=ret_index) + if ret_index: + corres, indices = corres + xy2, xy1 = corres[:, None].view(np.int32).T + + if ret_xy: + assert shape1 and shape2 + xy1 = np.unravel_index(xy1, shape1) + xy2 = np.unravel_index(xy2, shape2) + if ret_xy != 'y_x': + xy1 = xy1[0].base[:, ::-1] + xy2 = xy2[0].base[:, ::-1] + + if ret_index: + return xy1, xy2, indices + return xy1, xy2 + + +def fast_reciprocal_NNs(pts1, pts2, subsample_or_initxy1=8, ret_xy=True, pixel_tol=0, ret_basin=False, + device='cuda', **matcher_kw): + H1, W1, DIM1 = pts1.shape + H2, W2, DIM2 = pts2.shape + assert DIM1 == DIM2 + + pts1 = pts1.reshape(-1, DIM1) + pts2 = pts2.reshape(-1, DIM2) + + if isinstance(subsample_or_initxy1, int) and pixel_tol == 0: + S = subsample_or_initxy1 + y1, x1 = np.mgrid[S // 2:H1:S, S // 2:W1:S].reshape(2, -1) + max_iter = 10 + else: + x1, y1 = subsample_or_initxy1 + if isinstance(x1, torch.Tensor): + x1 = x1.cpu().numpy() + if isinstance(y1, torch.Tensor): + y1 = y1.cpu().numpy() + max_iter = 1 + + xy1 = np.int32(np.unique(x1 + W1 * y1)) # make sure there's no doublons + xy2 = np.full_like(xy1, -1) + old_xy1 = xy1.copy() + old_xy2 = xy2.copy() + + if 'dist' in matcher_kw or 'block_size' in matcher_kw \ + or (isinstance(device, str) and device.startswith('cuda')) \ + or (isinstance(device, torch.device) and device.type.startswith('cuda')): + pts1 = pts1.to(device) + pts2 = pts2.to(device) + tree1 = cdistMatcher(pts1, device=device) + tree2 = cdistMatcher(pts2, device=device) + else: + pts1, pts2 = to_numpy((pts1, pts2)) + tree1 = KDTree(pts1) + tree2 = KDTree(pts2) + + notyet = np.ones(len(xy1), dtype=bool) + basin = np.full((H1 * W1 + 1,), -1, dtype=np.int32) if ret_basin else None + + niter = 0 + # n_notyet = [len(notyet)] + while notyet.any(): + _, xy2[notyet] = to_numpy(tree2.query(pts1[xy1[notyet]], **matcher_kw)) + if not ret_basin: + notyet &= (old_xy2 != xy2) # remove points that have converged + + _, xy1[notyet] = to_numpy(tree1.query(pts2[xy2[notyet]], **matcher_kw)) + if ret_basin: + basin[old_xy1[notyet]] = xy1[notyet] + notyet &= (old_xy1 != xy1) # remove points that have converged + + # n_notyet.append(notyet.sum()) + niter += 1 + if niter >= max_iter: + break + + old_xy2[:] = xy2 + old_xy1[:] = xy1 + + # print('notyet_stats:', ' '.join(map(str, (n_notyet+[0]*10)[:max_iter]))) + + if pixel_tol > 0: + # in case we only want to match some specific points + # and still have some way of checking reciprocity + old_yx1 = np.unravel_index(old_xy1, (H1, W1))[0].base + new_yx1 = np.unravel_index(xy1, (H1, W1))[0].base + dis = np.linalg.norm(old_yx1 - new_yx1, axis=-1) + converged = dis < pixel_tol + if not isinstance(subsample_or_initxy1, int): + xy1 = old_xy1 # replace new points by old ones + else: + converged = ~notyet # converged correspondences + + # keep only unique correspondences, and sort on xy1 + xy1, xy2 = merge_corres(xy1[converged], xy2[converged], (H1, W1), (H2, W2), ret_xy=ret_xy) + if ret_basin: + return xy1, xy2, basin + return xy1, xy2 + + +def extract_correspondences_nonsym(A, B, confA, confB, subsample=8, device=None, ptmap_key='pred_desc', pixel_tol=0): + if '3d' in ptmap_key: + opt = dict(device='cpu', workers=32) + else: + opt = dict(device=device, dist='dot', block_size=2**13) + + # matching the two pairs + idx1 = [] + idx2 = [] + # merge corres from opposite pairs + HA, WA = A.shape[:2] + HB, WB = B.shape[:2] + if pixel_tol == 0: + nn1to2 = fast_reciprocal_NNs(A, B, subsample_or_initxy1=subsample, ret_xy=False, **opt) + nn2to1 = fast_reciprocal_NNs(B, A, subsample_or_initxy1=subsample, ret_xy=False, **opt) + else: + S = subsample + yA, xA = np.mgrid[S // 2:HA:S, S // 2:WA:S].reshape(2, -1) + yB, xB = np.mgrid[S // 2:HB:S, S // 2:WB:S].reshape(2, -1) + + nn1to2 = fast_reciprocal_NNs(A, B, subsample_or_initxy1=(xA, yA), ret_xy=False, pixel_tol=pixel_tol, **opt) + nn2to1 = fast_reciprocal_NNs(B, A, subsample_or_initxy1=(xB, yB), ret_xy=False, pixel_tol=pixel_tol, **opt) + + idx1 = np.r_[nn1to2[0], nn2to1[1]] + idx2 = np.r_[nn1to2[1], nn2to1[0]] + + c1 = confA.ravel()[idx1] + c2 = confB.ravel()[idx2] + + xy1, xy2, idx = merge_corres(idx1, idx2, (HA, WA), (HB, WB), ret_xy=True, ret_index=True) + conf = np.minimum(c1[idx], c2[idx]) + corres = (xy1.copy(), xy2.copy(), conf) + return todevice(corres, device) diff --git a/submodules/mast3r/mast3r/losses.py b/submodules/mast3r/mast3r/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..3a50f57481e436d7752dcbf2b414be3ea65ee76b --- /dev/null +++ b/submodules/mast3r/mast3r/losses.py @@ -0,0 +1,508 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Implementation of MASt3R training losses +# -------------------------------------------------------- +import torch +import torch.nn as nn +import numpy as np +from sklearn.metrics import average_precision_score + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.losses import BaseCriterion, Criterion, MultiLoss, Sum, ConfLoss +from dust3r.losses import Regr3D as Regr3D_dust3r +from dust3r.utils.geometry import (geotrf, inv, normalize_pointcloud) +from dust3r.inference import get_pred_pts3d +from dust3r.utils.geometry import get_joint_pointcloud_depth, get_joint_pointcloud_center_scale + + +def apply_log_to_norm(xyz): + d = xyz.norm(dim=-1, keepdim=True) + xyz = xyz / d.clip(min=1e-8) + xyz = xyz * torch.log1p(d) + return xyz + + +class Regr3D (Regr3D_dust3r): + def __init__(self, criterion, norm_mode='avg_dis', gt_scale=False, opt_fit_gt=False, + sky_loss_value=2, max_metric_scale=False, loss_in_log=False): + self.loss_in_log = loss_in_log + if norm_mode.startswith('?'): + # do no norm pts from metric scale datasets + self.norm_all = False + self.norm_mode = norm_mode[1:] + else: + self.norm_all = True + self.norm_mode = norm_mode + super().__init__(criterion, self.norm_mode, gt_scale) + + self.sky_loss_value = sky_loss_value + self.max_metric_scale = max_metric_scale + + def get_all_pts3d(self, gt1, gt2, pred1, pred2, dist_clip=None): + # everything is normalized w.r.t. camera of view1 + in_camera1 = inv(gt1['camera_pose']) + gt_pts1 = geotrf(in_camera1, gt1['pts3d']) # B,H,W,3 + gt_pts2 = geotrf(in_camera1, gt2['pts3d']) # B,H,W,3 + + valid1 = gt1['valid_mask'].clone() + valid2 = gt2['valid_mask'].clone() + + if dist_clip is not None: + # points that are too far-away == invalid + dis1 = gt_pts1.norm(dim=-1) # (B, H, W) + dis2 = gt_pts2.norm(dim=-1) # (B, H, W) + valid1 = valid1 & (dis1 <= dist_clip) + valid2 = valid2 & (dis2 <= dist_clip) + + if self.loss_in_log == 'before': + # this only make sense when depth_mode == 'linear' + gt_pts1 = apply_log_to_norm(gt_pts1) + gt_pts2 = apply_log_to_norm(gt_pts2) + + pr_pts1 = get_pred_pts3d(gt1, pred1, use_pose=False).clone() + pr_pts2 = get_pred_pts3d(gt2, pred2, use_pose=True).clone() + + if not self.norm_all: + if self.max_metric_scale: + B = valid1.shape[0] + # valid1: B, H, W + # torch.linalg.norm(gt_pts1, dim=-1) -> B, H, W + # dist1_to_cam1 -> reshape to B, H*W + dist1_to_cam1 = torch.where(valid1, torch.linalg.norm(gt_pts1, dim=-1), 0).view(B, -1) + dist2_to_cam1 = torch.where(valid2, torch.linalg.norm(gt_pts2, dim=-1), 0).view(B, -1) + + # is_metric_scale: B + # dist1_to_cam1.max(dim=-1).values -> B + gt1['is_metric_scale'] = gt1['is_metric_scale'] \ + & (dist1_to_cam1.max(dim=-1).values < self.max_metric_scale) \ + & (dist2_to_cam1.max(dim=-1).values < self.max_metric_scale) + gt2['is_metric_scale'] = gt1['is_metric_scale'] + + mask = ~gt1['is_metric_scale'] + else: + mask = torch.ones_like(gt1['is_metric_scale']) + # normalize 3d points + if self.norm_mode and mask.any(): + pr_pts1[mask], pr_pts2[mask] = normalize_pointcloud(pr_pts1[mask], pr_pts2[mask], self.norm_mode, + valid1[mask], valid2[mask]) + + if self.norm_mode and not self.gt_scale: + gt_pts1, gt_pts2, norm_factor = normalize_pointcloud(gt_pts1, gt_pts2, self.norm_mode, + valid1, valid2, ret_factor=True) + # apply the same normalization to prediction + pr_pts1[~mask] = pr_pts1[~mask] / norm_factor[~mask] + pr_pts2[~mask] = pr_pts2[~mask] / norm_factor[~mask] + + # return sky segmentation, making sure they don't include any labelled 3d points + sky1 = gt1['sky_mask'] & (~valid1) + sky2 = gt2['sky_mask'] & (~valid2) + return gt_pts1, gt_pts2, pr_pts1, pr_pts2, valid1, valid2, sky1, sky2, {} + + def compute_loss(self, gt1, gt2, pred1, pred2, **kw): + gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, sky1, sky2, monitoring = \ + self.get_all_pts3d(gt1, gt2, pred1, pred2, **kw) + + if self.sky_loss_value > 0: + assert self.criterion.reduction == 'none', 'sky_loss_value should be 0 if no conf loss' + # add the sky pixel as "valid" pixels... + mask1 = mask1 | sky1 + mask2 = mask2 | sky2 + + # loss on img1 side + pred_pts1 = pred_pts1[mask1] + gt_pts1 = gt_pts1[mask1] + if self.loss_in_log and self.loss_in_log != 'before': + # this only make sense when depth_mode == 'exp' + pred_pts1 = apply_log_to_norm(pred_pts1) + gt_pts1 = apply_log_to_norm(gt_pts1) + l1 = self.criterion(pred_pts1, gt_pts1) + + # loss on gt2 side + pred_pts2 = pred_pts2[mask2] + gt_pts2 = gt_pts2[mask2] + if self.loss_in_log and self.loss_in_log != 'before': + pred_pts2 = apply_log_to_norm(pred_pts2) + gt_pts2 = apply_log_to_norm(gt_pts2) + l2 = self.criterion(pred_pts2, gt_pts2) + + if self.sky_loss_value > 0: + assert self.criterion.reduction == 'none', 'sky_loss_value should be 0 if no conf loss' + # ... but force the loss to be high there + l1 = torch.where(sky1[mask1], self.sky_loss_value, l1) + l2 = torch.where(sky2[mask2], self.sky_loss_value, l2) + self_name = type(self).__name__ + details = {self_name + '_pts3d_1': float(l1.mean()), self_name + '_pts3d_2': float(l2.mean())} + return Sum((l1, mask1), (l2, mask2)), (details | monitoring) + + +class Regr3D_ShiftInv (Regr3D): + """ Same than Regr3D but invariant to depth shift. + """ + + def get_all_pts3d(self, gt1, gt2, pred1, pred2): + # compute unnormalized points + gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, sky1, sky2, monitoring = \ + super().get_all_pts3d(gt1, gt2, pred1, pred2) + + # compute median depth + gt_z1, gt_z2 = gt_pts1[..., 2], gt_pts2[..., 2] + pred_z1, pred_z2 = pred_pts1[..., 2], pred_pts2[..., 2] + gt_shift_z = get_joint_pointcloud_depth(gt_z1, gt_z2, mask1, mask2)[:, None, None] + pred_shift_z = get_joint_pointcloud_depth(pred_z1, pred_z2, mask1, mask2)[:, None, None] + + # subtract the median depth + gt_z1 -= gt_shift_z + gt_z2 -= gt_shift_z + pred_z1 -= pred_shift_z + pred_z2 -= pred_shift_z + + # monitoring = dict(monitoring, gt_shift_z=gt_shift_z.mean().detach(), pred_shift_z=pred_shift_z.mean().detach()) + return gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, sky1, sky2, monitoring + + +class Regr3D_ScaleInv (Regr3D): + """ Same than Regr3D but invariant to depth scale. + if gt_scale == True: enforce the prediction to take the same scale than GT + """ + + def get_all_pts3d(self, gt1, gt2, pred1, pred2): + # compute depth-normalized points + gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, sky1, sky2, monitoring = \ + super().get_all_pts3d(gt1, gt2, pred1, pred2) + + # measure scene scale + _, gt_scale = get_joint_pointcloud_center_scale(gt_pts1, gt_pts2, mask1, mask2) + _, pred_scale = get_joint_pointcloud_center_scale(pred_pts1, pred_pts2, mask1, mask2) + + # prevent predictions to be in a ridiculous range + pred_scale = pred_scale.clip(min=1e-3, max=1e3) + + # subtract the median depth + if self.gt_scale: + pred_pts1 *= gt_scale / pred_scale + pred_pts2 *= gt_scale / pred_scale + # monitoring = dict(monitoring, pred_scale=(pred_scale/gt_scale).mean()) + else: + gt_pts1 /= gt_scale + gt_pts2 /= gt_scale + pred_pts1 /= pred_scale + pred_pts2 /= pred_scale + # monitoring = dict(monitoring, gt_scale=gt_scale.mean(), pred_scale=pred_scale.mean().detach()) + + return gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, sky1, sky2, monitoring + + +class Regr3D_ScaleShiftInv (Regr3D_ScaleInv, Regr3D_ShiftInv): + # calls Regr3D_ShiftInv first, then Regr3D_ScaleInv + pass + + +def get_similarities(desc1, desc2, euc=False): + if euc: # euclidean distance in same range than similarities + dists = (desc1[:, :, None] - desc2[:, None]).norm(dim=-1) + sim = 1 / (1 + dists) + else: + # Compute similarities + sim = desc1 @ desc2.transpose(-2, -1) + return sim + + +class MatchingCriterion(BaseCriterion): + def __init__(self, reduction='mean', fp=torch.float32): + super().__init__(reduction) + self.fp = fp + + def forward(self, a, b, valid_matches=None, euc=False): + assert a.ndim >= 2 and 1 <= a.shape[-1], f'Bad shape = {a.shape}' + dist = self.loss(a.to(self.fp), b.to(self.fp), valid_matches, euc=euc) + # one dimension less or reduction to single value + assert (valid_matches is None and dist.ndim == a.ndim - + 1) or self.reduction in ['mean', 'sum', '1-mean', 'none'] + if self.reduction == 'none': + return dist + if self.reduction == 'sum': + return dist.sum() + if self.reduction == 'mean': + return dist.mean() if dist.numel() > 0 else dist.new_zeros(()) + if self.reduction == '1-mean': + return 1. - dist.mean() if dist.numel() > 0 else dist.new_ones(()) + raise ValueError(f'bad {self.reduction=} mode') + + def loss(self, a, b, valid_matches=None): + raise NotImplementedError + + +class InfoNCE(MatchingCriterion): + def __init__(self, temperature=0.07, eps=1e-8, mode='all', **kwargs): + super().__init__(**kwargs) + self.temperature = temperature + self.eps = eps + assert mode in ['all', 'proper', 'dual'] + self.mode = mode + + def loss(self, desc1, desc2, valid_matches=None, euc=False): + # valid positives are along diagonals + B, N, D = desc1.shape + B2, N2, D2 = desc2.shape + assert B == B2 and D == D2 + if valid_matches is None: + valid_matches = torch.ones([B, N], dtype=bool) + # torch.all(valid_matches.sum(dim=-1) > 0) some pairs have no matches???? + assert valid_matches.shape == torch.Size([B, N]) and valid_matches.sum() > 0 + + # Tempered similarities + sim = get_similarities(desc1, desc2, euc) / self.temperature + sim[sim.isnan()] = -torch.inf # ignore nans + # Softmax of positives with temperature + sim = sim.exp_() # save peak memory + positives = sim.diagonal(dim1=-2, dim2=-1) + + # Loss + if self.mode == 'all': # Previous InfoNCE + loss = -torch.log((positives / sim.sum(dim=-1).sum(dim=-1, keepdim=True)).clip(self.eps)) + elif self.mode == 'proper': # Proper InfoNCE + loss = -(torch.log((positives / sim.sum(dim=-2)).clip(self.eps)) + + torch.log((positives / sim.sum(dim=-1)).clip(self.eps))) + elif self.mode == 'dual': # Dual Softmax + loss = -(torch.log((positives**2 / sim.sum(dim=-1) / sim.sum(dim=-2)).clip(self.eps))) + else: + raise ValueError("This should not happen...") + return loss[valid_matches] + + +class APLoss (MatchingCriterion): + """ AP loss. + """ + + def __init__(self, nq='torch', min=0, max=1, euc=False, **kw): + super().__init__(**kw) + # Exact/True AP loss (not differentiable) + if nq == 0: + nq = 'sklearn' # special case + try: + self.compute_AP = eval('self.compute_true_AP_' + nq) + except: + raise ValueError("Unknown mode %s for AP loss" % nq) + + @staticmethod + def compute_true_AP_sklearn(scores, labels): + def compute_AP(label, score): + return average_precision_score(label, score) + + aps = scores.new_zeros((scores.shape[0], scores.shape[1])) + label_np = labels.cpu().numpy().astype(bool) + scores_np = scores.cpu().numpy() + for bi in range(scores_np.shape[0]): + for i in range(scores_np.shape[1]): + labels = label_np[bi, i, :] + if labels.sum() < 1: + continue + aps[bi, i] = compute_AP(labels, scores_np[bi, i, :]) + return aps + + @staticmethod + def compute_true_AP_torch(scores, labels): + assert scores.shape == labels.shape + B, N, M = labels.shape + dev = labels.device + with torch.no_grad(): + # sort scores + _, order = scores.sort(dim=-1, descending=True) + # sort labels accordingly + labels = labels[torch.arange(B, device=dev)[:, None, None].expand(order.shape), + torch.arange(N, device=dev)[None, :, None].expand(order.shape), + order] + # compute number of positives per query + npos = labels.sum(dim=-1) + assert torch.all(torch.isclose(npos, npos[0, 0]) + ), "only implemented for constant number of positives per query" + npos = int(npos[0, 0]) + # compute precision at each recall point + posrank = labels.nonzero()[:, -1].view(B, N, npos) + recall = torch.arange(1, 1 + npos, dtype=torch.float32, device=dev)[None, None, :].expand(B, N, npos) + precision = recall / (1 + posrank).float() + # average precision values at all recall points + aps = precision.mean(dim=-1) + + return aps + + def loss(self, desc1, desc2, valid_matches=None, euc=False): # if matches is None, positives are the diagonal + B, N1, D = desc1.shape + B2, N2, D2 = desc2.shape + assert B == B2 and D == D2 + + scores = get_similarities(desc1, desc2, euc) + + labels = torch.zeros([B, N1, N2], dtype=scores.dtype, device=scores.device) + + # allow all diagonal positives and only mask afterwards + labels.diagonal(dim1=-2, dim2=-1)[...] = 1. + apscore = self.compute_AP(scores, labels) + if valid_matches is not None: + apscore = apscore[valid_matches] + return apscore + + +class MatchingLoss (Criterion, MultiLoss): + """ + Matching loss per image + only compare pixels inside an image but not in the whole batch as what would be done usually + """ + + def __init__(self, criterion, withconf=False, use_pts3d=False, negatives_padding=0, blocksize=4096): + super().__init__(criterion) + self.negatives_padding = negatives_padding + self.use_pts3d = use_pts3d + self.blocksize = blocksize + self.withconf = withconf + + def add_negatives(self, outdesc2, desc2, batchid, x2, y2): + if self.negatives_padding: + B, H, W, D = desc2.shape + negatives = torch.ones([B, H, W], device=desc2.device, dtype=bool) + negatives[batchid, y2, x2] = False + sel = negatives & (negatives.view([B, -1]).cumsum(dim=-1).view(B, H, W) + <= self.negatives_padding) # take the N-first negatives + outdesc2 = torch.cat([outdesc2, desc2[sel].view([B, -1, D])], dim=1) + return outdesc2 + + def get_confs(self, pred1, pred2, sel1, sel2): + if self.withconf: + if self.use_pts3d: + outconfs1 = pred1['conf'][sel1] + outconfs2 = pred2['conf'][sel2] + else: + outconfs1 = pred1['desc_conf'][sel1] + outconfs2 = pred2['desc_conf'][sel2] + else: + outconfs1 = outconfs2 = None + return outconfs1, outconfs2 + + def get_descs(self, pred1, pred2): + if self.use_pts3d: + desc1, desc2 = pred1['pts3d'], pred2['pts3d_in_other_view'] + else: + desc1, desc2 = pred1['desc'], pred2['desc'] + return desc1, desc2 + + def get_matching_descs(self, gt1, gt2, pred1, pred2, **kw): + outdesc1 = outdesc2 = outconfs1 = outconfs2 = None + # Recover descs, GT corres and valid mask + desc1, desc2 = self.get_descs(pred1, pred2) + + (x1, y1), (x2, y2) = gt1['corres'].unbind(-1), gt2['corres'].unbind(-1) + valid_matches = gt1['valid_corres'] + + # Select descs that have GT matches + B, N = x1.shape + batchid = torch.arange(B)[:, None].repeat(1, N) # B, N + outdesc1, outdesc2 = desc1[batchid, y1, x1], desc2[batchid, y2, x2] # B, N, D + + # Padd with unused negatives + outdesc2 = self.add_negatives(outdesc2, desc2, batchid, x2, y2) + + # Gather confs if needed + sel1 = batchid, y1, x1 + sel2 = batchid, y2, x2 + outconfs1, outconfs2 = self.get_confs(pred1, pred2, sel1, sel2) + + return outdesc1, outdesc2, outconfs1, outconfs2, valid_matches, {'use_euclidean_dist': self.use_pts3d} + + def blockwise_criterion(self, descs1, descs2, confs1, confs2, valid_matches, euc, rng=np.random, shuffle=True): + loss = None + details = {} + B, N, D = descs1.shape + + if N <= self.blocksize: # Blocks are larger than provided descs, compute regular loss + loss = self.criterion(descs1, descs2, valid_matches, euc=euc) + else: # Compute criterion on the blockdiagonal only, after shuffling + # Shuffle if necessary + matches_perm = slice(None) + if shuffle: + matches_perm = np.stack([rng.choice(range(N), size=N, replace=False) for _ in range(B)]) + batchid = torch.tile(torch.arange(B), (N, 1)).T + matches_perm = batchid, matches_perm + + descs1 = descs1[matches_perm] + descs2 = descs2[matches_perm] + valid_matches = valid_matches[matches_perm] + + assert N % self.blocksize == 0, "Error, can't chunk block-diagonal, please check blocksize" + n_chunks = N // self.blocksize + descs1 = descs1.reshape([B * n_chunks, self.blocksize, D]) # [B*(N//blocksize), blocksize, D] + descs2 = descs2.reshape([B * n_chunks, self.blocksize, D]) # [B*(N//blocksize), blocksize, D] + valid_matches = valid_matches.view([B * n_chunks, self.blocksize]) + loss = self.criterion(descs1, descs2, valid_matches, euc=euc) + if self.withconf: + confs1, confs2 = map(lambda x: x[matches_perm], (confs1, confs2)) # apply perm to confidences if needed + + if self.withconf: + # split confidences between positives/negatives for loss computation + details['conf_pos'] = map(lambda x: x[valid_matches.view(B, -1)], (confs1, confs2)) + details['conf_neg'] = map(lambda x: x[~valid_matches.view(B, -1)], (confs1, confs2)) + details['Conf1_std'] = confs1.std() + details['Conf2_std'] = confs2.std() + + return loss, details + + def compute_loss(self, gt1, gt2, pred1, pred2, **kw): + # Gather preds and GT + descs1, descs2, confs1, confs2, valid_matches, monitoring = self.get_matching_descs( + gt1, gt2, pred1, pred2, **kw) + + # loss on matches + loss, details = self.blockwise_criterion(descs1, descs2, confs1, confs2, + valid_matches, euc=monitoring.pop('use_euclidean_dist', False)) + + details[type(self).__name__] = float(loss.mean()) + return loss, (details | monitoring) + + +class ConfMatchingLoss(ConfLoss): + """ Weight matching by learned confidence. Same as ConfLoss but for a matching criterion + Assuming the input matching_loss is a match-level loss. + """ + + def __init__(self, pixel_loss, alpha=1., confmode='prod', neg_conf_loss_quantile=False): + super().__init__(pixel_loss, alpha) + self.pixel_loss.withconf = True + self.confmode = confmode + self.neg_conf_loss_quantile = neg_conf_loss_quantile + + def aggregate_confs(self, confs1, confs2): # get the confidences resulting from the two view predictions + if self.confmode == 'prod': + confs = confs1 * confs2 if confs1 is not None and confs2 is not None else 1. + elif self.confmode == 'mean': + confs = .5 * (confs1 + confs2) if confs1 is not None and confs2 is not None else 1. + else: + raise ValueError(f"Unknown conf mode {self.confmode}") + return confs + + def compute_loss(self, gt1, gt2, pred1, pred2, **kw): + # compute per-pixel loss + loss, details = self.pixel_loss(gt1, gt2, pred1, pred2, **kw) + # Recover confidences for positive and negative samples + conf1_pos, conf2_pos = details.pop('conf_pos') + conf1_neg, conf2_neg = details.pop('conf_neg') + conf_pos = self.aggregate_confs(conf1_pos, conf2_pos) + + # weight Matching loss by confidence on positives + conf_pos, log_conf_pos = self.get_conf_log(conf_pos) + conf_loss = loss * conf_pos - self.alpha * log_conf_pos + # average + nan protection (in case of no valid pixels at all) + conf_loss = conf_loss.mean() if conf_loss.numel() > 0 else 0 + # Add negative confs loss to give some supervision signal to confidences for pixels that are not matched in GT + if self.neg_conf_loss_quantile: + conf_neg = torch.cat([conf1_neg, conf2_neg]) + conf_neg, log_conf_neg = self.get_conf_log(conf_neg) + + # recover quantile that will be used for negatives loss value assignment + neg_loss_value = torch.quantile(loss, self.neg_conf_loss_quantile).detach() + neg_loss = neg_loss_value * conf_neg - self.alpha * log_conf_neg + + neg_loss = neg_loss.mean() if neg_loss.numel() > 0 else 0 + conf_loss = conf_loss + neg_loss + + return conf_loss, dict(matching_conf_loss=float(conf_loss), **details) diff --git a/submodules/mast3r/mast3r/model.py b/submodules/mast3r/mast3r/model.py new file mode 100644 index 0000000000000000000000000000000000000000..f328c5e43b8e98f2ec960e4d25e6f235ac543544 --- /dev/null +++ b/submodules/mast3r/mast3r/model.py @@ -0,0 +1,68 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# MASt3R model class +# -------------------------------------------------------- +import torch +import torch.nn.functional as F +import os + +from mast3r.catmlp_dpt_head import mast3r_head_factory + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.model import AsymmetricCroCo3DStereo # noqa +from dust3r.utils.misc import transpose_to_landscape # noqa + + +inf = float('inf') + + +def load_model(model_path, device, verbose=True): + if verbose: + print('... loading model from', model_path) + ckpt = torch.load(model_path, map_location='cpu') + args = ckpt['args'].model.replace("ManyAR_PatchEmbed", "PatchEmbedDust3R") + if 'landscape_only' not in args: + args = args[:-1] + ', landscape_only=False)' + else: + args = args.replace(" ", "").replace('landscape_only=True', 'landscape_only=False') + assert "landscape_only=False" in args + if verbose: + print(f"instantiating : {args}") + net = eval(args) + s = net.load_state_dict(ckpt['model'], strict=False) + if verbose: + print(s) + return net.to(device) + + +class AsymmetricMASt3R(AsymmetricCroCo3DStereo): + def __init__(self, desc_mode=('norm'), two_confs=False, desc_conf_mode=None, **kwargs): + self.desc_mode = desc_mode + self.two_confs = two_confs + self.desc_conf_mode = desc_conf_mode + super().__init__(**kwargs) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, **kw): + if os.path.isfile(pretrained_model_name_or_path): + return load_model(pretrained_model_name_or_path, device='cpu') + else: + return super(AsymmetricMASt3R, cls).from_pretrained(pretrained_model_name_or_path, **kw) + + def set_downstream_head(self, output_mode, head_type, landscape_only, depth_mode, conf_mode, patch_size, img_size, **kw): + assert img_size[0] % patch_size == 0 and img_size[ + 1] % patch_size == 0, f'{img_size=} must be multiple of {patch_size=}' + self.output_mode = output_mode + self.head_type = head_type + self.depth_mode = depth_mode + self.conf_mode = conf_mode + if self.desc_conf_mode is None: + self.desc_conf_mode = conf_mode + # allocate heads + self.downstream_head1 = mast3r_head_factory(head_type, output_mode, self, has_conf=bool(conf_mode)) + self.downstream_head2 = mast3r_head_factory(head_type, output_mode, self, has_conf=bool(conf_mode)) + # magic wrapper + self.head1 = transpose_to_landscape(self.downstream_head1, activate=landscape_only) + self.head2 = transpose_to_landscape(self.downstream_head2, activate=landscape_only) diff --git a/submodules/mast3r/mast3r/utils/__init__.py b/submodules/mast3r/mast3r/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7dd877d649ce4dbd749dd7195a8b34c0f91d4f0 --- /dev/null +++ b/submodules/mast3r/mast3r/utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). \ No newline at end of file diff --git a/submodules/mast3r/mast3r/utils/coarse_to_fine.py b/submodules/mast3r/mast3r/utils/coarse_to_fine.py new file mode 100644 index 0000000000000000000000000000000000000000..c062e8608f82c608f2d605d69a95a7e0f301b3cf --- /dev/null +++ b/submodules/mast3r/mast3r/utils/coarse_to_fine.py @@ -0,0 +1,214 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# coarse to fine utilities +# -------------------------------------------------------- +import numpy as np + + +def crop_tag(cell): + return f'[{cell[1]}:{cell[3]},{cell[0]}:{cell[2]}]' + + +def crop_slice(cell): + return slice(cell[1], cell[3]), slice(cell[0], cell[2]) + + +def _start_pos(total_size, win_size, overlap): + # we must have AT LEAST overlap between segments + # first segment starts at 0, last segment starts at total_size-win_size + assert 0 <= overlap < 1 + assert total_size >= win_size + spacing = win_size * (1 - overlap) + last_pt = total_size - win_size + n_windows = 2 + int((last_pt - 1) // spacing) + return np.linspace(0, last_pt, n_windows).round().astype(int) + + +def multiple_of_16(x): + return (x // 16) * 16 + + +def _make_overlapping_grid(H, W, size, overlap): + H_win = multiple_of_16(H * size // max(H, W)) + W_win = multiple_of_16(W * size // max(H, W)) + x = _start_pos(W, W_win, overlap) + y = _start_pos(H, H_win, overlap) + grid = np.stack(np.meshgrid(x, y, indexing='xy'), axis=-1) + grid = np.concatenate((grid, grid + (W_win, H_win)), axis=-1) + return grid.reshape(-1, 4) + + +def _cell_size(cell2): + width, height = cell2[:, 2] - cell2[:, 0], cell2[:, 3] - cell2[:, 1] + assert width.min() >= 0 + assert height.min() >= 0 + return width, height + + +def _norm_windows(cell2, H2, W2, forced_resolution=None): + # make sure the window aspect ratio is 3/4, or the output resolution is forced_resolution if defined + outcell = cell2.copy() + width, height = _cell_size(cell2) + width2, height2 = width.clip(max=W2), height.clip(max=H2) + if forced_resolution is None: + width2[width < height] = (height2[width < height] * 3.01 / 4).clip(max=W2) + height2[width >= height] = (width2[width >= height] * 3.01 / 4).clip(max=H2) + else: + forced_H, forced_W = forced_resolution + width2[:] = forced_W + height2[:] = forced_H + + half = (width2 - width) / 2 + outcell[:, 0] -= half + outcell[:, 2] += half + half = (height2 - height) / 2 + outcell[:, 1] -= half + outcell[:, 3] += half + + # proj to integers + outcell = np.floor(outcell).astype(int) + # Take care of flooring errors + tmpw, tmph = _cell_size(outcell) + outcell[:, 0] += tmpw.astype(tmpw.dtype) - width2.astype(tmpw.dtype) + outcell[:, 1] += tmph.astype(tmpw.dtype) - height2.astype(tmpw.dtype) + + # make sure 0 <= x < W2 and 0 <= y < H2 + outcell[:, 0::2] -= outcell[:, [0]].clip(max=0) + outcell[:, 1::2] -= outcell[:, [1]].clip(max=0) + outcell[:, 0::2] -= outcell[:, [2]].clip(min=W2) - W2 + outcell[:, 1::2] -= outcell[:, [3]].clip(min=H2) - H2 + + width, height = _cell_size(outcell) + assert np.all(width == width2.astype(width.dtype)) and np.all( + height == height2.astype(height.dtype)), "Error, output is not of the expected shape." + assert np.all(width <= W2) + assert np.all(height <= H2) + return outcell + + +def _weight_pixels(cell, pix, assigned, gauss_var=2): + center = cell.reshape(-1, 2, 2).mean(axis=1) + width, height = _cell_size(cell) + + # square distance between each cell center and each point + dist = (center[:, None] - pix[None]) / np.c_[width, height][:, None] + dist2 = np.square(dist).sum(axis=-1) + + assert assigned.shape == dist2.shape + res = np.where(assigned, np.exp(-gauss_var * dist2), 0) + return res + + +def pos2d_in_rect(p1, cell1): + x, y = p1.T + l, t, r, b = cell1 + assigned = (l <= x) & (x < r) & (t <= y) & (y < b) + return assigned + + +def _score_cell(cell1, H2, W2, p1, p2, min_corres=10, forced_resolution=None): + assert p1.shape == p2.shape + + # compute keypoint assignment + assigned = pos2d_in_rect(p1, cell1[None].T) + assert assigned.shape == (len(cell1), len(p1)) + + # remove cells without correspondences + valid_cells = assigned.sum(axis=1) >= min_corres + cell1 = cell1[valid_cells] + assigned = assigned[valid_cells] + if not valid_cells.any(): + return cell1, cell1, assigned + + # fill-in the assigned points in both image + assigned_p1 = np.empty((len(cell1), len(p1), 2), dtype=np.float32) + assigned_p2 = np.empty((len(cell1), len(p2), 2), dtype=np.float32) + assigned_p1[:] = p1[None] + assigned_p2[:] = p2[None] + assigned_p1[~assigned] = np.nan + assigned_p2[~assigned] = np.nan + + # find the median center and scale of assigned points in each cell + # cell_center1 = np.nanmean(assigned_p1, axis=1) + cell_center2 = np.nanmean(assigned_p2, axis=1) + im1_q25, im1_q75 = np.nanquantile(assigned_p1, (0.1, 0.9), axis=1) + im2_q25, im2_q75 = np.nanquantile(assigned_p2, (0.1, 0.9), axis=1) + + robust_std1 = (im1_q75 - im1_q25).clip(20.) + robust_std2 = (im2_q75 - im2_q25).clip(20.) + + cell_size1 = (cell1[:, 2:4] - cell1[:, 0:2]) + cell_size2 = cell_size1 * robust_std2 / robust_std1 + cell2 = np.c_[cell_center2 - cell_size2 / 2, cell_center2 + cell_size2 / 2] + + # make sure cell bounds are valid + cell2 = _norm_windows(cell2, H2, W2, forced_resolution=forced_resolution) + + # compute correspondence weights + corres_weights = _weight_pixels(cell1, p1, assigned) * _weight_pixels(cell2, p2, assigned) + + # return a list of window pairs and assigned correspondences + return cell1, cell2, corres_weights + + +def greedy_selection(corres_weights, target=0.9): + # corres_weight = (n_cell_pair, n_corres) matrix. + # If corres_weight[c,p]>0, means that correspondence p is visible in cell pair p + assert 0 < target <= 1 + corres_weights = corres_weights.copy() + + total = corres_weights.max(axis=0).sum() + target *= total + + # init = empty + res = [] + cur = np.zeros(corres_weights.shape[1]) # current selection + + while cur.sum() < target: + # pick the nex best cell pair + best = corres_weights.sum(axis=1).argmax() + res.append(best) + + # update current + cur += corres_weights[best] + # print('appending', best, 'with score', corres_weights[best].sum(), '-->', cur.sum()) + + # remove from all other views + corres_weights = (corres_weights - corres_weights[best]).clip(min=0) + + return res + + +def select_pairs_of_crops(img_q, img_b, pos2d_in_query, pos2d_in_ref, maxdim=512, overlap=.5, forced_resolution=None): + # prepare the overlapping cells + grid_q = _make_overlapping_grid(*img_q.shape[:2], maxdim, overlap) + grid_b = _make_overlapping_grid(*img_b.shape[:2], maxdim, overlap) + + assert forced_resolution is None or len(forced_resolution) == 2 + if isinstance(forced_resolution[0], int) or not len(forced_resolution[0]) == 2: + forced_resolution1 = forced_resolution2 = forced_resolution + else: + assert len(forced_resolution[1]) == 2 + forced_resolution1 = forced_resolution[0] + forced_resolution2 = forced_resolution[1] + + # Make sure crops respect constraints + grid_q = _norm_windows(grid_q.astype(float), *img_q.shape[:2], forced_resolution=forced_resolution1) + grid_b = _norm_windows(grid_b.astype(float), *img_b.shape[:2], forced_resolution=forced_resolution2) + + # score cells + pairs_q = _score_cell(grid_q, *img_b.shape[:2], pos2d_in_query, pos2d_in_ref, forced_resolution=forced_resolution2) + pairs_b = _score_cell(grid_b, *img_q.shape[:2], pos2d_in_ref, pos2d_in_query, forced_resolution=forced_resolution1) + pairs_b = pairs_b[1], pairs_b[0], pairs_b[2] # cellq, cellb, corres_weights + + # greedy selection until all correspondences are generated + cell1, cell2, corres_weights = map(np.concatenate, zip(pairs_q, pairs_b)) + if len(corres_weights) == 0: + return # tolerated for empty generators + order = greedy_selection(corres_weights, target=0.9) + + for i in order: + def pair_tag(qi, bi): return (str(qi) + crop_tag(cell1[i]), str(bi) + crop_tag(cell2[i])) + yield cell1[i], cell2[i], pair_tag diff --git a/submodules/mast3r/mast3r/utils/collate.py b/submodules/mast3r/mast3r/utils/collate.py new file mode 100644 index 0000000000000000000000000000000000000000..72ee3a437b87ef7049dcd03b93e594a8325b780c --- /dev/null +++ b/submodules/mast3r/mast3r/utils/collate.py @@ -0,0 +1,62 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Collate extensions +# -------------------------------------------------------- + +import torch +import collections +from torch.utils.data._utils.collate import default_collate_fn_map, default_collate_err_msg_format +from typing import Callable, Dict, Optional, Tuple, Type, Union, List + + +def cat_collate_tensor_fn(batch, *, collate_fn_map): + return torch.cat(batch, dim=0) + + +def cat_collate_list_fn(batch, *, collate_fn_map: Optional[Dict[Union[Type, Tuple[Type, ...]], Callable]] = None): + return [item for bb in batch for item in bb] # concatenate all lists + + +cat_collate_fn_map = default_collate_fn_map.copy() +cat_collate_fn_map[torch.Tensor] = cat_collate_tensor_fn +cat_collate_fn_map[List] = cat_collate_list_fn +cat_collate_fn_map[type(None)] = lambda _, **kw: None # When some Nones, simply return a single None + + +def cat_collate(batch, *, collate_fn_map: Optional[Dict[Union[Type, Tuple[Type, ...]], Callable]] = None): + r"""Custom collate function that concatenates stuff instead of stacking them, and handles NoneTypes """ + elem = batch[0] + elem_type = type(elem) + + if collate_fn_map is not None: + if elem_type in collate_fn_map: + return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map) + + for collate_type in collate_fn_map: + if isinstance(elem, collate_type): + return collate_fn_map[collate_type](batch, collate_fn_map=collate_fn_map) + + if isinstance(elem, collections.abc.Mapping): + try: + return elem_type({key: cat_collate([d[key] for d in batch], collate_fn_map=collate_fn_map) for key in elem}) + except TypeError: + # The mapping type may not support `__init__(iterable)`. + return {key: cat_collate([d[key] for d in batch], collate_fn_map=collate_fn_map) for key in elem} + elif isinstance(elem, tuple) and hasattr(elem, '_fields'): # namedtuple + return elem_type(*(cat_collate(samples, collate_fn_map=collate_fn_map) for samples in zip(*batch))) + elif isinstance(elem, collections.abc.Sequence): + transposed = list(zip(*batch)) # It may be accessed twice, so we use a list. + + if isinstance(elem, tuple): + # Backwards compatibility. + return [cat_collate(samples, collate_fn_map=collate_fn_map) for samples in transposed] + else: + try: + return elem_type([cat_collate(samples, collate_fn_map=collate_fn_map) for samples in transposed]) + except TypeError: + # The sequence type may not support `__init__(iterable)` (e.g., `range`). + return [cat_collate(samples, collate_fn_map=collate_fn_map) for samples in transposed] + + raise TypeError(default_collate_err_msg_format.format(elem_type)) diff --git a/submodules/mast3r/mast3r/utils/misc.py b/submodules/mast3r/mast3r/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..1a5403c67116f5156e47537df8fbcfcacb7bb474 --- /dev/null +++ b/submodules/mast3r/mast3r/utils/misc.py @@ -0,0 +1,17 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilitary functions for MASt3R +# -------------------------------------------------------- +import os +import hashlib + + +def mkdir_for(f): + os.makedirs(os.path.dirname(f), exist_ok=True) + return f + + +def hash_md5(s): + return hashlib.md5(s.encode('utf-8')).hexdigest() diff --git a/submodules/mast3r/mast3r/utils/path_to_dust3r.py b/submodules/mast3r/mast3r/utils/path_to_dust3r.py new file mode 100644 index 0000000000000000000000000000000000000000..ebfd78cb432ef45ee30bb893f7f90fff08474b93 --- /dev/null +++ b/submodules/mast3r/mast3r/utils/path_to_dust3r.py @@ -0,0 +1,19 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# dust3r submodule import +# -------------------------------------------------------- + +import sys +import os.path as path +HERE_PATH = path.normpath(path.dirname(__file__)) +DUSt3R_REPO_PATH = path.normpath(path.join(HERE_PATH, '../../dust3r')) +DUSt3R_LIB_PATH = path.join(DUSt3R_REPO_PATH, 'dust3r') +# check the presence of models directory in repo to be sure its cloned +if path.isdir(DUSt3R_LIB_PATH): + # workaround for sibling import + sys.path.insert(0, DUSt3R_REPO_PATH) +else: + raise ImportError(f"dust3r is not initialized, could not find: {DUSt3R_LIB_PATH}.\n " + "Did you forget to run 'git submodule update --init --recursive' ?") diff --git a/submodules/mast3r/requirements.txt b/submodules/mast3r/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ff88936c77d98f85c1a4d5ae4bfcf374e332f4b3 --- /dev/null +++ b/submodules/mast3r/requirements.txt @@ -0,0 +1 @@ +scikit-learn \ No newline at end of file diff --git a/submodules/mast3r/train.py b/submodules/mast3r/train.py new file mode 100644 index 0000000000000000000000000000000000000000..57a7689c408f63701cf4287ada50802f1a5cef6a --- /dev/null +++ b/submodules/mast3r/train.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# training executable for MASt3R +# -------------------------------------------------------- +from mast3r.model import AsymmetricMASt3R +from mast3r.losses import ConfMatchingLoss, MatchingLoss, APLoss, Regr3D, InfoNCE, Regr3D_ScaleShiftInv +from mast3r.datasets import ARKitScenes, BlendedMVS, Co3d, MegaDepth, ScanNetpp, StaticThings3D, Waymo, WildRGBD + +import mast3r.utils.path_to_dust3r # noqa +# add mast3r classes to dust3r imports +import dust3r.training +dust3r.training.AsymmetricMASt3R = AsymmetricMASt3R +dust3r.training.Regr3D = Regr3D +dust3r.training.Regr3D_ScaleShiftInv = Regr3D_ScaleShiftInv +dust3r.training.MatchingLoss = MatchingLoss +dust3r.training.ConfMatchingLoss = ConfMatchingLoss +dust3r.training.InfoNCE = InfoNCE +dust3r.training.APLoss = APLoss + +import dust3r.datasets +dust3r.datasets.ARKitScenes = ARKitScenes +dust3r.datasets.BlendedMVS = BlendedMVS +dust3r.datasets.Co3d = Co3d +dust3r.datasets.MegaDepth = MegaDepth +dust3r.datasets.ScanNetpp = ScanNetpp +dust3r.datasets.StaticThings3D = StaticThings3D +dust3r.datasets.Waymo = Waymo +dust3r.datasets.WildRGBD = WildRGBD + +from dust3r.training import get_args_parser as dust3r_get_args_parser # noqa +from dust3r.training import train # noqa + + +def get_args_parser(): + parser = dust3r_get_args_parser() + # change defaults + parser.prog = 'MASt3R training' + parser.set_defaults(model="AsymmetricMASt3R(patch_embed_cls='ManyAR_PatchEmbed')") + return parser + + +if __name__ == '__main__': + args = get_args_parser() + args = args.parse_args() + train(args) diff --git a/submodules/mast3r/visloc.py b/submodules/mast3r/visloc.py new file mode 100644 index 0000000000000000000000000000000000000000..8e460169068d3d82308538aef6c14c756e9848ce --- /dev/null +++ b/submodules/mast3r/visloc.py @@ -0,0 +1,538 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# visloc script with support for coarse to fine +# -------------------------------------------------------- +import os +import numpy as np +import random +import torch +import torchvision.transforms as tvf +import argparse +from tqdm import tqdm +from PIL import Image +import math + +from mast3r.model import AsymmetricMASt3R +from mast3r.fast_nn import fast_reciprocal_NNs +from mast3r.utils.coarse_to_fine import select_pairs_of_crops, crop_slice +from mast3r.utils.collate import cat_collate, cat_collate_fn_map +from mast3r.utils.misc import mkdir_for +from mast3r.datasets.utils.cropping import crop_to_homography + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.inference import inference, loss_of_one_batch +from dust3r.utils.geometry import geotrf, colmap_to_opencv_intrinsics, opencv_to_colmap_intrinsics +from dust3r.datasets.utils.transforms import ImgNorm +from dust3r_visloc.datasets import * +from dust3r_visloc.localization import run_pnp +from dust3r_visloc.evaluation import get_pose_error, aggregate_stats, export_results +from dust3r_visloc.datasets.utils import get_HW_resolution, rescale_points3d + + +def get_args_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=str, required=True, help="visloc dataset to eval") + parser_weights = parser.add_mutually_exclusive_group(required=True) + parser_weights.add_argument("--weights", type=str, help="path to the model weights", default=None) + parser_weights.add_argument("--model_name", type=str, help="name of the model weights", + choices=["MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric"]) + + parser.add_argument("--confidence_threshold", type=float, default=1.001, + help="confidence values higher than threshold are invalid") + parser.add_argument('--pixel_tol', default=5, type=int) + + parser.add_argument("--coarse_to_fine", action='store_true', default=False, + help="do the matching from coarse to fine") + parser.add_argument("--max_image_size", type=int, default=None, + help="max image size for the fine resolution") + parser.add_argument("--c2f_crop_with_homography", action='store_true', default=False, + help="when using coarse to fine, crop with homographies to keep cx, cy centered") + + parser.add_argument("--device", type=str, default='cuda', help="pytorch device") + parser.add_argument("--pnp_mode", type=str, default="cv2", choices=['cv2', 'poselib', 'pycolmap'], + help="pnp lib to use") + parser_reproj = parser.add_mutually_exclusive_group() + parser_reproj.add_argument("--reprojection_error", type=float, default=5.0, help="pnp reprojection error") + parser_reproj.add_argument("--reprojection_error_diag_ratio", type=float, default=None, + help="pnp reprojection error as a ratio of the diagonal of the image") + + parser.add_argument("--max_batch_size", type=int, default=48, + help="max batch size for inference on crops when using coarse to fine") + parser.add_argument("--pnp_max_points", type=int, default=100_000, help="pnp maximum number of points kept") + parser.add_argument("--viz_matches", type=int, default=0, help="debug matches") + + parser.add_argument("--output_dir", type=str, default=None, help="output path") + parser.add_argument("--output_label", type=str, default='', help="prefix for results files") + return parser + + +@torch.no_grad() +def coarse_matching(query_view, map_view, model, device, pixel_tol, fast_nn_params): + # prepare batch + imgs = [] + for idx, img in enumerate([query_view['rgb_rescaled'], map_view['rgb_rescaled']]): + imgs.append(dict(img=img.unsqueeze(0), true_shape=np.int32([img.shape[1:]]), + idx=idx, instance=str(idx))) + output = inference([tuple(imgs)], model, device, batch_size=1, verbose=False) + pred1, pred2 = output['pred1'], output['pred2'] + conf_list = [pred1['desc_conf'].squeeze(0).cpu().numpy(), pred2['desc_conf'].squeeze(0).cpu().numpy()] + desc_list = [pred1['desc'].squeeze(0).detach(), pred2['desc'].squeeze(0).detach()] + + # find 2D-2D matches between the two images + PQ, PM = desc_list[0], desc_list[1] + if len(PQ) == 0 or len(PM) == 0: + return [], [], [], [] + + if pixel_tol == 0: + matches_im_map, matches_im_query = fast_reciprocal_NNs(PM, PQ, subsample_or_initxy1=8, **fast_nn_params) + HM, WM = map_view['rgb_rescaled'].shape[1:] + HQ, WQ = query_view['rgb_rescaled'].shape[1:] + # ignore small border around the edge + valid_matches_map = (matches_im_map[:, 0] >= 3) & (matches_im_map[:, 0] < WM - 3) & ( + matches_im_map[:, 1] >= 3) & (matches_im_map[:, 1] < HM - 3) + valid_matches_query = (matches_im_query[:, 0] >= 3) & (matches_im_query[:, 0] < WQ - 3) & ( + matches_im_query[:, 1] >= 3) & (matches_im_query[:, 1] < HQ - 3) + valid_matches = valid_matches_map & valid_matches_query + matches_im_map = matches_im_map[valid_matches] + matches_im_query = matches_im_query[valid_matches] + valid_pts3d = [] + matches_confs = [] + else: + yM, xM = torch.where(map_view['valid_rescaled']) + matches_im_map, matches_im_query = fast_reciprocal_NNs(PM, PQ, (xM, yM), pixel_tol=pixel_tol, **fast_nn_params) + valid_pts3d = map_view['pts3d_rescaled'].cpu().numpy()[matches_im_map[:, 1], matches_im_map[:, 0]] + matches_confs = np.minimum( + conf_list[1][matches_im_map[:, 1], matches_im_map[:, 0]], + conf_list[0][matches_im_query[:, 1], matches_im_query[:, 0]] + ) + # from cv2 to colmap + matches_im_query = matches_im_query.astype(np.float64) + matches_im_map = matches_im_map.astype(np.float64) + matches_im_query[:, 0] += 0.5 + matches_im_query[:, 1] += 0.5 + matches_im_map[:, 0] += 0.5 + matches_im_map[:, 1] += 0.5 + # rescale coordinates + matches_im_query = geotrf(query_view['to_orig'], matches_im_query, norm=True) + matches_im_map = geotrf(map_view['to_orig'], matches_im_map, norm=True) + # from colmap back to cv2 + matches_im_query[:, 0] -= 0.5 + matches_im_query[:, 1] -= 0.5 + matches_im_map[:, 0] -= 0.5 + matches_im_map[:, 1] -= 0.5 + return valid_pts3d, matches_im_query, matches_im_map, matches_confs + + +@torch.no_grad() +def crops_inference(pairs, model, device, batch_size=48, verbose=True): + assert len(pairs) == 2, "Error, data should be a tuple of dicts containing the batch of image pairs" + # Forward a possibly big bunch of data, by blocks of batch_size + B = pairs[0]['img'].shape[0] + if B < batch_size: + return loss_of_one_batch(pairs, model, None, device=device, symmetrize_batch=False) + preds = [] + for ii in range(0, B, batch_size): + sel = slice(ii, ii + min(B - ii, batch_size)) + temp_data = [{}, {}] + for di in [0, 1]: + temp_data[di] = {kk: pairs[di][kk][sel] + for kk in pairs[di].keys() if pairs[di][kk] is not None} # copy chunk for forward + preds.append(loss_of_one_batch(temp_data, model, + None, device=device, symmetrize_batch=False)) # sequential forward + # Merge all preds + return cat_collate(preds, collate_fn_map=cat_collate_fn_map) + + +@torch.no_grad() +def fine_matching(query_views, map_views, model, device, max_batch_size, pixel_tol, fast_nn_params): + assert pixel_tol > 0 + output = crops_inference([query_views, map_views], + model, device, batch_size=max_batch_size, verbose=False) + pred1, pred2 = output['pred1'], output['pred2'] + descs1 = pred1['desc'].clone() + descs2 = pred2['desc'].clone() + confs1 = pred1['desc_conf'].clone() + confs2 = pred2['desc_conf'].clone() + + # Compute matches + valid_pts3d, matches_im_map, matches_im_query, matches_confs = [], [], [], [] + for ppi, (pp1, pp2, cc11, cc21) in enumerate(zip(descs1, descs2, confs1, confs2)): + valid_ppi = map_views['valid'][ppi] + pts3d_ppi = map_views['pts3d'][ppi].cpu().numpy() + conf_list_ppi = [cc11.cpu().numpy(), cc21.cpu().numpy()] + + y_ppi, x_ppi = torch.where(valid_ppi) + matches_im_map_ppi, matches_im_query_ppi = fast_reciprocal_NNs(pp2, pp1, (x_ppi, y_ppi), + pixel_tol=pixel_tol, **fast_nn_params) + + valid_pts3d_ppi = pts3d_ppi[matches_im_map_ppi[:, 1], matches_im_map_ppi[:, 0]] + matches_confs_ppi = np.minimum( + conf_list_ppi[1][matches_im_map_ppi[:, 1], matches_im_map_ppi[:, 0]], + conf_list_ppi[0][matches_im_query_ppi[:, 1], matches_im_query_ppi[:, 0]] + ) + # inverse operation where we uncrop pixel coordinates + matches_im_map_ppi = geotrf(map_views['to_orig'][ppi].cpu().numpy(), matches_im_map_ppi.copy(), norm=True) + matches_im_query_ppi = geotrf(query_views['to_orig'][ppi].cpu().numpy(), matches_im_query_ppi.copy(), norm=True) + + matches_im_map.append(matches_im_map_ppi) + matches_im_query.append(matches_im_query_ppi) + valid_pts3d.append(valid_pts3d_ppi) + matches_confs.append(matches_confs_ppi) + + if len(valid_pts3d) == 0: + return [], [], [], [] + + matches_im_map = np.concatenate(matches_im_map, axis=0) + matches_im_query = np.concatenate(matches_im_query, axis=0) + valid_pts3d = np.concatenate(valid_pts3d, axis=0) + matches_confs = np.concatenate(matches_confs, axis=0) + return valid_pts3d, matches_im_query, matches_im_map, matches_confs + + +def crop(img, mask, pts3d, crop, intrinsics=None): + out_cropped_img = img.clone() + if mask is not None: + out_cropped_mask = mask.clone() + else: + out_cropped_mask = None + if pts3d is not None: + out_cropped_pts3d = pts3d.clone() + else: + out_cropped_pts3d = None + to_orig = torch.eye(3, device=img.device) + + # If intrinsics available, crop and apply rectifying homography. Otherwise, just crop + if intrinsics is not None: + K_old = intrinsics + imsize, K_new, R, H = crop_to_homography(K_old, crop) + # apply homography to image + H /= H[2, 2] + homo8 = H.ravel().tolist()[:8] + # From float tensor to uint8 PIL Image + pilim = Image.fromarray((255 * (img + 1.) / 2).to(torch.uint8).numpy()) + pilout_cropped_img = pilim.transform(imsize, Image.Transform.PERSPECTIVE, + homo8, resample=Image.Resampling.BICUBIC) + + # From uint8 PIL Image to float tensor + out_cropped_img = 2. * torch.tensor(np.array(pilout_cropped_img)).to(img) / 255. - 1. + if out_cropped_mask is not None: + pilmask = Image.fromarray((255 * out_cropped_mask).to(torch.uint8).numpy()) + pilout_cropped_mask = pilmask.transform( + imsize, Image.Transform.PERSPECTIVE, homo8, resample=Image.Resampling.NEAREST) + out_cropped_mask = torch.from_numpy(np.array(pilout_cropped_mask) > 0).to(out_cropped_mask.dtype) + if out_cropped_pts3d is not None: + out_cropped_pts3d = out_cropped_pts3d.numpy() + out_cropped_X = np.array(Image.fromarray(out_cropped_pts3d[:, :, 0]).transform(imsize, + Image.Transform.PERSPECTIVE, + homo8, + resample=Image.Resampling.NEAREST)) + out_cropped_Y = np.array(Image.fromarray(out_cropped_pts3d[:, :, 1]).transform(imsize, + Image.Transform.PERSPECTIVE, + homo8, + resample=Image.Resampling.NEAREST)) + out_cropped_Z = np.array(Image.fromarray(out_cropped_pts3d[:, :, 2]).transform(imsize, + Image.Transform.PERSPECTIVE, + homo8, + resample=Image.Resampling.NEAREST)) + + out_cropped_pts3d = torch.from_numpy(np.stack([out_cropped_X, out_cropped_Y, out_cropped_Z], axis=-1)) + + to_orig = torch.tensor(H, device=img.device) + else: + out_cropped_img = img[crop_slice(crop)] + if out_cropped_mask is not None: + out_cropped_mask = out_cropped_mask[crop_slice(crop)] + if out_cropped_pts3d is not None: + out_cropped_pts3d = out_cropped_pts3d[crop_slice(crop)] + to_orig[:2, -1] = torch.tensor(crop[:2]) + + return out_cropped_img, out_cropped_mask, out_cropped_pts3d, to_orig + + +def resize_image_to_max(max_image_size, rgb, K): + W, H = rgb.size + if max_image_size and max(W, H) > max_image_size: + islandscape = (W >= H) + if islandscape: + WMax = max_image_size + HMax = int(H * (WMax / W)) + else: + HMax = max_image_size + WMax = int(W * (HMax / H)) + resize_op = tvf.Compose([ImgNorm, tvf.Resize(size=[HMax, WMax])]) + rgb_tensor = resize_op(rgb).permute(1, 2, 0) + to_orig_max = np.array([[W / WMax, 0, 0], + [0, H / HMax, 0], + [0, 0, 1]]) + to_resize_max = np.array([[WMax / W, 0, 0], + [0, HMax / H, 0], + [0, 0, 1]]) + + # Generate new camera parameters + new_K = opencv_to_colmap_intrinsics(K) + new_K[0, :] *= WMax / W + new_K[1, :] *= HMax / H + new_K = colmap_to_opencv_intrinsics(new_K) + else: + rgb_tensor = ImgNorm(rgb).permute(1, 2, 0) + to_orig_max = np.eye(3) + to_resize_max = np.eye(3) + HMax, WMax = H, W + new_K = K + return rgb_tensor, new_K, to_orig_max, to_resize_max, (HMax, WMax) + + +if __name__ == '__main__': + parser = get_args_parser() + args = parser.parse_args() + conf_thr = args.confidence_threshold + device = args.device + pnp_mode = args.pnp_mode + assert args.pixel_tol > 0 + reprojection_error = args.reprojection_error + reprojection_error_diag_ratio = args.reprojection_error_diag_ratio + pnp_max_points = args.pnp_max_points + viz_matches = args.viz_matches + + if args.weights is not None: + weights_path = args.weights + else: + weights_path = "naver/" + args.model_name + model = AsymmetricMASt3R.from_pretrained(weights_path).to(args.device) + fast_nn_params = dict(device=device, dist='dot', block_size=2**13) + dataset = eval(args.dataset) + dataset.set_resolution(model) + + query_names = [] + poses_pred = [] + pose_errors = [] + angular_errors = [] + params_str = f'tol_{args.pixel_tol}' + ("_c2f" if args.coarse_to_fine else '') + if args.max_image_size is not None: + params_str = params_str + f'_{args.max_image_size}' + if args.coarse_to_fine and args.c2f_crop_with_homography: + params_str = params_str + '_with_homography' + for idx in tqdm(range(len(dataset))): + views = dataset[(idx)] # 0 is the query + query_view = views[0] + map_views = views[1:] + query_names.append(query_view['image_name']) + + query_pts2d = [] + query_pts3d = [] + maxdim = max(model.patch_embed.img_size) + query_rgb_tensor, query_K, query_to_orig_max, query_to_resize_max, (HQ, WQ) = resize_image_to_max( + args.max_image_size, query_view['rgb'], query_view['intrinsics']) + + # pairs of crops have the same resolution + query_resolution = get_HW_resolution(HQ, WQ, maxdim=maxdim, patchsize=model.patch_embed.patch_size) + for map_view in map_views: + if args.output_dir is not None: + cache_file = os.path.join(args.output_dir, 'matches', params_str, + query_view['image_name'], map_view['image_name'] + '.npz') + else: + cache_file = None + + if cache_file is not None and os.path.isfile(cache_file): + matches = np.load(cache_file) + valid_pts3d = matches['valid_pts3d'] + matches_im_query = matches['matches_im_query'] + matches_im_map = matches['matches_im_map'] + matches_conf = matches['matches_conf'] + else: + # coarse matching + if args.coarse_to_fine and (maxdim < max(WQ, HQ)): + # use all points + _, coarse_matches_im0, coarse_matches_im1, _ = coarse_matching(query_view, map_view, model, device, + 0, fast_nn_params) + + # visualize a few matches + if viz_matches > 0: + num_matches = coarse_matches_im1.shape[0] + print(f'found {num_matches} matches') + + viz_imgs = [np.array(query_view['rgb']), np.array(map_view['rgb'])] + from matplotlib import pyplot as pl + n_viz = viz_matches + match_idx_to_viz = np.round(np.linspace(0, num_matches - 1, n_viz)).astype(int) + viz_matches_im_query = coarse_matches_im0[match_idx_to_viz] + viz_matches_im_map = coarse_matches_im1[match_idx_to_viz] + + H0, W0, H1, W1 = *viz_imgs[0].shape[:2], *viz_imgs[1].shape[:2] + img0 = np.pad(viz_imgs[0], ((0, max(H1 - H0, 0)), (0, 0), (0, 0)), + 'constant', constant_values=0) + img1 = np.pad(viz_imgs[1], ((0, max(H0 - H1, 0)), (0, 0), (0, 0)), + 'constant', constant_values=0) + img = np.concatenate((img0, img1), axis=1) + pl.figure() + pl.imshow(img) + cmap = pl.get_cmap('jet') + for i in range(n_viz): + (x0, y0), (x1, y1) = viz_matches_im_query[i].T, viz_matches_im_map[i].T + pl.plot([x0, x1 + W0], [y0, y1], '-+', + color=cmap(i / (n_viz - 1)), scalex=False, scaley=False) + pl.show(block=True) + + valid_all = map_view['valid'] + pts3d = map_view['pts3d'] + + WM_full, HM_full = map_view['rgb'].size + map_rgb_tensor, map_K, map_to_orig_max, map_to_resize_max, (HM, WM) = resize_image_to_max( + args.max_image_size, map_view['rgb'], map_view['intrinsics']) + if WM_full != WM or HM_full != HM: + y_full, x_full = torch.where(valid_all) + pos2d_cv2 = torch.stack([x_full, y_full], dim=-1).cpu().numpy().astype(np.float64) + sparse_pts3d = pts3d[y_full, x_full].cpu().numpy() + _, _, pts3d_max, valid_max = rescale_points3d( + pos2d_cv2, sparse_pts3d, map_to_resize_max, HM, WM) + pts3d = torch.from_numpy(pts3d_max) + valid_all = torch.from_numpy(valid_max) + + coarse_matches_im0 = geotrf(query_to_resize_max, coarse_matches_im0, norm=True) + coarse_matches_im1 = geotrf(map_to_resize_max, coarse_matches_im1, norm=True) + + crops1, crops2 = [], [] + crops_v1, crops_p1 = [], [] + to_orig1, to_orig2 = [], [] + map_resolution = get_HW_resolution(HM, WM, maxdim=maxdim, patchsize=model.patch_embed.patch_size) + + for crop_q, crop_b, pair_tag in select_pairs_of_crops(map_rgb_tensor, + query_rgb_tensor, + coarse_matches_im1, + coarse_matches_im0, + maxdim=maxdim, + overlap=.5, + forced_resolution=[map_resolution, + query_resolution]): + # Per crop processing + if not args.c2f_crop_with_homography: + map_K = None + query_K = None + + c1, v1, p1, trf1 = crop(map_rgb_tensor, valid_all, pts3d, crop_q, map_K) + c2, _, _, trf2 = crop(query_rgb_tensor, None, None, crop_b, query_K) + crops1.append(c1) + crops2.append(c2) + crops_v1.append(v1) + crops_p1.append(p1) + to_orig1.append(trf1) + to_orig2.append(trf2) + + if len(crops1) == 0 or len(crops2) == 0: + valid_pts3d, matches_im_query, matches_im_map, matches_conf = [], [], [], [] + else: + crops1, crops2 = torch.stack(crops1), torch.stack(crops2) + if len(crops1.shape) == 3: + crops1, crops2 = crops1[None], crops2[None] + crops_v1 = torch.stack(crops_v1) + crops_p1 = torch.stack(crops_p1) + to_orig1, to_orig2 = torch.stack(to_orig1), torch.stack(to_orig2) + map_crop_view = dict(img=crops1.permute(0, 3, 1, 2), + instance=['1' for _ in range(crops1.shape[0])], + valid=crops_v1, pts3d=crops_p1, + to_orig=to_orig1) + query_crop_view = dict(img=crops2.permute(0, 3, 1, 2), + instance=['2' for _ in range(crops2.shape[0])], + to_orig=to_orig2) + + # Inference and Matching + valid_pts3d, matches_im_query, matches_im_map, matches_conf = fine_matching(query_crop_view, + map_crop_view, + model, device, + args.max_batch_size, + args.pixel_tol, + fast_nn_params) + matches_im_query = geotrf(query_to_orig_max, matches_im_query, norm=True) + matches_im_map = geotrf(map_to_orig_max, matches_im_map, norm=True) + else: + # use only valid 2d points + valid_pts3d, matches_im_query, matches_im_map, matches_conf = coarse_matching(query_view, map_view, + model, device, + args.pixel_tol, + fast_nn_params) + if cache_file is not None: + mkdir_for(cache_file) + np.savez(cache_file, valid_pts3d=valid_pts3d, matches_im_query=matches_im_query, + matches_im_map=matches_im_map, matches_conf=matches_conf) + + # apply conf + if len(matches_conf) > 0: + mask = matches_conf >= conf_thr + valid_pts3d = valid_pts3d[mask] + matches_im_query = matches_im_query[mask] + matches_im_map = matches_im_map[mask] + matches_conf = matches_conf[mask] + + # visualize a few matches + if viz_matches > 0: + num_matches = matches_im_map.shape[0] + print(f'found {num_matches} matches') + + viz_imgs = [np.array(query_view['rgb']), np.array(map_view['rgb'])] + from matplotlib import pyplot as pl + n_viz = viz_matches + match_idx_to_viz = np.round(np.linspace(0, num_matches - 1, n_viz)).astype(int) + viz_matches_im_query = matches_im_query[match_idx_to_viz] + viz_matches_im_map = matches_im_map[match_idx_to_viz] + + H0, W0, H1, W1 = *viz_imgs[0].shape[:2], *viz_imgs[1].shape[:2] + img0 = np.pad(viz_imgs[0], ((0, max(H1 - H0, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img1 = np.pad(viz_imgs[1], ((0, max(H0 - H1, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img = np.concatenate((img0, img1), axis=1) + pl.figure() + pl.imshow(img) + cmap = pl.get_cmap('jet') + for i in range(n_viz): + (x0, y0), (x1, y1) = viz_matches_im_query[i].T, viz_matches_im_map[i].T + pl.plot([x0, x1 + W0], [y0, y1], '-+', color=cmap(i / (n_viz - 1)), scalex=False, scaley=False) + pl.show(block=True) + + if len(valid_pts3d) == 0: + pass + else: + query_pts3d.append(valid_pts3d) + query_pts2d.append(matches_im_query) + + if len(query_pts2d) == 0: + success = False + pr_querycam_to_world = None + else: + query_pts2d = np.concatenate(query_pts2d, axis=0).astype(np.float32) + query_pts3d = np.concatenate(query_pts3d, axis=0) + if len(query_pts2d) > pnp_max_points: + idxs = random.sample(range(len(query_pts2d)), pnp_max_points) + query_pts3d = query_pts3d[idxs] + query_pts2d = query_pts2d[idxs] + + W, H = query_view['rgb'].size + if reprojection_error_diag_ratio is not None: + reprojection_error_img = reprojection_error_diag_ratio * math.sqrt(W**2 + H**2) + else: + reprojection_error_img = reprojection_error + success, pr_querycam_to_world = run_pnp(query_pts2d, query_pts3d, + query_view['intrinsics'], query_view['distortion'], + pnp_mode, reprojection_error_img, img_size=[W, H]) + + if not success: + abs_transl_error = float('inf') + abs_angular_error = float('inf') + else: + abs_transl_error, abs_angular_error = get_pose_error(pr_querycam_to_world, query_view['cam_to_world']) + + pose_errors.append(abs_transl_error) + angular_errors.append(abs_angular_error) + poses_pred.append(pr_querycam_to_world) + + xp_label = params_str + f'_conf_{conf_thr}' + if args.output_label: + xp_label = args.output_label + "_" + xp_label + if reprojection_error_diag_ratio is not None: + xp_label = xp_label + f'_reproj_diag_{reprojection_error_diag_ratio}' + else: + xp_label = xp_label + f'_reproj_err_{reprojection_error}' + export_results(args.output_dir, xp_label, query_names, poses_pred) + out_string = aggregate_stats(f'{args.dataset}', pose_errors, angular_errors) + print(out_string) diff --git a/wheel/curope-0.0.0-cp310-cp310-linux_x86_64.whl b/wheel/curope-0.0.0-cp310-cp310-linux_x86_64.whl new file mode 100644 index 0000000000000000000000000000000000000000..311da7ede4cf0f740cec0b9eb6f2f656d9e7e78b --- /dev/null +++ b/wheel/curope-0.0.0-cp310-cp310-linux_x86_64.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a645b73a0ec4daae24dd6b6a911e20f4ae7654701d28984b5ba215796212358 +size 2544406 diff --git a/wheel/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl b/wheel/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl new file mode 100644 index 0000000000000000000000000000000000000000..7ae808d4b686e13c4695295ea0a26bc76309c8ab --- /dev/null +++ b/wheel/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c44d2440da91b41bb32ed29ad55956166172bae6215c76523cb9bf12a9fb87d +size 3789734 diff --git a/wheel/flash_attn-2.6.3+cu123torch2.1cxx11abiFALSE-cp310-cp310-linux_x86_64.whl b/wheel/flash_attn-2.6.3+cu123torch2.1cxx11abiFALSE-cp310-cp310-linux_x86_64.whl new file mode 100644 index 0000000000000000000000000000000000000000..ff925a499cf6ff011e84b3a67aef85c5a7c3a969 --- /dev/null +++ b/wheel/flash_attn-2.6.3+cu123torch2.1cxx11abiFALSE-cp310-cp310-linux_x86_64.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edf1fa13f79c5dc6adc092eb70d5218d954a5d03040383f513c57b8df60d5707 +size 186975250 diff --git a/wheel/pointops-1.0-cp310-cp310-linux_x86_64.whl b/wheel/pointops-1.0-cp310-cp310-linux_x86_64.whl new file mode 100644 index 0000000000000000000000000000000000000000..941405e0be3df7b87bc82f6171319ceb3d5837ca --- /dev/null +++ b/wheel/pointops-1.0-cp310-cp310-linux_x86_64.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e97eea62d00912cf09c0a64f0302465d3320b3d4e89191ffddeb517340bc7c29 +size 6034393 diff --git a/wheel/simple_knn-0.0.0-cp310-cp310-linux_x86_64.whl b/wheel/simple_knn-0.0.0-cp310-cp310-linux_x86_64.whl new file mode 100644 index 0000000000000000000000000000000000000000..c11e0f4a1d0b1ab6d90c543fabce6fb07a2df5d8 --- /dev/null +++ b/wheel/simple_knn-0.0.0-cp310-cp310-linux_x86_64.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8c9a85dfe83276010ca695c371c65ec0f91e2ecb41b627b8258c91fca3b0c85 +size 3465490 diff --git a/wheel/torch_scatter-2.1.2+pt21cu121-cp310-cp310-linux_x86_64.whl b/wheel/torch_scatter-2.1.2+pt21cu121-cp310-cp310-linux_x86_64.whl new file mode 100644 index 0000000000000000000000000000000000000000..ba06ece5de6e83f82791545fd63ecd2daffd64fe --- /dev/null +++ b/wheel/torch_scatter-2.1.2+pt21cu121-cp310-cp310-linux_x86_64.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f8d47fab8667b153ca5ca89b616c82f7610814bd25e1b1ee081e789bed3a76f +size 10843773