Release training script

Former-commit-id: 16f73b66904d9d45e67de3ae95f7fe0e54f097f8

README.md

@@ -169,7 +169,11 @@ Download SAM ViT-H pre-trained weights from the [link](https://dl.fbaipublicfile
 
 ### Training
 ```
-deepspeed --master_port=24999 train_ds.py --version="PATH_TO_LLaVA_Wegihts" --dataset_dir='./dataset' --vision_pretrained="PATH_TO_SAM_Weights" --exp_name="lisa-7b"
+deepspeed --master_port=24999 train_ds.py \
+  --version="PATH_TO_LLaVA_Wegihts" \
+  --dataset_dir='./dataset' \
+  --vision_pretrained="PATH_TO_SAM_Weights" \
+  --exp_name="lisa-7b"
 ```
 When training is finished, to get the full model weight:
 ```
@@ -178,7 +182,13 @@ cd ./runs/lisa-7b/ckpt_model && python zero_to_fp32.py . ../pytorch_model.bin
 
 ### Validation
 ```
-deepspeed --master_port=24999 train_ds.py --version="PATH_TO_LLaVA_Wegihts" --dataset_dir='./dataset' --vision_pretrained="PATH_TO_SAM_Weights" --exp_name="lisa-7b" --weight='PATH_TO_pytorch_model.bin' --eval_only
+deepspeed --master_port=24999 train_ds.py \
+  --version="PATH_TO_LLaVA_Wegihts" \
+  --dataset_dir='./dataset' \
+  --vision_pretrained="PATH_TO_SAM_Weights" \
+  --exp_name="lisa-7b" \
+  --weight='PATH_TO_pytorch_model.bin' \
+  --eval_only
 ```
 
  

utils/reason_seg_dataset.py

@@ -76,12 +76,13 @@ class ReasonSegDataset(torch.utils.data.Dataset):
             ) as f:
                 items = json.load(f)
             for item in items:
-                img_name = item["image_path"].split("/")[-1]
+                img_name = item["image"]
                 self.img_to_explanation[img_name] = {
                     "query": item["query"],
                     "outputs": item["outputs"],
                 }
 
+            print("len(self.img_to_explanation): ", len(self.img_to_explanation))
 
     def __len__(self):
         return self.samples_per_epoch

utils/sem_seg_dataset.py

@@ -104,10 +104,6 @@ def init_paco_lvis(base_image_dir):
             obj, part = cat_split
             obj = obj.split("_(")[0]
             part = part.split("_(")[0]
-            # if random.random() < 0.5:
-            #   name = obj + " " + part
-            # else:
-            #   name = "the {} of the {}".format(part, obj)
             name = (obj, part)
         class_map_paco_lvis[cat["id"]] = name
     img_ids = coco_api_paco_lvis.getImgIds()