training.log

[2025-02-20 17:55:10,206][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,207][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,207][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,207][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,207][__main__][INFO] - 
Initializing training for seed 242
[2025-02-20 17:55:10,209][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,209][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,210][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,210][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,210][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,210][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,210][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,210][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,213][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,213][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,213][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:10,213][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:27,037][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:27,037][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:27,037][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:27,038][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:27,053][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:27,053][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:27,053][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:27,053][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:27,053][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:27,053][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:27,053][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:27,053][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:31,375][__main__][INFO] - Training config for seed 242:
{'callbacks': {'rollout_lh': {'tasks': {'_target_': 'calvin_env.envs.tasks.Tasks', 'tasks': {'rotate_red_block_right': ['rotate_object', 'block_red', -60], 'rotate_red_block_left': ['rotate_object', 'block_red', 60], 'rotate_blue_block_right': ['rotate_object', 'block_blue', -60], 'rotate_blue_block_left': ['rotate_object', 'block_blue', 60], 'rotate_pink_block_right': ['rotate_object', 'block_pink', -60], 'rotate_pink_block_left': ['rotate_object', 'block_pink', 60], 'push_red_block_right': ['push_object', 'block_red', 0.1, 0], 'push_red_block_left': ['push_object', 'block_red', -0.1, 0], 'push_blue_block_right': ['push_object', 'block_blue', 0.1, 0], 'push_blue_block_left': ['push_object', 'block_blue', -0.1, 0], 'push_pink_block_right': ['push_object', 'block_pink', 0.1, 0], 'push_pink_block_left': ['push_object', 'block_pink', -0.1, 0], 'move_slider_left': ['move_door_rel', 'base__slide', 0.15], 'move_slider_right': ['move_door_rel', 'base__slide', -0.15], 'open_drawer': ['move_door_rel', 'base__drawer', 0.12], 'close_drawer': ['move_door_rel', 'base__drawer', -0.12], 'lift_red_block_table': ['lift_object', 'block_red', 0.05, 'table', 'base_link'], 'lift_red_block_slider': ['lift_object', 'block_red', 0.03, 'table', 'plank_link'], 'lift_red_block_drawer': ['lift_object', 'block_red', 0.05, 'table', 'drawer_link'], 'lift_blue_block_table': ['lift_object', 'block_blue', 0.05, 'table', 'base_link'], 'lift_blue_block_slider': ['lift_object', 'block_blue', 0.03, 'table', 'plank_link'], 'lift_blue_block_drawer': ['lift_object', 'block_blue', 0.05, 'table', 'drawer_link'], 'lift_pink_block_table': ['lift_object', 'block_pink', 0.05, 'table', 'base_link'], 'lift_pink_block_slider': ['lift_object', 'block_pink', 0.03, 'table', 'plank_link'], 'lift_pink_block_drawer': ['lift_object', 'block_pink', 0.05, 'table', 'drawer_link'], 'place_in_slider': ['place_object', 'table', 'plank_link'], 'place_in_drawer': ['place_object', 'table', 'drawer_link'], 'stack_block': ['stack_objects'], 'unstack_block': ['unstack_objects'], 'turn_on_lightbulb': ['toggle_light', 'lightbulb', 0, 1], 'turn_off_lightbulb': ['toggle_light', 'lightbulb', 1, 0], 'turn_on_led': ['toggle_light', 'led', 0, 1], 'turn_off_led': ['toggle_light', 'led', 1, 0], 'push_into_drawer': ['push_object_into', ['block_red', 'block_blue', 'block_pink'], 'table', 'base_link', 'table', 'drawer_link']}}, 'val_annotations': {'rotate_red_block_right': ['take the red block and rotate it to the right'], 'rotate_red_block_left': ['take the red block and rotate it to the left'], 'rotate_blue_block_right': ['take the blue block and rotate it to the right'], 'rotate_blue_block_left': ['take the blue block and rotate it to the left'], 'rotate_pink_block_right': ['take the pink block and rotate it to the right'], 'rotate_pink_block_left': ['take the pink block and rotate it to the left'], 'push_red_block_right': ['go push the red block right'], 'push_red_block_left': ['go push the red block left'], 'push_blue_block_right': ['go push the blue block right'], 'push_blue_block_left': ['go push the blue block left'], 'push_pink_block_right': ['go push the pink block right'], 'push_pink_block_left': ['go push the pink block left'], 'move_slider_left': ['push the sliding door to the left side'], 'move_slider_right': ['push the sliding door to the right side'], 'open_drawer': ['pull the handle to open the drawer'], 'close_drawer': ['push the handle to close the drawer'], 'lift_red_block_table': ['grasp and lift the red block'], 'lift_blue_block_table': ['grasp and lift the blue block'], 'lift_pink_block_table': ['grasp and lift the pink block'], 'lift_red_block_slider': ['lift the red block from the sliding cabinet'], 'lift_blue_block_slider': ['lift the blue block from the sliding cabinet'], 'lift_pink_block_slider': ['lift the pink block from the sliding cabinet'], 'lift_red_block_drawer': ['Take the red block from the drawer'], 'lift_blue_block_drawer': ['Take the blue block from the drawer'], 'lift_pink_block_drawer': ['Take the pink block from the drawer'], 'place_in_slider': ['store the grasped block in the sliding cabinet'], 'place_in_drawer': ['store the grasped block in the drawer'], 'push_into_drawer': ['slide the block that it falls into the drawer'], 'stack_block': ['stack the grasped block'], 'unstack_block': ['remove the stacked block'], 'turn_on_lightbulb': ['use the switch to turn on the light bulb'], 'turn_off_lightbulb': ['use the switch to turn off the light bulb'], 'turn_on_led': ['press the button to turn on the led light'], 'turn_off_led': ['press the button to turn off the led light']}, '_target_': 'flower.rollout.rollout_long_horizon.RolloutLongHorizon', '_recursive_': False, 'env_cfg': {'_target_': 'flower.wrappers.hulc_wrapper.HulcWrapper'}, 'skip_epochs': '${rollout_lh_skip_epochs}', 'rollout_freq': 5, 'num_videos': 0, 'num_sequences': 1000, 'replan_freq': 30, 'ep_len': 360, 'empty_cache': False, 'log_video_to_file': False, 'save_dir': './videos', 'lang_folder': '${lang_folder}', 'debug': False}, 'ema': {'_target_': 'flower.callbacks.ema.EMA', 'decay': 0.999, 'start_step': 0, 'save_ema_weights_in_callback_state': True, 'evaluate_ema_weights_instead': True, 'power': 0.6666666666666666, 'inv_gamma': 1.0, 'min_value': 0.0, 'max_value': 0.9999}, 'checkpoint': {'_target_': 'pytorch_lightning.callbacks.ModelCheckpoint', 'save_top_k': 1, 'verbose': True, 'monitor': 'eval_lh/avg_seq_len', 'mode': 'max', 'dirpath': 'saved_models', 'filename': '{epoch:02d}_{eval_lh/avg_seq_len:.2f}', 'every_n_epochs': '${callbacks.rollout_lh.rollout_freq}'}}, 'datamodule': {'transforms': {'train': {'rgb_static': [{'_target_': 'torchvision.transforms.Resize', 'size': 224, 'antialias': True}, {'_target_': 'flower.utils.transforms.RandomShiftsAug', 'pad': 10}, {'_target_': 'flower.utils.transforms.ScaleImageTensor'}, {'_target_': 'torchvision.transforms.Normalize', 'mean': [0.48145466, 0.4578275, 0.40821073], 'std': [0.26862954, 0.26130258, 0.27577711]}], 'rgb_gripper': [{'_target_': 'torchvision.transforms.Resize', 'size': 224, 'antialias': True}, {'_target_': 'flower.utils.transforms.RandomShiftsAug', 'pad': 4}, {'_target_': 'flower.utils.transforms.ScaleImageTensor'}, {'_target_': 'torchvision.transforms.Normalize', 'mean': [0.48145466, 0.4578275, 0.40821073], 'std': [0.26862954, 0.26130258, 0.27577711]}]}, 'val': {'rgb_static': [{'_target_': 'torchvision.transforms.Resize', 'size': 224, 'antialias': True}, {'_target_': 'flower.utils.transforms.ScaleImageTensor'}, {'_target_': 'torchvision.transforms.Normalize', 'mean': [0.48145466, 0.4578275, 0.40821073], 'std': [0.26862954, 0.26130258, 0.27577711]}], 'rgb_gripper': [{'_target_': 'torchvision.transforms.Resize', 'size': 224, 'antialias': True}, {'_target_': 'flower.utils.transforms.ScaleImageTensor'}, {'_target_': 'torchvision.transforms.Normalize', 'mean': [0.48145466, 0.4578275, 0.40821073], 'std': [0.26862954, 0.26130258, 0.27577711]}]}}, '_target_': 'flower.datasets.hulc_data_module.HulcDataModule', '_recursive_': False, 'root_data_dir': '${root_data_dir}', 'action_space': 7, 'action_max': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 'action_min': [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1], 'shuffle_val': False, 'observation_space': {'rgb_obs': ['rgb_static', 'rgb_gripper'], 'depth_obs': [], 'state_obs': ['robot_obs'], 'actions': ['rel_actions'], 'language': ['language']}, 'proprioception_dims': {'n_state_obs': 8, 'keep_indices': [[0, 7], [14, 15]], 'robot_orientation_idx': [3, 6], 'normalize': True, 'normalize_robot_orientation': True}, 'datasets': {'lang_dataset': {'_target_': 'flower.datasets.disk_dataset.ExtendedDiskDataset', 'key': 'lang', 'save_format': 'npz', 'batch_size': '${batch_size}', 'min_window_size': '${act_seq_len}', 'max_window_size': '${act_seq_len}', 'proprio_state': '${datamodule.proprioception_dims}', 'obs_space': '${datamodule.observation_space}', 'skip_frames': 1, 'pad': False, 'lang_folder': '${lang_folder}', 'aux_lang_loss_window': 8, 'num_workers': '${num_workers}', 'action_seq_len': '${act_seq_len}', 'obs_seq_len': '${obs_seq_len}', 'future_range': 1, 'use_extracted_rel_actions': '${use_extracted_rel_actions}'}}}, 'model': {'_target_': 'flower.models.flower.FLOWERVLA', '_recursive_': False, 'vlm_path': 'microsoft/Florence-2-large', 'freeze_florence': False, 'freeze_vision_tower': False, 'vlm_prompt_style': 'default', 'token_dropout': 0.1, 'multistep': '${multistep}', 'num_sampling_steps': 4, 'lowdim_obs_dim': 7, 'action_dim': 7, 'act_window_size': 10, 'load_pretrained': True, 'pretrained_model_path': '/home/hk-project-sustainebot/ft4740/code/flower_vla_policy/logs/runs/2025-02-05/10-17-02/360000_model_weights.pt', 'use_second_view': True, 'second_view_key': 'image_wrist', 'action_type_adaln': True, 'use_causal_attention': True, 'use_cross_attn': True, 'use_adaln_cond': False, 'use_readout_token': False, 'use_proprio': False, 'return_act_chunk': False, 'sampling_type': 'uniform', 'dit_dim': 1024, 'n_heads': 16, 'n_layers': 18, 'attn_pdrop': 0.1, 'resid_pdrop': 0.1, 'mlp_pdrop': 0.1, 'use_rope': True, 'use_nope': False, 'query_seq_len': 100, 'rope_theta': 32.0, 'optimizer_type': 'adamw', 'optimizer': {'_target_': 'torch.optim.AdamW', 'transformer_weight_decay': 0.05, 'learning_rate': 2e-05, 'betas': [0.9, 0.95]}, 'lr_scheduler': {'lr_scheduler': {'init_lr': 2e-05, 'init_lr_scale': 0.1, 'final_lr_scale': 0.1, 'total_steps': 50000, 'phase_ratio': '(0.05, 0.1, 0.85)', 'lr': 5e-05}}}, 'root_data_dir': '/hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D', 'lang_folder': 'lang_clip_resnet50', 'log_dir': './logs', 'slurm': False, 'seed': 242, 'device': 'cuda', 'batch_size': 8, 'devices': 4, 'act_dim': 7, 'proprio_dims': 7, 'obs_dim': 512, 'goal_dim': 512, 'obs_seq_len': 1, 'act_seq_len': 10, 'multistep': 10, 'p_last_state': 0, 'gen_img_res': 112, 'max_epochs': 35, 'rollout_lh_skip_epochs': 9, 'num_workers': 12, 'benchmark_name': 'calvin_abc', 'use_extracted_rel_actions': True, 'trainer': {'devices': '${devices}', 'precision': 'bf16-mixed', 'max_epochs': '${max_epochs}', 'sync_batchnorm': True, 'accelerator': 'gpu', 'strategy': 'ddp', 'limit_train_batches': 1000, 'limit_val_batches': 4}, 'logger': {'_target_': 'pytorch_lightning.loggers.WandbLogger', 'save_dir': '.', 'name': '2025-02-20/17-55-09_seed242', 'group': '2025-02-20', 'log_model': False, 'project': '${benchmark_name}', 'entity': 'bennoq', 'id': '2025-02-20_17-55-09_seed242'}}
[2025-02-20 17:55:31,538][__main__][INFO] - Git commit: 0cc472e37ca44ca89bf334e692a2dee801bb8e91
[2025-02-20 17:55:31,697][__main__][INFO] - * CUDA:
	- GPU:
		- NVIDIA A100-SXM4-40GB
		- NVIDIA A100-SXM4-40GB
		- NVIDIA A100-SXM4-40GB
		- NVIDIA A100-SXM4-40GB
	- available:         True
	- version:           12.1
* Packages:
	- numpy:             1.26.4
	- pyTorch_debug:     False
	- pyTorch_version:   2.2.2+cu121
	- pytorch-lightning: 2.0.8
	- tqdm:              4.67.1
[2025-02-20 17:55:31,837][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:31,837][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:31,837][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:31,838][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 17:55:42,746][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training
[2025-02-20 17:55:42,746][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training
[2025-02-20 17:55:42,746][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training
[2025-02-20 17:55:42,746][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 17:55:42,746][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 17:55:42,746][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 17:55:42,746][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training
[2025-02-20 17:55:42,746][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 17:55:43,723][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training/extracted
[2025-02-20 17:55:43,723][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training/extracted
[2025-02-20 17:55:43,723][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training/extracted
[2025-02-20 17:55:43,723][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training/extracted
[2025-02-20 17:55:43,747][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation
[2025-02-20 17:55:43,747][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 17:55:43,747][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation
[2025-02-20 17:55:43,747][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation
[2025-02-20 17:55:43,747][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 17:55:43,747][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation
[2025-02-20 17:55:43,747][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 17:55:43,747][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 17:55:43,888][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation/extracted
[2025-02-20 17:55:43,888][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation/extracted
[2025-02-20 17:55:43,889][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation/extracted
[2025-02-20 17:55:43,889][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation/extracted
[2025-02-20 17:55:47,940][flower.wrappers.hulc_wrapper][INFO] - EGL_DEVICE_ID 0 <==> CUDA_DEVICE_ID 0
[2025-02-20 17:55:48,028][calvin_env.envs.play_table_env][INFO] - Loading EGL plugin (may segfault on misconfigured systems)...
[2025-02-20 17:55:49,128][calvin_env.envs.play_table_env][INFO] - Successfully loaded egl plugin
[2025-02-20 17:55:49,128][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 17:55:49,130][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 17:55:49,144][calvin_env.robot.robot][INFO] - Loading robot
[2025-02-20 17:55:49,192][flower.wrappers.hulc_wrapper][INFO] - EGL_DEVICE_ID 1 <==> CUDA_DEVICE_ID 1
[2025-02-20 17:55:49,261][calvin_env.envs.play_table_env][INFO] - Loading EGL plugin (may segfault on misconfigured systems)...
[2025-02-20 17:55:49,291][calvin_env.envs.play_table_env][INFO] - Resetting simulation
[2025-02-20 17:55:49,292][calvin_env.envs.play_table_env][INFO] - Setting gravity
[2025-02-20 17:55:49,624][calvin_env.envs.play_table_env][INFO] - Successfully loaded egl plugin
[2025-02-20 17:55:49,624][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 17:55:49,625][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 17:55:49,632][calvin_env.robot.robot][INFO] - Loading robot
[2025-02-20 17:55:49,643][calvin_env.envs.play_table_env][INFO] - Resetting simulation
[2025-02-20 17:55:49,644][calvin_env.envs.play_table_env][INFO] - Setting gravity
[2025-02-20 17:55:49,883][flower.wrappers.hulc_wrapper][INFO] - Initialized PlayTableEnv for device cuda:0
[2025-02-20 17:55:49,898][flower.evaluation.multistep_sequences][INFO] - Start generating evaluation sequences.
[2025-02-20 17:55:49,900][flower.wrappers.hulc_wrapper][INFO] - EGL_DEVICE_ID 2 <==> CUDA_DEVICE_ID 2
[2025-02-20 17:55:49,969][calvin_env.envs.play_table_env][INFO] - Loading EGL plugin (may segfault on misconfigured systems)...
[2025-02-20 17:55:50,109][flower.wrappers.hulc_wrapper][INFO] - Initialized PlayTableEnv for device cuda:1
[2025-02-20 17:55:50,115][flower.evaluation.multistep_sequences][INFO] - Start generating evaluation sequences.
[2025-02-20 17:55:50,318][flower.wrappers.hulc_wrapper][INFO] - EGL_DEVICE_ID 3 <==> CUDA_DEVICE_ID 3
[2025-02-20 17:55:50,349][calvin_env.envs.play_table_env][INFO] - Successfully loaded egl plugin
[2025-02-20 17:55:50,350][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 17:55:50,351][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 17:55:50,358][calvin_env.robot.robot][INFO] - Loading robot
[2025-02-20 17:55:50,368][calvin_env.envs.play_table_env][INFO] - Resetting simulation
[2025-02-20 17:55:50,369][calvin_env.envs.play_table_env][INFO] - Setting gravity
[2025-02-20 17:55:50,386][calvin_env.envs.play_table_env][INFO] - Loading EGL plugin (may segfault on misconfigured systems)...
[2025-02-20 17:55:50,649][calvin_env.envs.play_table_env][INFO] - Successfully loaded egl plugin
[2025-02-20 17:55:50,649][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 17:55:50,651][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 17:55:50,657][calvin_env.robot.robot][INFO] - Loading robot
[2025-02-20 17:55:50,668][calvin_env.envs.play_table_env][INFO] - Resetting simulation
[2025-02-20 17:55:50,669][calvin_env.envs.play_table_env][INFO] - Setting gravity
[2025-02-20 17:55:50,775][flower.wrappers.hulc_wrapper][INFO] - Initialized PlayTableEnv for device cuda:2
[2025-02-20 17:55:50,780][flower.evaluation.multistep_sequences][INFO] - Start generating evaluation sequences.
[2025-02-20 17:55:51,050][flower.wrappers.hulc_wrapper][INFO] - Initialized PlayTableEnv for device cuda:3
[2025-02-20 17:55:51,056][flower.evaluation.multistep_sequences][INFO] - Start generating evaluation sequences.
[2025-02-20 17:56:07,918][flower.evaluation.multistep_sequences][INFO] - Done generating evaluation sequences.
[2025-02-20 17:56:07,984][flower.evaluation.multistep_sequences][INFO] - Done generating evaluation sequences.
[2025-02-20 17:56:08,452][flower.evaluation.multistep_sequences][INFO] - Done generating evaluation sequences.
[2025-02-20 17:56:08,577][flower.evaluation.multistep_sequences][INFO] - Done generating evaluation sequences.
[2025-02-20 17:56:09,481][root][INFO] - Creating EMA weights copy.
[2025-02-20 17:56:09,481][root][INFO] - Creating EMA weights copy.
[2025-02-20 17:56:09,481][root][INFO] - Creating EMA weights copy.
[2025-02-20 17:56:09,481][root][INFO] - Creating EMA weights copy.
[2025-02-20 19:41:56,252][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 994 / 1000 sequences, SR: 99.4%
[2025-02-20 19:41:56,254][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 940 / 1000 sequences, SR: 94.0%
[2025-02-20 19:41:56,254][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 857 / 1000 sequences, SR: 85.7%
[2025-02-20 19:41:56,255][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 760 / 1000 sequences, SR: 76.0%
[2025-02-20 19:41:56,255][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 661 / 1000 sequences, SR: 66.1%
[2025-02-20 19:41:56,256][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.2
[2025-02-20 20:52:47,125][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 996 / 1000 sequences, SR: 99.6%
[2025-02-20 20:52:47,126][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 945 / 1000 sequences, SR: 94.5%
[2025-02-20 20:52:47,126][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 859 / 1000 sequences, SR: 85.9%
[2025-02-20 20:52:47,127][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 764 / 1000 sequences, SR: 76.4%
[2025-02-20 20:52:47,127][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 681 / 1000 sequences, SR: 68.1%
[2025-02-20 20:52:47,128][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.2
[2025-02-20 22:02:57,576][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 997 / 1000 sequences, SR: 99.7%
[2025-02-20 22:02:57,577][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 962 / 1000 sequences, SR: 96.2%
[2025-02-20 22:02:57,577][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 890 / 1000 sequences, SR: 89.0%
[2025-02-20 22:02:57,578][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 820 / 1000 sequences, SR: 82.0%
[2025-02-20 22:02:57,578][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 736 / 1000 sequences, SR: 73.6%
[2025-02-20 22:02:57,579][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.4
[2025-02-20 23:12:59,280][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 997 / 1000 sequences, SR: 99.7%
[2025-02-20 23:12:59,281][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 965 / 1000 sequences, SR: 96.5%
[2025-02-20 23:12:59,281][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 900 / 1000 sequences, SR: 90.0%
[2025-02-20 23:12:59,281][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 826 / 1000 sequences, SR: 82.6%
[2025-02-20 23:12:59,282][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 750 / 1000 sequences, SR: 75.0%
[2025-02-20 23:12:59,282][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.4
[2025-02-21 00:22:01,055][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 997 / 1000 sequences, SR: 99.7%
[2025-02-21 00:22:01,056][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 972 / 1000 sequences, SR: 97.2%
[2025-02-21 00:22:01,056][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 921 / 1000 sequences, SR: 92.1%
[2025-02-21 00:22:01,056][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 862 / 1000 sequences, SR: 86.2%
[2025-02-21 00:22:01,057][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 785 / 1000 sequences, SR: 78.5%
[2025-02-21 00:22:01,057][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.5
[2025-02-21 01:32:42,416][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 993 / 1000 sequences, SR: 99.3%
[2025-02-21 01:32:42,417][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 959 / 1000 sequences, SR: 95.9%
[2025-02-21 01:32:42,417][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 905 / 1000 sequences, SR: 90.5%
[2025-02-21 01:32:42,417][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 848 / 1000 sequences, SR: 84.8%
[2025-02-21 01:32:42,418][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 775 / 1000 sequences, SR: 77.5%
[2025-02-21 01:32:42,418][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.5
[2025-02-21 01:32:46,517][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 01:32:46,517][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 01:32:46,517][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 01:32:46,518][__main__][INFO] - GPU 3 Memory: Allocated: 3.73GB, Reserved: 3.83GB
[2025-02-21 01:32:46,526][__main__][INFO] - GPU 0 Memory: Allocated: 3.73GB, Reserved: 3.81GB
[2025-02-21 01:32:46,526][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 01:32:46,527][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 01:32:46,527][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 01:32:46,531][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 01:32:46,531][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 01:32:46,532][__main__][INFO] - GPU 2 Memory: Allocated: 3.73GB, Reserved: 3.81GB
[2025-02-21 01:32:46,532][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 01:32:46,589][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 01:32:46,590][__main__][INFO] - GPU 1 Memory: Allocated: 3.73GB, Reserved: 3.83GB
[2025-02-21 01:32:46,590][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 01:32:46,590][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB