training.log

[2025-02-20 16:59:06,623][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,624][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,624][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,624][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,624][__main__][INFO] - 
Initializing training for seed 242
[2025-02-20 16:59:06,628][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,628][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,628][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,628][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,630][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,630][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,630][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,630][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,631][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,632][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,632][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:06,632][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:19,503][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:19,503][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:19,504][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:19,504][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:19,533][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:19,533][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:19,533][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:19,533][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:19,536][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:19,536][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:19,537][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:19,537][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:24,921][__main__][INFO] - Training config for seed 242:
{'callbacks': {'rollout_lh': {'tasks': {'_target_': 'calvin_env.envs.tasks.Tasks', 'tasks': {'rotate_red_block_right': ['rotate_object', 'block_red', -60], 'rotate_red_block_left': ['rotate_object', 'block_red', 60], 'rotate_blue_block_right': ['rotate_object', 'block_blue', -60], 'rotate_blue_block_left': ['rotate_object', 'block_blue', 60], 'rotate_pink_block_right': ['rotate_object', 'block_pink', -60], 'rotate_pink_block_left': ['rotate_object', 'block_pink', 60], 'push_red_block_right': ['push_object', 'block_red', 0.1, 0], 'push_red_block_left': ['push_object', 'block_red', -0.1, 0], 'push_blue_block_right': ['push_object', 'block_blue', 0.1, 0], 'push_blue_block_left': ['push_object', 'block_blue', -0.1, 0], 'push_pink_block_right': ['push_object', 'block_pink', 0.1, 0], 'push_pink_block_left': ['push_object', 'block_pink', -0.1, 0], 'move_slider_left': ['move_door_rel', 'base__slide', 0.15], 'move_slider_right': ['move_door_rel', 'base__slide', -0.15], 'open_drawer': ['move_door_rel', 'base__drawer', 0.12], 'close_drawer': ['move_door_rel', 'base__drawer', -0.12], 'lift_red_block_table': ['lift_object', 'block_red', 0.05, 'table', 'base_link'], 'lift_red_block_slider': ['lift_object', 'block_red', 0.03, 'table', 'plank_link'], 'lift_red_block_drawer': ['lift_object', 'block_red', 0.05, 'table', 'drawer_link'], 'lift_blue_block_table': ['lift_object', 'block_blue', 0.05, 'table', 'base_link'], 'lift_blue_block_slider': ['lift_object', 'block_blue', 0.03, 'table', 'plank_link'], 'lift_blue_block_drawer': ['lift_object', 'block_blue', 0.05, 'table', 'drawer_link'], 'lift_pink_block_table': ['lift_object', 'block_pink', 0.05, 'table', 'base_link'], 'lift_pink_block_slider': ['lift_object', 'block_pink', 0.03, 'table', 'plank_link'], 'lift_pink_block_drawer': ['lift_object', 'block_pink', 0.05, 'table', 'drawer_link'], 'place_in_slider': ['place_object', 'table', 'plank_link'], 'place_in_drawer': ['place_object', 'table', 'drawer_link'], 'stack_block': ['stack_objects'], 'unstack_block': ['unstack_objects'], 'turn_on_lightbulb': ['toggle_light', 'lightbulb', 0, 1], 'turn_off_lightbulb': ['toggle_light', 'lightbulb', 1, 0], 'turn_on_led': ['toggle_light', 'led', 0, 1], 'turn_off_led': ['toggle_light', 'led', 1, 0], 'push_into_drawer': ['push_object_into', ['block_red', 'block_blue', 'block_pink'], 'table', 'base_link', 'table', 'drawer_link']}}, 'val_annotations': {'rotate_red_block_right': ['take the red block and rotate it to the right'], 'rotate_red_block_left': ['take the red block and rotate it to the left'], 'rotate_blue_block_right': ['take the blue block and rotate it to the right'], 'rotate_blue_block_left': ['take the blue block and rotate it to the left'], 'rotate_pink_block_right': ['take the pink block and rotate it to the right'], 'rotate_pink_block_left': ['take the pink block and rotate it to the left'], 'push_red_block_right': ['go push the red block right'], 'push_red_block_left': ['go push the red block left'], 'push_blue_block_right': ['go push the blue block right'], 'push_blue_block_left': ['go push the blue block left'], 'push_pink_block_right': ['go push the pink block right'], 'push_pink_block_left': ['go push the pink block left'], 'move_slider_left': ['push the sliding door to the left side'], 'move_slider_right': ['push the sliding door to the right side'], 'open_drawer': ['pull the handle to open the drawer'], 'close_drawer': ['push the handle to close the drawer'], 'lift_red_block_table': ['grasp and lift the red block'], 'lift_blue_block_table': ['grasp and lift the blue block'], 'lift_pink_block_table': ['grasp and lift the pink block'], 'lift_red_block_slider': ['lift the red block from the sliding cabinet'], 'lift_blue_block_slider': ['lift the blue block from the sliding cabinet'], 'lift_pink_block_slider': ['lift the pink block from the sliding cabinet'], 'lift_red_block_drawer': ['Take the red block from the drawer'], 'lift_blue_block_drawer': ['Take the blue block from the drawer'], 'lift_pink_block_drawer': ['Take the pink block from the drawer'], 'place_in_slider': ['store the grasped block in the sliding cabinet'], 'place_in_drawer': ['store the grasped block in the drawer'], 'push_into_drawer': ['slide the block that it falls into the drawer'], 'stack_block': ['stack the grasped block'], 'unstack_block': ['remove the stacked block'], 'turn_on_lightbulb': ['use the switch to turn on the light bulb'], 'turn_off_lightbulb': ['use the switch to turn off the light bulb'], 'turn_on_led': ['press the button to turn on the led light'], 'turn_off_led': ['press the button to turn off the led light']}, '_target_': 'flower.rollout.rollout_long_horizon.RolloutLongHorizon', '_recursive_': False, 'env_cfg': {'_target_': 'flower.wrappers.hulc_wrapper.HulcWrapper'}, 'skip_epochs': '${rollout_lh_skip_epochs}', 'rollout_freq': 5, 'num_videos': 0, 'num_sequences': 1000, 'replan_freq': 30, 'ep_len': 360, 'empty_cache': False, 'log_video_to_file': False, 'save_dir': './videos', 'lang_folder': '${lang_folder}', 'debug': False}, 'ema': {'_target_': 'flower.callbacks.ema.EMA', 'decay': 0.999, 'start_step': 0, 'save_ema_weights_in_callback_state': True, 'evaluate_ema_weights_instead': True, 'power': 0.6666666666666666, 'inv_gamma': 1.0, 'min_value': 0.0, 'max_value': 0.9999}, 'checkpoint': {'_target_': 'pytorch_lightning.callbacks.ModelCheckpoint', 'save_top_k': 1, 'verbose': True, 'monitor': 'eval_lh/avg_seq_len', 'mode': 'max', 'dirpath': 'saved_models', 'filename': '{epoch:02d}_{eval_lh/avg_seq_len:.2f}', 'every_n_epochs': '${callbacks.rollout_lh.rollout_freq}'}}, 'datamodule': {'transforms': {'train': {'rgb_static': [{'_target_': 'torchvision.transforms.Resize', 'size': 224, 'antialias': True}, {'_target_': 'flower.utils.transforms.RandomShiftsAug', 'pad': 10}, {'_target_': 'flower.utils.transforms.ScaleImageTensor'}, {'_target_': 'torchvision.transforms.Normalize', 'mean': [0.48145466, 0.4578275, 0.40821073], 'std': [0.26862954, 0.26130258, 0.27577711]}], 'rgb_gripper': [{'_target_': 'torchvision.transforms.Resize', 'size': 224, 'antialias': True}, {'_target_': 'flower.utils.transforms.RandomShiftsAug', 'pad': 4}, {'_target_': 'flower.utils.transforms.ScaleImageTensor'}, {'_target_': 'torchvision.transforms.Normalize', 'mean': [0.48145466, 0.4578275, 0.40821073], 'std': [0.26862954, 0.26130258, 0.27577711]}]}, 'val': {'rgb_static': [{'_target_': 'torchvision.transforms.Resize', 'size': 224, 'antialias': True}, {'_target_': 'flower.utils.transforms.ScaleImageTensor'}, {'_target_': 'torchvision.transforms.Normalize', 'mean': [0.48145466, 0.4578275, 0.40821073], 'std': [0.26862954, 0.26130258, 0.27577711]}], 'rgb_gripper': [{'_target_': 'torchvision.transforms.Resize', 'size': 224, 'antialias': True}, {'_target_': 'flower.utils.transforms.ScaleImageTensor'}, {'_target_': 'torchvision.transforms.Normalize', 'mean': [0.48145466, 0.4578275, 0.40821073], 'std': [0.26862954, 0.26130258, 0.27577711]}]}}, '_target_': 'flower.datasets.hulc_data_module.HulcDataModule', '_recursive_': False, 'root_data_dir': '${root_data_dir}', 'action_space': 7, 'action_max': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 'action_min': [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1], 'shuffle_val': False, 'observation_space': {'rgb_obs': ['rgb_static', 'rgb_gripper'], 'depth_obs': [], 'state_obs': ['robot_obs'], 'actions': ['rel_actions'], 'language': ['language']}, 'proprioception_dims': {'n_state_obs': 8, 'keep_indices': [[0, 7], [14, 15]], 'robot_orientation_idx': [3, 6], 'normalize': True, 'normalize_robot_orientation': True}, 'datasets': {'lang_dataset': {'_target_': 'flower.datasets.disk_dataset.ExtendedDiskDataset', 'key': 'lang', 'save_format': 'npz', 'batch_size': '${batch_size}', 'min_window_size': '${act_seq_len}', 'max_window_size': '${act_seq_len}', 'proprio_state': '${datamodule.proprioception_dims}', 'obs_space': '${datamodule.observation_space}', 'skip_frames': 1, 'pad': False, 'lang_folder': '${lang_folder}', 'aux_lang_loss_window': 8, 'num_workers': '${num_workers}', 'action_seq_len': '${act_seq_len}', 'obs_seq_len': '${obs_seq_len}', 'future_range': 1, 'use_extracted_rel_actions': '${use_extracted_rel_actions}'}}}, 'model': {'_target_': 'flower.models.flower.FLOWERVLA', '_recursive_': False, 'vlm_path': 'microsoft/Florence-2-large', 'freeze_florence': False, 'freeze_vision_tower': False, 'vlm_prompt_style': 'default', 'token_dropout': 0.1, 'multistep': '${multistep}', 'num_sampling_steps': 4, 'lowdim_obs_dim': 7, 'action_dim': 7, 'act_window_size': 10, 'load_pretrained': True, 'pretrained_model_path': '/home/hk-project-sustainebot/ft4740/code/flower_vla/logs/runs/2025-02-19/11-15-41/200000_model_weights.pt', 'use_second_view': True, 'second_view_key': 'image_wrist', 'action_type_adaln': True, 'use_causal_attention': True, 'use_cross_attn': True, 'use_adaln_cond': False, 'use_readout_token': False, 'use_proprio': False, 'return_act_chunk': False, 'sampling_type': 'uniform', 'dit_dim': 1024, 'n_heads': 16, 'n_layers': 18, 'attn_pdrop': 0.1, 'resid_pdrop': 0.1, 'mlp_pdrop': 0.1, 'use_rope': True, 'use_nope': False, 'query_seq_len': 120, 'rope_theta': 32.0, 'optimizer_type': 'adamw', 'optimizer': {'_target_': 'torch.optim.AdamW', 'transformer_weight_decay': 0.05, 'learning_rate': 2e-05, 'betas': [0.9, 0.95]}, 'lr_scheduler': {'lr_scheduler': {'init_lr': 2e-05, 'init_lr_scale': 0.1, 'final_lr_scale': 0.1, 'total_steps': 50000, 'phase_ratio': '(0.05, 0.1, 0.85)', 'lr': 5e-05}}}, 'root_data_dir': '/hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D', 'lang_folder': 'lang_clip_resnet50', 'log_dir': './logs', 'slurm': False, 'seed': 242, 'device': 'cuda', 'batch_size': 8, 'devices': 4, 'act_dim': 7, 'proprio_dims': 7, 'obs_dim': 512, 'goal_dim': 512, 'obs_seq_len': 1, 'act_seq_len': 10, 'multistep': 10, 'p_last_state': 0, 'gen_img_res': 112, 'max_epochs': 35, 'rollout_lh_skip_epochs': 9, 'num_workers': 12, 'benchmark_name': 'calvin_abc', 'use_extracted_rel_actions': True, 'trainer': {'devices': '${devices}', 'precision': 'bf16-mixed', 'max_epochs': '${max_epochs}', 'sync_batchnorm': True, 'accelerator': 'gpu', 'strategy': 'ddp', 'limit_train_batches': 1000, 'limit_val_batches': 4}, 'logger': {'_target_': 'pytorch_lightning.loggers.WandbLogger', 'save_dir': '.', 'name': '2025-02-20/16-59-06_seed242', 'group': '2025-02-20', 'log_model': False, 'project': '${benchmark_name}', 'entity': 'bennoq', 'id': '2025-02-20_16-59-06_seed242'}}
[2025-02-20 16:59:25,297][__main__][INFO] - Git commit: 0cc472e37ca44ca89bf334e692a2dee801bb8e91
[2025-02-20 16:59:25,456][__main__][INFO] - * CUDA:
	- GPU:
		- NVIDIA A100-SXM4-40GB
		- NVIDIA A100-SXM4-40GB
		- NVIDIA A100-SXM4-40GB
		- NVIDIA A100-SXM4-40GB
	- available:         True
	- version:           12.1
* Packages:
	- numpy:             1.26.4
	- pyTorch_debug:     False
	- pyTorch_version:   2.2.2+cu121
	- pytorch-lightning: 2.0.8
	- tqdm:              4.67.1
[2025-02-20 16:59:25,594][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:25,595][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:25,595][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:25,595][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-20 16:59:38,179][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training
[2025-02-20 16:59:38,179][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 16:59:38,179][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training
[2025-02-20 16:59:38,179][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 16:59:38,179][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training
[2025-02-20 16:59:38,179][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 16:59:38,182][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training
[2025-02-20 16:59:38,182][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 16:59:38,940][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training/extracted
[2025-02-20 16:59:38,940][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training/extracted
[2025-02-20 16:59:38,940][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training/extracted
[2025-02-20 16:59:38,940][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/training/extracted
[2025-02-20 16:59:38,963][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation
[2025-02-20 16:59:38,963][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation
[2025-02-20 16:59:38,963][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 16:59:38,963][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 16:59:38,963][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation
[2025-02-20 16:59:38,963][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 16:59:38,964][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation
[2025-02-20 16:59:38,964][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-02-20 16:59:39,222][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation/extracted
[2025-02-20 16:59:39,222][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation/extracted
[2025-02-20 16:59:39,222][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation/extracted
[2025-02-20 16:59:39,222][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D/validation/extracted
[2025-02-20 16:59:43,088][flower.wrappers.hulc_wrapper][INFO] - EGL_DEVICE_ID 0 <==> CUDA_DEVICE_ID 0
[2025-02-20 16:59:43,160][calvin_env.envs.play_table_env][INFO] - Loading EGL plugin (may segfault on misconfigured systems)...
[2025-02-20 16:59:43,802][calvin_env.envs.play_table_env][INFO] - Successfully loaded egl plugin
[2025-02-20 16:59:43,803][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 16:59:43,804][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 16:59:43,865][calvin_env.robot.robot][INFO] - Loading robot
[2025-02-20 16:59:43,940][calvin_env.envs.play_table_env][INFO] - Resetting simulation
[2025-02-20 16:59:43,941][calvin_env.envs.play_table_env][INFO] - Setting gravity
[2025-02-20 16:59:43,987][flower.wrappers.hulc_wrapper][INFO] - EGL_DEVICE_ID 1 <==> CUDA_DEVICE_ID 1
[2025-02-20 16:59:44,058][calvin_env.envs.play_table_env][INFO] - Loading EGL plugin (may segfault on misconfigured systems)...
[2025-02-20 16:59:44,430][flower.wrappers.hulc_wrapper][INFO] - EGL_DEVICE_ID 2 <==> CUDA_DEVICE_ID 2
[2025-02-20 16:59:44,474][calvin_env.envs.play_table_env][INFO] - Successfully loaded egl plugin
[2025-02-20 16:59:44,475][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 16:59:44,476][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 16:59:44,482][calvin_env.robot.robot][INFO] - Loading robot
[2025-02-20 16:59:44,494][calvin_env.envs.play_table_env][INFO] - Resetting simulation
[2025-02-20 16:59:44,494][calvin_env.envs.play_table_env][INFO] - Setting gravity
[2025-02-20 16:59:44,499][calvin_env.envs.play_table_env][INFO] - Loading EGL plugin (may segfault on misconfigured systems)...
[2025-02-20 16:59:44,739][calvin_env.envs.play_table_env][INFO] - Successfully loaded egl plugin
[2025-02-20 16:59:44,739][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 16:59:44,740][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 16:59:44,747][calvin_env.robot.robot][INFO] - Loading robot
[2025-02-20 16:59:44,758][calvin_env.envs.play_table_env][INFO] - Resetting simulation
[2025-02-20 16:59:44,759][calvin_env.envs.play_table_env][INFO] - Setting gravity
[2025-02-20 16:59:44,859][flower.wrappers.hulc_wrapper][INFO] - Initialized PlayTableEnv for device cuda:0
[2025-02-20 16:59:44,866][flower.evaluation.multistep_sequences][INFO] - Start generating evaluation sequences.
[2025-02-20 16:59:44,916][flower.wrappers.hulc_wrapper][INFO] - Initialized PlayTableEnv for device cuda:1
[2025-02-20 16:59:44,921][flower.evaluation.multistep_sequences][INFO] - Start generating evaluation sequences.
[2025-02-20 16:59:44,996][flower.wrappers.hulc_wrapper][INFO] - EGL_DEVICE_ID 3 <==> CUDA_DEVICE_ID 3
[2025-02-20 16:59:45,063][calvin_env.envs.play_table_env][INFO] - Loading EGL plugin (may segfault on misconfigured systems)...
[2025-02-20 16:59:45,160][flower.wrappers.hulc_wrapper][INFO] - Initialized PlayTableEnv for device cuda:2
[2025-02-20 16:59:45,166][flower.evaluation.multistep_sequences][INFO] - Start generating evaluation sequences.
[2025-02-20 16:59:45,250][calvin_env.envs.play_table_env][INFO] - Successfully loaded egl plugin
[2025-02-20 16:59:45,250][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 16:59:45,251][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-02-20 16:59:45,258][calvin_env.robot.robot][INFO] - Loading robot
[2025-02-20 16:59:45,268][calvin_env.envs.play_table_env][INFO] - Resetting simulation
[2025-02-20 16:59:45,269][calvin_env.envs.play_table_env][INFO] - Setting gravity
[2025-02-20 16:59:45,647][flower.wrappers.hulc_wrapper][INFO] - Initialized PlayTableEnv for device cuda:3
[2025-02-20 16:59:45,652][flower.evaluation.multistep_sequences][INFO] - Start generating evaluation sequences.
[2025-02-20 17:00:02,685][flower.evaluation.multistep_sequences][INFO] - Done generating evaluation sequences.
[2025-02-20 17:00:02,705][flower.evaluation.multistep_sequences][INFO] - Done generating evaluation sequences.
[2025-02-20 17:00:02,945][flower.evaluation.multistep_sequences][INFO] - Done generating evaluation sequences.
[2025-02-20 17:00:03,750][flower.evaluation.multistep_sequences][INFO] - Done generating evaluation sequences.
[2025-02-20 17:00:04,633][root][INFO] - Creating EMA weights copy.
[2025-02-20 17:00:04,633][root][INFO] - Creating EMA weights copy.
[2025-02-20 17:00:04,633][root][INFO] - Creating EMA weights copy.
[2025-02-20 17:00:04,633][root][INFO] - Creating EMA weights copy.
[2025-02-20 18:47:58,516][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 975 / 1000 sequences, SR: 97.5%
[2025-02-20 18:47:58,517][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 889 / 1000 sequences, SR: 88.9%
[2025-02-20 18:47:58,518][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 786 / 1000 sequences, SR: 78.6%
[2025-02-20 18:47:58,518][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 680 / 1000 sequences, SR: 68.0%
[2025-02-20 18:47:58,519][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 576 / 1000 sequences, SR: 57.6%
[2025-02-20 18:47:58,520][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 3.9
[2025-02-20 19:58:54,135][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 992 / 1000 sequences, SR: 99.2%
[2025-02-20 19:58:54,136][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 938 / 1000 sequences, SR: 93.8%
[2025-02-20 19:58:54,137][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 851 / 1000 sequences, SR: 85.1%
[2025-02-20 19:58:54,137][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 766 / 1000 sequences, SR: 76.6%
[2025-02-20 19:58:54,137][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 667 / 1000 sequences, SR: 66.7%
[2025-02-20 19:58:54,138][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.2
[2025-02-20 21:10:27,963][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 964 / 1000 sequences, SR: 96.4%
[2025-02-20 21:10:27,963][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 896 / 1000 sequences, SR: 89.6%
[2025-02-20 21:10:27,964][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 802 / 1000 sequences, SR: 80.2%
[2025-02-20 21:10:27,964][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 716 / 1000 sequences, SR: 71.6%
[2025-02-20 21:10:27,965][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 617 / 1000 sequences, SR: 61.7%
[2025-02-20 21:10:27,965][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.0
[2025-02-20 22:19:44,108][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 988 / 1000 sequences, SR: 98.8%
[2025-02-20 22:19:44,108][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 943 / 1000 sequences, SR: 94.3%
[2025-02-20 22:19:44,109][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 873 / 1000 sequences, SR: 87.3%
[2025-02-20 22:19:44,109][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 796 / 1000 sequences, SR: 79.6%
[2025-02-20 22:19:44,110][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 724 / 1000 sequences, SR: 72.4%
[2025-02-20 22:19:44,110][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.3
[2025-02-20 23:29:22,469][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 984 / 1000 sequences, SR: 98.4%
[2025-02-20 23:29:22,470][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 940 / 1000 sequences, SR: 94.0%
[2025-02-20 23:29:22,470][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 879 / 1000 sequences, SR: 87.9%
[2025-02-20 23:29:22,471][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 817 / 1000 sequences, SR: 81.7%
[2025-02-20 23:29:22,471][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 741 / 1000 sequences, SR: 74.1%
[2025-02-20 23:29:22,472][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.4
[2025-02-21 00:39:37,121][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 978 / 1000 sequences, SR: 97.8%
[2025-02-21 00:39:37,122][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 925 / 1000 sequences, SR: 92.5%
[2025-02-21 00:39:37,122][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 868 / 1000 sequences, SR: 86.8%
[2025-02-21 00:39:37,123][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 793 / 1000 sequences, SR: 79.3%
[2025-02-21 00:39:37,123][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 722 / 1000 sequences, SR: 72.2%
[2025-02-21 00:39:37,124][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.3
[2025-02-21 00:39:41,371][__main__][INFO] - GPU 0 Memory: Allocated: 3.73GB, Reserved: 3.79GB
[2025-02-21 00:39:41,372][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 00:39:41,372][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 00:39:41,372][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 00:39:41,427][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 00:39:41,428][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 00:39:41,428][__main__][INFO] - GPU 2 Memory: Allocated: 3.73GB, Reserved: 3.83GB
[2025-02-21 00:39:41,428][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 00:39:41,451][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 00:39:41,451][__main__][INFO] - GPU 1 Memory: Allocated: 3.73GB, Reserved: 3.79GB
[2025-02-21 00:39:41,451][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 00:39:41,451][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 00:39:41,456][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 00:39:41,457][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 00:39:41,457][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-02-21 00:39:41,457][__main__][INFO] - GPU 3 Memory: Allocated: 3.73GB, Reserved: 3.81GB