training.logs

[2025-03-13 16:21:05,614][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,614][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,614][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,614][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,615][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,615][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,615][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,615][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,616][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,616][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,616][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,616][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,619][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,620][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,620][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,620][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:05,620][__main__][INFO] - 
Initializing training for seed 342
[2025-03-13 16:21:23,227][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:23,228][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:23,228][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:23,228][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:23,240][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:23,241][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:23,241][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:23,241][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:23,249][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:23,250][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:23,250][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:23,250][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:28,039][__main__][INFO] - Training config for seed 342:
{'callbacks': {'rollout_lh': {'tasks': {'_target_': 'calvin_env.envs.tasks.Tasks', 'tasks': {'rotate_red_block_right': ['rotate_object', 'block_red', -60], 'rotate_red_block_left': ['rotate_object', 'block_red', 60], 'rotate_blue_block_right': ['rotate_object', 'block_blue', -60], 'rotate_blue_block_left': ['rotate_object', 'block_blue', 60], 'rotate_pink_block_right': ['rotate_object', 'block_pink', -60], 'rotate_pink_block_left': ['rotate_object', 'block_pink', 60], 'push_red_block_right': ['push_object', 'block_red', 0.1, 0], 'push_red_block_left': ['push_object', 'block_red', -0.1, 0], 'push_blue_block_right': ['push_object', 'block_blue', 0.1, 0], 'push_blue_block_left': ['push_object', 'block_blue', -0.1, 0], 'push_pink_block_right': ['push_object', 'block_pink', 0.1, 0], 'push_pink_block_left': ['push_object', 'block_pink', -0.1, 0], 'move_slider_left': ['move_door_rel', 'base__slide', 0.15], 'move_slider_right': ['move_door_rel', 'base__slide', -0.15], 'open_drawer': ['move_door_rel', 'base__drawer', 0.12], 'close_drawer': ['move_door_rel', 'base__drawer', -0.12], 'lift_red_block_table': ['lift_object', 'block_red', 0.05, 'table', 'base_link'], 'lift_red_block_slider': ['lift_object', 'block_red', 0.03, 'table', 'plank_link'], 'lift_red_block_drawer': ['lift_object', 'block_red', 0.05, 'table', 'drawer_link'], 'lift_blue_block_table': ['lift_object', 'block_blue', 0.05, 'table', 'base_link'], 'lift_blue_block_slider': ['lift_object', 'block_blue', 0.03, 'table', 'plank_link'], 'lift_blue_block_drawer': ['lift_object', 'block_blue', 0.05, 'table', 'drawer_link'], 'lift_pink_block_table': ['lift_object', 'block_pink', 0.05, 'table', 'base_link'], 'lift_pink_block_slider': ['lift_object', 'block_pink', 0.03, 'table', 'plank_link'], 'lift_pink_block_drawer': ['lift_object', 'block_pink', 0.05, 'table', 'drawer_link'], 'place_in_slider': ['place_object', 'table', 'plank_link'], 'place_in_drawer': ['place_object', 'table', 'drawer_link'], 'stack_block': ['stack_objects'], 'unstack_block': ['unstack_objects'], 'turn_on_lightbulb': ['toggle_light', 'lightbulb', 0, 1], 'turn_off_lightbulb': ['toggle_light', 'lightbulb', 1, 0], 'turn_on_led': ['toggle_light', 'led', 0, 1], 'turn_off_led': ['toggle_light', 'led', 1, 0], 'push_into_drawer': ['push_object_into', ['block_red', 'block_blue', 'block_pink'], 'table', 'base_link', 'table', 'drawer_link']}}, 'val_annotations': {'rotate_red_block_right': ['take the red block and rotate it to the right'], 'rotate_red_block_left': ['take the red block and rotate it to the left'], 'rotate_blue_block_right': ['take the blue block and rotate it to the right'], 'rotate_blue_block_left': ['take the blue block and rotate it to the left'], 'rotate_pink_block_right': ['take the pink block and rotate it to the right'], 'rotate_pink_block_left': ['take the pink block and rotate it to the left'], 'push_red_block_right': ['go push the red block right'], 'push_red_block_left': ['go push the red block left'], 'push_blue_block_right': ['go push the blue block right'], 'push_blue_block_left': ['go push the blue block left'], 'push_pink_block_right': ['go push the pink block right'], 'push_pink_block_left': ['go push the pink block left'], 'move_slider_left': ['push the sliding door to the left side'], 'move_slider_right': ['push the sliding door to the right side'], 'open_drawer': ['pull the handle to open the drawer'], 'close_drawer': ['push the handle to close the drawer'], 'lift_red_block_table': ['grasp and lift the red block'], 'lift_blue_block_table': ['grasp and lift the blue block'], 'lift_pink_block_table': ['grasp and lift the pink block'], 'lift_red_block_slider': ['lift the red block from the sliding cabinet'], 'lift_blue_block_slider': ['lift the blue block from the sliding cabinet'], 'lift_pink_block_slider': ['lift the pink block from the sliding cabinet'], 'lift_red_block_drawer': ['Take the red block from the drawer'], 'lift_blue_block_drawer': ['Take the blue block from the drawer'], 'lift_pink_block_drawer': ['Take the pink block from the drawer'], 'place_in_slider': ['store the grasped block in the sliding cabinet'], 'place_in_drawer': ['store the grasped block in the drawer'], 'push_into_drawer': ['slide the block that it falls into the drawer'], 'stack_block': ['stack the grasped block'], 'unstack_block': ['remove the stacked block'], 'turn_on_lightbulb': ['use the switch to turn on the light bulb'], 'turn_off_lightbulb': ['use the switch to turn off the light bulb'], 'turn_on_led': ['press the button to turn on the led light'], 'turn_off_led': ['press the button to turn off the led light']}, '_target_': 'flower.rollout.rollout_long_horizon.RolloutLongHorizon', '_recursive_': False, 'env_cfg': {'_target_': 'flower.wrappers.hulc_wrapper.HulcWrapper'}, 'skip_epochs': '${rollout_lh_skip_epochs}', 'rollout_freq': 5, 'num_videos': 0, 'num_sequences': 1000, 'replan_freq': 30, 'ep_len': 360, 'empty_cache': False, 'log_video_to_file': False, 'save_dir': './videos', 'lang_folder': '${lang_folder}', 'debug': False}, 'ema': {'_target_': 'flower.callbacks.ema.EMA', 'decay': 0.999, 'start_step': 0, 'save_ema_weights_in_callback_state': True, 'evaluate_ema_weights_instead': True, 'power': 0.6666666666666666, 'inv_gamma': 1.0, 'min_value': 0.0, 'max_value': 0.9999}, 'checkpoint': {'_target_': 'pytorch_lightning.callbacks.ModelCheckpoint', 'save_top_k': 1, 'verbose': True, 'monitor': 'eval_lh/avg_seq_len', 'mode': 'max', 'dirpath': 'saved_models', 'filename': '{epoch:02d}_{eval_lh/avg_seq_len:.2f}', 'every_n_epochs': '${callbacks.rollout_lh.rollout_freq}'}}, 'datamodule': {'transforms': {'train': {'rgb_static': [{'_target_': 'torchvision.transforms.Resize', 'size': 224, 'antialias': True}, {'_target_': 'flower.utils.transforms.RandomShiftsAug', 'pad': 10}, {'_target_': 'flower.utils.transforms.ScaleImageTensor'}, {'_target_': 'torchvision.transforms.Normalize', 'mean': [0.48145466, 0.4578275, 0.40821073], 'std': [0.26862954, 0.26130258, 0.27577711]}], 'rgb_gripper': [{'_target_': 'torchvision.transforms.Resize', 'size': 224, 'antialias': True}, {'_target_': 'flower.utils.transforms.RandomShiftsAug', 'pad': 4}, {'_target_': 'flower.utils.transforms.ScaleImageTensor'}, {'_target_': 'torchvision.transforms.Normalize', 'mean': [0.48145466, 0.4578275, 0.40821073], 'std': [0.26862954, 0.26130258, 0.27577711]}]}, 'val': {'rgb_static': [{'_target_': 'torchvision.transforms.Resize', 'size': 224, 'antialias': True}, {'_target_': 'flower.utils.transforms.ScaleImageTensor'}, {'_target_': 'torchvision.transforms.Normalize', 'mean': [0.48145466, 0.4578275, 0.40821073], 'std': [0.26862954, 0.26130258, 0.27577711]}], 'rgb_gripper': [{'_target_': 'torchvision.transforms.Resize', 'size': 224, 'antialias': True}, {'_target_': 'flower.utils.transforms.ScaleImageTensor'}, {'_target_': 'torchvision.transforms.Normalize', 'mean': [0.48145466, 0.4578275, 0.40821073], 'std': [0.26862954, 0.26130258, 0.27577711]}]}}, '_target_': 'flower.datasets.hulc_data_module.HulcDataModule', '_recursive_': False, 'root_data_dir': '${root_data_dir}', 'action_space': 7, 'action_max': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 'action_min': [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1], 'shuffle_val': False, 'observation_space': {'rgb_obs': ['rgb_static', 'rgb_gripper'], 'depth_obs': [], 'state_obs': ['robot_obs'], 'actions': ['rel_actions'], 'language': ['language']}, 'proprioception_dims': {'n_state_obs': 8, 'keep_indices': [[0, 7], [14, 15]], 'robot_orientation_idx': [3, 6], 'normalize': True, 'normalize_robot_orientation': True}, 'datasets': {'lang_dataset': {'_target_': 'flower.datasets.disk_dataset.ExtendedDiskDataset', 'key': 'lang', 'save_format': 'npz', 'batch_size': '${batch_size}', 'min_window_size': '${act_seq_len}', 'max_window_size': '${act_seq_len}', 'proprio_state': '${datamodule.proprioception_dims}', 'obs_space': '${datamodule.observation_space}', 'skip_frames': 1, 'pad': False, 'lang_folder': '${lang_folder}', 'aux_lang_loss_window': 8, 'num_workers': '${num_workers}', 'action_seq_len': '${act_seq_len}', 'obs_seq_len': '${obs_seq_len}', 'future_range': 1, 'use_extracted_rel_actions': '${use_extracted_rel_actions}'}}}, 'model': {'_target_': 'flower.models.flower.FLOWERVLA', '_recursive_': False, 'vlm_path': 'microsoft/Florence-2-large', 'freeze_florence': False, 'freeze_vision_tower': False, 'vlm_prompt_style': 'default', 'token_dropout': 0.1, 'multistep': '${multistep}', 'num_sampling_steps': 4, 'lowdim_obs_dim': 7, 'action_dim': 7, 'act_window_size': 10, 'load_pretrained': True, 'pretrained_model_path': '/home/hk-project-sustainebot/ft4740/code/flower_vla_policy/logs/runs/2025-02-05/10-17-02/360000_model_weights.pt', 'use_second_view': True, 'second_view_key': 'image_wrist', 'action_type_adaln': True, 'use_causal_attention': True, 'use_cross_attn': True, 'use_adaln_cond': False, 'use_readout_token': False, 'use_proprio': False, 'return_act_chunk': False, 'sampling_type': 'uniform', 'dit_dim': 1024, 'n_heads': 16, 'n_layers': 18, 'attn_pdrop': 0.1, 'resid_pdrop': 0.1, 'mlp_pdrop': 0.1, 'use_rope': True, 'use_nope': False, 'query_seq_len': 100, 'rope_theta': 32.0, 'optimizer_type': 'adamw', 'optimizer': {'_target_': 'torch.optim.AdamW', 'transformer_weight_decay': 0.05, 'learning_rate': 1e-05, 'betas': [0.9, 0.99]}, 'lr_scheduler': {'lr_scheduler': {'init_lr': 1e-05, 'init_lr_scale': 0.1, 'final_lr_scale': 0.5, 'total_steps': 50000, 'phase_ratio': '(0.05, 0.1, 0.85)', 'lr': 1e-05}}}, 'root_data_dir': '/hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D', 'lang_folder': 'lang_clip_resnet50', 'log_dir': './logs', 'slurm': False, 'seed': 342, 'device': 'cuda', 'batch_size': 8, 'devices': 4, 'act_dim': 7, 'proprio_dims': 7, 'obs_dim': 512, 'goal_dim': 512, 'obs_seq_len': 1, 'act_seq_len': 10, 'multistep': 10, 'p_last_state': 0, 'gen_img_res': 112, 'max_epochs': 35, 'rollout_lh_skip_epochs': 19, 'num_workers': 12, 'benchmark_name': 'calvin_abc', 'use_extracted_rel_actions': True, 'trainer': {'devices': '${devices}', 'precision': 'bf16-mixed', 'max_epochs': '${max_epochs}', 'sync_batchnorm': True, 'accelerator': 'gpu', 'strategy': 'ddp', 'limit_train_batches': 1000, 'limit_val_batches': 4}, 'logger': {'_target_': 'pytorch_lightning.loggers.WandbLogger', 'save_dir': '.', 'name': '2025-03-13/16-21-05_seed342', 'group': '2025-03-13', 'log_model': False, 'project': '${benchmark_name}', 'entity': 'bennoq', 'id': '2025-03-13_16-21-05_seed342'}}
[2025-03-13 16:21:28,162][__main__][INFO] - Git commit: 0cc472e37ca44ca89bf334e692a2dee801bb8e91
[2025-03-13 16:21:28,324][__main__][INFO] - * CUDA:
	- GPU:
		- NVIDIA A100-SXM4-40GB
		- NVIDIA A100-SXM4-40GB
		- NVIDIA A100-SXM4-40GB
		- NVIDIA A100-SXM4-40GB
	- available:         True
	- version:           12.1
* Packages:
	- numpy:             1.26.4
	- pyTorch_debug:     False
	- pyTorch_version:   2.2.2+cu121
	- pytorch-lightning: 2.0.8
	- tqdm:              4.67.1
[2025-03-13 16:21:28,459][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:28,459][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:28,460][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:28,460][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 16:21:41,935][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/training
[2025-03-13 16:21:41,935][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/training
[2025-03-13 16:21:41,935][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-03-13 16:21:41,935][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/training
[2025-03-13 16:21:41,935][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/training
[2025-03-13 16:21:41,935][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-03-13 16:21:41,935][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-03-13 16:21:41,935][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-03-13 16:21:42,839][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/training/extracted
[2025-03-13 16:21:42,849][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/training/extracted
[2025-03-13 16:21:42,849][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/training/extracted
[2025-03-13 16:21:42,860][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/training/extracted
[2025-03-13 16:21:42,868][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/validation
[2025-03-13 16:21:42,868][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-03-13 16:21:42,877][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/validation
[2025-03-13 16:21:42,877][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/validation
[2025-03-13 16:21:42,877][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-03-13 16:21:42,877][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-03-13 16:21:42,888][flower.datasets.base_dataset][INFO] - loading dataset at /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/validation
[2025-03-13 16:21:42,888][flower.datasets.base_dataset][INFO] - finished loading dataset
[2025-03-13 16:21:42,921][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/validation/extracted
[2025-03-13 16:21:42,921][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/validation/extracted
[2025-03-13 16:21:42,921][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/validation/extracted
[2025-03-13 16:21:42,927][flower.datasets.disk_dataset][INFO] - Extracted files loaded from /hkfs/work/workspace/scratch/ft4740-play3/task_ABCD_D/validation/extracted
[2025-03-13 16:21:51,549][flower.wrappers.hulc_wrapper][INFO] - EGL_DEVICE_ID 0 <==> CUDA_DEVICE_ID 0
[2025-03-13 16:21:51,621][calvin_env.envs.play_table_env][INFO] - Loading EGL plugin (may segfault on misconfigured systems)...
[2025-03-13 16:21:52,200][calvin_env.envs.play_table_env][INFO] - Successfully loaded egl plugin
[2025-03-13 16:21:52,201][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-03-13 16:21:52,203][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-03-13 16:21:52,211][calvin_env.robot.robot][INFO] - Loading robot
[2025-03-13 16:21:52,226][calvin_env.envs.play_table_env][INFO] - Resetting simulation
[2025-03-13 16:21:52,227][calvin_env.envs.play_table_env][INFO] - Setting gravity
[2025-03-13 16:21:52,270][flower.wrappers.hulc_wrapper][INFO] - EGL_DEVICE_ID 1 <==> CUDA_DEVICE_ID 1
[2025-03-13 16:21:52,343][calvin_env.envs.play_table_env][INFO] - Loading EGL plugin (may segfault on misconfigured systems)...
[2025-03-13 16:21:52,744][flower.wrappers.hulc_wrapper][INFO] - Initialized PlayTableEnv for device cuda:0
[2025-03-13 16:21:52,751][flower.evaluation.multistep_sequences][INFO] - Start generating evaluation sequences.
[2025-03-13 16:21:53,150][calvin_env.envs.play_table_env][INFO] - Successfully loaded egl plugin
[2025-03-13 16:21:53,151][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-03-13 16:21:53,152][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-03-13 16:21:53,159][calvin_env.robot.robot][INFO] - Loading robot
[2025-03-13 16:21:53,170][calvin_env.envs.play_table_env][INFO] - Resetting simulation
[2025-03-13 16:21:53,171][calvin_env.envs.play_table_env][INFO] - Setting gravity
[2025-03-13 16:21:53,605][flower.wrappers.hulc_wrapper][INFO] - EGL_DEVICE_ID 2 <==> CUDA_DEVICE_ID 2
[2025-03-13 16:21:53,634][flower.wrappers.hulc_wrapper][INFO] - Initialized PlayTableEnv for device cuda:1
[2025-03-13 16:21:53,641][flower.evaluation.multistep_sequences][INFO] - Start generating evaluation sequences.
[2025-03-13 16:21:53,673][calvin_env.envs.play_table_env][INFO] - Loading EGL plugin (may segfault on misconfigured systems)...
[2025-03-13 16:21:53,988][flower.wrappers.hulc_wrapper][INFO] - EGL_DEVICE_ID 3 <==> CUDA_DEVICE_ID 3
[2025-03-13 16:21:53,998][calvin_env.envs.play_table_env][INFO] - Successfully loaded egl plugin
[2025-03-13 16:21:53,998][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-03-13 16:21:54,000][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-03-13 16:21:54,006][calvin_env.robot.robot][INFO] - Loading robot
[2025-03-13 16:21:54,017][calvin_env.envs.play_table_env][INFO] - Resetting simulation
[2025-03-13 16:21:54,018][calvin_env.envs.play_table_env][INFO] - Setting gravity
[2025-03-13 16:21:54,055][calvin_env.envs.play_table_env][INFO] - Loading EGL plugin (may segfault on misconfigured systems)...
[2025-03-13 16:21:54,248][calvin_env.envs.play_table_env][INFO] - Successfully loaded egl plugin
[2025-03-13 16:21:54,248][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-03-13 16:21:54,250][calvin_env.envs.play_table_env][INFO] - Connected to server with id: 0
[2025-03-13 16:21:54,256][calvin_env.robot.robot][INFO] - Loading robot
[2025-03-13 16:21:54,267][calvin_env.envs.play_table_env][INFO] - Resetting simulation
[2025-03-13 16:21:54,268][calvin_env.envs.play_table_env][INFO] - Setting gravity
[2025-03-13 16:21:54,409][flower.wrappers.hulc_wrapper][INFO] - Initialized PlayTableEnv for device cuda:2
[2025-03-13 16:21:54,414][flower.evaluation.multistep_sequences][INFO] - Start generating evaluation sequences.
[2025-03-13 16:21:54,646][flower.wrappers.hulc_wrapper][INFO] - Initialized PlayTableEnv for device cuda:3
[2025-03-13 16:21:54,652][flower.evaluation.multistep_sequences][INFO] - Start generating evaluation sequences.
[2025-03-13 16:22:10,167][flower.evaluation.multistep_sequences][INFO] - Done generating evaluation sequences.
[2025-03-13 16:22:11,390][flower.evaluation.multistep_sequences][INFO] - Done generating evaluation sequences.
[2025-03-13 16:22:12,574][flower.evaluation.multistep_sequences][INFO] - Done generating evaluation sequences.
[2025-03-13 16:22:12,625][flower.evaluation.multistep_sequences][INFO] - Done generating evaluation sequences.
[2025-03-13 16:22:13,497][root][INFO] - Creating EMA weights copy.
[2025-03-13 16:22:13,497][root][INFO] - Creating EMA weights copy.
[2025-03-13 16:22:13,497][root][INFO] - Creating EMA weights copy.
[2025-03-13 16:22:13,497][root][INFO] - Creating EMA weights copy.
[2025-03-13 19:16:44,728][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 997 / 1000 sequences, SR: 99.7%
[2025-03-13 19:16:44,730][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 970 / 1000 sequences, SR: 97.0%
[2025-03-13 19:16:44,731][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 929 / 1000 sequences, SR: 92.9%
[2025-03-13 19:16:44,731][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 884 / 1000 sequences, SR: 88.4%
[2025-03-13 19:16:44,731][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 820 / 1000 sequences, SR: 82.0%
[2025-03-13 19:16:44,732][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.6
[2025-03-13 20:26:46,350][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 992 / 1000 sequences, SR: 99.2%
[2025-03-13 20:26:46,351][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 966 / 1000 sequences, SR: 96.6%
[2025-03-13 20:26:46,352][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 928 / 1000 sequences, SR: 92.8%
[2025-03-13 20:26:46,352][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 886 / 1000 sequences, SR: 88.6%
[2025-03-13 20:26:46,352][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 843 / 1000 sequences, SR: 84.3%
[2025-03-13 20:26:46,353][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.6
[2025-03-13 21:36:35,642][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 991 / 1000 sequences, SR: 99.1%
[2025-03-13 21:36:35,643][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 978 / 1000 sequences, SR: 97.8%
[2025-03-13 21:36:35,644][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 952 / 1000 sequences, SR: 95.2%
[2025-03-13 21:36:35,644][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 924 / 1000 sequences, SR: 92.4%
[2025-03-13 21:36:35,644][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 878 / 1000 sequences, SR: 87.8%
[2025-03-13 21:36:35,645][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.7
[2025-03-13 22:46:24,924][flower.rollout.rollout_long_horizon][INFO] - 1 / 5 subtasks: 985 / 1000 sequences, SR: 98.5%
[2025-03-13 22:46:24,925][flower.rollout.rollout_long_horizon][INFO] - 2 / 5 subtasks: 950 / 1000 sequences, SR: 95.0%
[2025-03-13 22:46:24,926][flower.rollout.rollout_long_horizon][INFO] - 3 / 5 subtasks: 909 / 1000 sequences, SR: 90.9%
[2025-03-13 22:46:24,926][flower.rollout.rollout_long_horizon][INFO] - 4 / 5 subtasks: 880 / 1000 sequences, SR: 88.0%
[2025-03-13 22:46:24,926][flower.rollout.rollout_long_horizon][INFO] - 5 / 5 subtasks: 834 / 1000 sequences, SR: 83.4%
[2025-03-13 22:46:24,928][flower.rollout.rollout_long_horizon][INFO] - Average successful sequence length: 4.6
[2025-03-13 22:46:29,020][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 22:46:29,021][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 22:46:29,021][__main__][INFO] - GPU 2 Memory: Allocated: 3.73GB, Reserved: 3.83GB
[2025-03-13 22:46:29,021][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 22:46:29,094][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 22:46:29,095][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 22:46:29,095][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 22:46:29,095][__main__][INFO] - GPU 3 Memory: Allocated: 3.73GB, Reserved: 3.79GB
[2025-03-13 22:46:29,101][__main__][INFO] - GPU 0 Memory: Allocated: 3.73GB, Reserved: 3.79GB
[2025-03-13 22:46:29,101][__main__][INFO] - GPU 1 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 22:46:29,102][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 22:46:29,102][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 22:46:29,104][__main__][INFO] - GPU 0 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 22:46:29,104][__main__][INFO] - GPU 1 Memory: Allocated: 3.73GB, Reserved: 3.81GB
[2025-03-13 22:46:29,104][__main__][INFO] - GPU 2 Memory: Allocated: 0.00GB, Reserved: 0.00GB
[2025-03-13 22:46:29,105][__main__][INFO] - GPU 3 Memory: Allocated: 0.00GB, Reserved: 0.00GB