| { |
| "processor_class": "Gr00tN1d6Processor", |
| "processor_kwargs": { |
| "modality_configs": { |
| "behavior_r1_pro": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "observation.images.rgb.head_256_256", |
| "observation.images.rgb.left_wrist_256_256", |
| "observation.images.rgb.right_wrist_256_256" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "robot_pos", |
| "robot_ori_cos", |
| "robot_ori_sin", |
| "robot_2d_ori", |
| "robot_2d_ori_cos", |
| "robot_2d_ori_sin", |
| "robot_lin_vel", |
| "robot_ang_vel", |
| "arm_left_qpos", |
| "arm_left_qpos_sin", |
| "arm_left_qpos_cos", |
| "eef_left_pos", |
| "eef_left_quat", |
| "gripper_left_qpos", |
| "arm_right_qpos", |
| "arm_right_qpos_sin", |
| "arm_right_qpos_cos", |
| "eef_right_pos", |
| "eef_right_quat", |
| "gripper_right_qpos", |
| "trunk_qpos" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31 |
| ], |
| "modality_keys": [ |
| "base", |
| "torso", |
| "left_arm", |
| "left_gripper", |
| "right_arm", |
| "right_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "RELATIVE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": "trunk_qpos", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "RELATIVE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": "arm_left_qpos", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "RELATIVE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": "arm_right_qpos", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "annotation.human.coarse_action" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "gr1": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "ego_view_bg_crop_pad_res256_freq20" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "left_arm", |
| "right_arm", |
| "left_hand", |
| "right_hand", |
| "waist" |
| ], |
| "sin_cos_embedding_keys": [ |
| "left_arm", |
| "right_arm", |
| "left_hand", |
| "right_hand", |
| "waist" |
| ], |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15 |
| ], |
| "modality_keys": [ |
| "left_arm", |
| "right_arm", |
| "left_hand", |
| "right_hand", |
| "waist" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "RELATIVE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "RELATIVE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "RELATIVE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "RELATIVE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "task" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "robocasa_panda_omron": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "res256_image_side_0", |
| "res256_image_side_1", |
| "res256_image_wrist_0" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "end_effector_position_relative", |
| "end_effector_rotation_relative", |
| "gripper_qpos", |
| "base_position", |
| "base_rotation" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15 |
| ], |
| "modality_keys": [ |
| "end_effector_position", |
| "end_effector_rotation", |
| "gripper_close", |
| "base_motion", |
| "control_mode" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "annotation.human.action.task_description" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "cmr_versius": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "endoscope" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "left_pose", |
| "left_gripper", |
| "right_pose", |
| "right_gripper", |
| "translation_scaling", |
| "rotation_scaling", |
| "hapticengaged_left", |
| "hapticengaged_right" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": [ |
| "left_pose", |
| "left_gripper", |
| "right_pose", |
| "right_gripper" |
| ], |
| "min_max_embedding_keys": null, |
| "pass_through_keys": [ |
| "translation_scaling", |
| "rotation_scaling", |
| "hapticengaged_left", |
| "hapticengaged_right" |
| ], |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 2, |
| 4, |
| 6, |
| 8, |
| 10, |
| 12, |
| 14, |
| 16, |
| 18, |
| 20, |
| 22, |
| 24, |
| 26, |
| 28, |
| 30, |
| 32, |
| 34, |
| 36, |
| 38, |
| 40, |
| 42, |
| 44, |
| 46, |
| 48, |
| 50, |
| 52, |
| 54, |
| 56, |
| 58, |
| 60, |
| 62, |
| 64, |
| 66, |
| 68, |
| 70, |
| 72, |
| 74, |
| 76, |
| 78, |
| 80, |
| 82, |
| 84, |
| 86, |
| 88, |
| 90, |
| 92, |
| 94, |
| 96, |
| 98, |
| 100 |
| ], |
| "modality_keys": [ |
| "left_pose", |
| "left_gripper", |
| "right_pose", |
| "right_gripper", |
| "hapticengaged_left", |
| "hapticengaged_right" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": [ |
| "hapticengaged_left", |
| "hapticengaged_right" |
| ], |
| "action_configs": [ |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "left_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": "translation_scaling", |
| "rotation_scaling_key": "rotation_scaling", |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": "left_gripper", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": true |
| }, |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "right_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": "translation_scaling", |
| "rotation_scaling_key": "rotation_scaling", |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": "right_gripper", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": true |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "skip", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "skip", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "annotation.human.task_description" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "ucsd_dvrk": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "camera_left" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "psm_retraction_pose", |
| "psm_retraction_gripper", |
| "psm_cutter_pose", |
| "psm_cutter_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": [ |
| "psm_retraction_pose", |
| "psm_retraction_gripper", |
| "psm_cutter_pose", |
| "psm_cutter_gripper" |
| ], |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 35, |
| 36, |
| 37, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46, |
| 47, |
| 48, |
| 49, |
| 50 |
| ], |
| "modality_keys": [ |
| "psm_retraction_pose", |
| "psm_retraction_gripper", |
| "psm_cutter_pose", |
| "psm_cutter_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm_retraction_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "wxyz", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "wxyz", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm_cutter_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "wxyz", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "wxyz", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "task" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "jhu_imerse_dvrk": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "endoscope_left", |
| "wrist_left", |
| "wrist_right" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 35, |
| 36, |
| 37, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46, |
| 47, |
| 48, |
| 49, |
| 50 |
| ], |
| "modality_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm1_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm2_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "annotation.human.task_description" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "obuda_dvrk": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "endoscope_left", |
| "wrist_left", |
| "wrist_right" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 35, |
| 36, |
| 37, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46, |
| 47, |
| 48, |
| 49 |
| ], |
| "modality_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm1_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm2_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "task" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "stanford_dvrk_real": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "endoscope_left" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 35, |
| 36, |
| 37, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46, |
| 47, |
| 48, |
| 49 |
| ], |
| "modality_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm1_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "euler", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "euler", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm2_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "euler", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "euler", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "task" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "tud_tundra_ur5e": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "laparoscope_left" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "joint_position", |
| "eef_pose" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": [ |
| "joint_position" |
| ], |
| "min_max_embedding_keys": null, |
| "pass_through_keys": [ |
| "eef_pose" |
| ], |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 35, |
| 36, |
| 37, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46, |
| 47, |
| 48, |
| 49, |
| 50 |
| ], |
| "modality_keys": [ |
| "eef_pose", |
| "gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "eef_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "task" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "jhu_lscr_dvrk_smarts": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "endoscope_left", |
| "camera_side_view" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16 |
| ], |
| "modality_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm1_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm2_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "annotation.task" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "jhu_imerse_dvrk_mono": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "endoscope_left" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 35, |
| 36, |
| 37, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46, |
| 47, |
| 48, |
| 49 |
| ], |
| "modality_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm1_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm2_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "annotation.human.task_description" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "rob_surgical_bitrack": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "endoscope" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "left_pose", |
| "right_pose", |
| "aux_pose" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": [ |
| "left_pose", |
| "right_pose", |
| "aux_pose" |
| ], |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 35, |
| 36, |
| 37, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46, |
| 47, |
| 48, |
| 49 |
| ], |
| "modality_keys": [ |
| "left_pose", |
| "right_pose", |
| "aux_pose" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "left_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "euler", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "euler", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "right_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "euler", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "euler", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "aux_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "euler", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "euler", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "annotation.instruction" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "turin_mitic_ex_vivo": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "endoscope_left" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "psm1_joints", |
| "psm2_joints", |
| "psm1_pose", |
| "psm2_pose" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": [ |
| "psm1_joints", |
| "psm2_joints" |
| ], |
| "min_max_embedding_keys": null, |
| "pass_through_keys": [ |
| "psm1_pose", |
| "psm2_pose" |
| ], |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 35, |
| 36, |
| 37, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46, |
| 47, |
| 48, |
| 49, |
| 50 |
| ], |
| "modality_keys": [ |
| "psm1_pose", |
| "psm2_pose" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm1_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm2_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "annotation.instruction" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "ustc_torin_tuodao": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "endoscope_left" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "left_joints", |
| "right_joints", |
| "left_pose", |
| "right_pose" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": [ |
| "left_joints", |
| "right_joints" |
| ], |
| "min_max_embedding_keys": null, |
| "pass_through_keys": [ |
| "left_pose", |
| "right_pose" |
| ], |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 35, |
| 36, |
| 37, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46, |
| 47, |
| 48, |
| 49 |
| ], |
| "modality_keys": [ |
| "left_pose", |
| "left_gripper", |
| "right_pose", |
| "right_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "left_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "right_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "annotation.instruction" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "hamlyn_dvrk_30hz": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "endoscope", |
| "wrist_left", |
| "wrist_right" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "left_arm_pose", |
| "left_arm_gripper", |
| "right_arm_pose", |
| "right_arm_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": [ |
| "left_arm_pose", |
| "left_arm_gripper", |
| "right_arm_pose", |
| "right_arm_gripper" |
| ], |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 35, |
| 36, |
| 37, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46, |
| 47, |
| 48, |
| 49 |
| ], |
| "modality_keys": [ |
| "left_arm_pose", |
| "left_arm_gripper", |
| "right_arm_pose", |
| "right_arm_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "left_arm_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "wxyz", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "wxyz", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "right_arm_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "wxyz", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "wxyz", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "task" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "ucb_dvrk": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "camera_left" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "psm1_joints", |
| "psm1_gripper", |
| "psm2_joints", |
| "psm2_gripper", |
| "psm1_pose", |
| "psm2_pose" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": [ |
| "psm1_joints", |
| "psm1_gripper", |
| "psm2_joints", |
| "psm2_gripper" |
| ], |
| "min_max_embedding_keys": null, |
| "pass_through_keys": [ |
| "psm1_pose", |
| "psm2_pose" |
| ], |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 35, |
| 36, |
| 37, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46, |
| 47, |
| 48, |
| 49 |
| ], |
| "modality_keys": [ |
| "psm1_pose", |
| "psm1_gripper", |
| "psm2_pose", |
| "psm2_gripper" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm1_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "psm2_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| }, |
| { |
| "rep": "ABSOLUTE", |
| "type": "NON_EEF", |
| "format": "DEFAULT", |
| "state_key": null, |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "rot6d", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "task" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| }, |
| "jhu_imerse_star_il": { |
| "video": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "endoscope_left", |
| "wrist_left" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| }, |
| "state": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "kuka_joint_pos", |
| "endo360_joint_pos", |
| "kuka_pose" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": [ |
| "kuka_joint_pos", |
| "endo360_joint_pos" |
| ], |
| "min_max_embedding_keys": null, |
| "pass_through_keys": [ |
| "kuka_pose" |
| ], |
| "action_configs": null |
| }, |
| "action": { |
| "delta_indices": [ |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 35, |
| 36, |
| 37, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46, |
| 47, |
| 48, |
| 49, |
| 50 |
| ], |
| "modality_keys": [ |
| "kuka_pose" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": [ |
| { |
| "rep": "REL_XYZ_ROT6D", |
| "type": "EEF", |
| "format": "XYZ_ROT6D", |
| "state_key": "kuka_pose", |
| "normalization_type": "percentile", |
| "input_rotation_format": "quat", |
| "input_quat_order": "xyzw", |
| "reference_rotation_format": "quat", |
| "reference_quat_order": "xyzw", |
| "translation_scaling_key": null, |
| "rotation_scaling_key": null, |
| "hold_through_clutch": false |
| } |
| ] |
| }, |
| "language": { |
| "delta_indices": [ |
| 0 |
| ], |
| "modality_keys": [ |
| "annotation.human.task_description" |
| ], |
| "sin_cos_embedding_keys": null, |
| "mean_std_embedding_keys": null, |
| "min_max_embedding_keys": null, |
| "pass_through_keys": null, |
| "action_configs": null |
| } |
| } |
| }, |
| "image_crop_size": [ |
| 224, |
| 392 |
| ], |
| "image_target_size": [ |
| 236, |
| 414 |
| ], |
| "use_albumentations": false, |
| "random_rotation_angle": 5, |
| "color_jitter_params": { |
| "brightness": 0.12, |
| "contrast": 0.15, |
| "saturation": 0.15, |
| "hue": 0.02 |
| }, |
| "shortest_image_edge": null, |
| "crop_fraction": null, |
| "model_name": "nvidia/Eagle-Block2A-2B-v2", |
| "model_type": "eagle", |
| "formalize_language": true, |
| "max_state_dim": 128, |
| "max_action_dim": 128, |
| "max_action_horizon": 50, |
| "use_percentiles": false, |
| "clip_outliers": true, |
| "apply_sincos_state_encoding": true, |
| "use_relative_action": true |
| } |
| } |
|
|
