Add files using upload-large-folder tool

checkpoints/steps_30000_pytorch_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee2ae998dba854f33a43c7940ee8cf0addb0fda11bd9392574400b6614be0e9b
+size 8935435793

config.json

@@ -0,0 +1,115 @@
+{
+  "run_id": "0903_libero_10_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm",
+  "run_root_dir": "./playground/Checkpoints",
+  "seed": 42,
+  "trackers": [
+    "jsonl",
+    "wandb"
+  ],
+  "wandb_entity": "michaelyu-1101-fudanuniversity",
+  "wandb_project": "Internvla",
+  "is_debug": false,
+  "framework": {
+    "framework_py": "DinoQFormerACT",
+    "qwenvl": {
+      "base_vlm": "/mnt/phwfile/efm_t/zhuyangkun_tmp_need_del/exp/exp_08_09/manip_sys2_qwen25_3b_onevision_molmo_a0all_refsp20/checkpoint-20000",
+      "attn_implementation": "flash_attention_2",
+      "vl_hidden_dim": 2048
+    },
+    "dino": {
+      "dino_backbone": "dinov2_vitl14"
+    },
+    "layer_qformer": {
+      "qformer_end_layer": 37,
+      "qformer_start_layer": 36,
+      "num_query_tokens": 64,
+      "grad_scale": 0.5
+    },
+    "action_model": {
+      "action_model_type": "DiT-B",
+      "action_hidden_dim": 768,
+      "action_dim": 7,
+      "use_ema": false,
+      "future_action_window_size": 7,
+      "past_action_window_size": 0,
+      "repeated_diffusion_steps": 8
+    },
+    "reduce_in_full_precision": true
+  },
+  "datasets": {
+    "vlm_data": {
+      "dataformat": "llava_json",
+      "dataset_use": "asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en",
+      "eval_dataset": "aokvqa_cauldron_llava_format",
+      "data_flatten": false,
+      "base_interval": 2,
+      "max_pixels": 50176,
+      "min_pixels": 784,
+      "fix_image_size": [
+        224,
+        224
+      ],
+      "model_max_length": 1024,
+      "model_type": "qwen2.5vl",
+      "per_device_batch_size": 4
+    },
+    "vla_data": {
+      "dataset_py": "lerobot_libero",
+      "data_root_dir": "playground/Datasets/LEROBOT_LIBERO_DATA",
+      "data_mix": "libero_10",
+      "action_type": "delta_qpos",
+      "CoT_prompt": "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format.",
+      "CoT_answer": "bbox",
+      "default_image_resolution": [
+        3,
+        224,
+        224
+      ],
+      "per_device_batch_size": 16,
+      "load_all_data_for_training": true,
+      "obs": [
+        "image_0"
+      ]
+    }
+  },
+  "trainer": {
+    "epochs": 100,
+    "max_train_steps": 100000,
+    "num_warmup_steps": 5000,
+    "save_interval": 10000,
+    "eval_interval": 1000,
+    "learning_rate": {
+      "base": 2.5e-05
+    },
+    "lr_scheduler_type": "cosine_with_min_lr",
+    "scheduler_specific_kwargs": {
+      "min_lr": 1e-06
+    },
+    "freeze_modules": "",
+    "loss_scale": {
+      "vla": 1.0,
+      "vlm": 0.1
+    },
+    "max_grad_norm": 1.0,
+    "warmup_ratio": 0.1,
+    "weight_decay": 0.0,
+    "logging_frequency": 10,
+    "gradient_clipping": 1.0,
+    "gradient_accumulation_steps": 1,
+    "optimizer": {
+      "name": "AdamW",
+      "betas": [
+        0.9,
+        0.95
+      ],
+      "eps": 1e-08,
+      "weight_decay": 1e-08
+    },
+    "is_resume": false,
+    "resume_epoch": null,
+    "resume_step": null,
+    "enable_gradient_checkpointing": true,
+    "enable_mixed_precision_training": true
+  },
+  "output_dir": "./playground/Checkpoints/0903_libero_10_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm"
+}

config.yaml

@@ -0,0 +1,96 @@
+run_id: 0903_libero_10_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm
+run_root_dir: ./playground/Checkpoints
+seed: 42
+trackers:
+- jsonl
+- wandb
+wandb_entity: michaelyu-1101-fudanuniversity
+wandb_project: Internvla
+is_debug: false
+framework:
+  framework_py: DinoQFormerACT
+  qwenvl:
+    base_vlm: /mnt/phwfile/efm_t/zhuyangkun_tmp_need_del/exp/exp_08_09/manip_sys2_qwen25_3b_onevision_molmo_a0all_refsp20/checkpoint-20000
+    attn_implementation: flash_attention_2
+    vl_hidden_dim: 2048
+  dino:
+    dino_backbone: dinov2_vitl14
+  layer_qformer:
+    qformer_end_layer: 37
+    qformer_start_layer: 36
+    num_query_tokens: 64
+    grad_scale: 0.5
+  action_model:
+    action_model_type: DiT-B
+    action_hidden_dim: 768
+    action_dim: 7
+    use_ema: false
+    future_action_window_size: 7
+    past_action_window_size: 0
+    repeated_diffusion_steps: 8
+  reduce_in_full_precision: true
+datasets:
+  vlm_data:
+    dataformat: llava_json
+    dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en
+    eval_dataset: aokvqa_cauldron_llava_format
+    data_flatten: false
+    base_interval: 2
+    max_pixels: 50176
+    min_pixels: 784
+    fix_image_size:
+    - 224
+    - 224
+    model_max_length: 1024
+    model_type: qwen2.5vl
+    per_device_batch_size: 4
+  vla_data:
+    dataset_py: lerobot_libero
+    data_root_dir: playground/Datasets/LEROBOT_LIBERO_DATA
+    data_mix: libero_10
+    action_type: delta_qpos
+    CoT_prompt: Your task is {instruction}. To identify the key objects for your task.
+      Locate their bounding boxes in [x1,y1,x2,y2] format.
+    CoT_answer: bbox
+    default_image_resolution:
+    - 3
+    - 224
+    - 224
+    per_device_batch_size: 16
+    load_all_data_for_training: true
+    obs:
+    - image_0
+trainer:
+  epochs: 100
+  max_train_steps: 100000
+  num_warmup_steps: 5000
+  save_interval: 10000
+  eval_interval: 1000
+  learning_rate:
+    base: 2.5e-05
+  lr_scheduler_type: cosine_with_min_lr
+  scheduler_specific_kwargs:
+    min_lr: 1.0e-06
+  freeze_modules: ''
+  loss_scale:
+    vla: 1.0
+    vlm: 0.1
+  max_grad_norm: 1.0
+  warmup_ratio: 0.1
+  weight_decay: 0.0
+  logging_frequency: 10
+  gradient_clipping: 1.0
+  gradient_accumulation_steps: 1
+  optimizer:
+    name: AdamW
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    weight_decay: 1.0e-08
+  is_resume: false
+  resume_epoch: null
+  resume_step: null
+  enable_gradient_checkpointing: true
+  enable_mixed_precision_training: true
+output_dir: ./playground/Checkpoints/0903_libero_10_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm

dataset_statistics.json

@@ -0,0 +1,133 @@
+{
+  "franka": {
+    "action": {
+      "mean": [
+        0.01820324920117855,
+        0.05858374014496803,
+        -0.05592384561896324,
+        0.004626928828656673,
+        0.00289608770981431,
+        -0.007673131301999092,
+        0.5457824468612671
+      ],
+      "std": [
+        0.2825464606285095,
+        0.35904666781425476,
+        0.3673802614212036,
+        0.03770702704787254,
+        0.05429719388484955,
+        0.08725254982709885,
+        0.49815231561660767
+      ],
+      "max": [
+        0.9375,
+        0.9375,
+        0.9375,
+        0.30000001192092896,
+        0.29357144236564636,
+        0.375,
+        1.0
+      ],
+      "min": [
+        -0.9375,
+        -0.9375,
+        -0.9375,
+        -0.23642857372760773,
+        -0.3053571283817291,
+        -0.3675000071525574,
+        0.0
+      ],
+      "q01": [
+        -0.6348214149475098,
+        -0.7741071581840515,
+        -0.7633928656578064,
+        -0.09749999642372131,
+        -0.14819999992847435,
+        -0.2742857038974762,
+        0.0
+      ],
+      "q99": [
+        0.7714285850524902,
+        0.8464285731315613,
+        0.9375,
+        0.13928571343421936,
+        0.15964286029338837,
+        0.3246428668498993,
+        1.0
+      ],
+      "mask": [
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        false
+      ]
+    },
+    "state": {
+      "mean": [
+        -0.04190658777952194,
+        0.03539430722594261,
+        0.8257141709327698,
+        2.908308267593384,
+        -0.5562185049057007,
+        -0.16649018228054047,
+        0.028316624462604523,
+        -0.028561657294631004
+      ],
+      "std": [
+        0.10743364691734314,
+        0.14424669742584229,
+        0.2572328448295593,
+        0.3441362977027893,
+        1.234421730041504,
+        0.3579835891723633,
+        0.013308707624673843,
+        0.013174631632864475
+      ],
+      "max": [
+        0.21031762659549713,
+        0.39128610491752625,
+        1.3332009315490723,
+        3.6714255809783936,
+        3.560650587081909,
+        1.386339545249939,
+        0.04160946607589722,
+        0.0013633022317662835
+      ],
+      "min": [
+        -0.4828203022480011,
+        -0.3255046010017395,
+        0.445506751537323,
+        1.1321442127227783,
+        -3.641430377960205,
+        -1.842738389968872,
+        -0.0010040868073701859,
+        -0.04111652821302414
+      ],
+      "q01": [
+        -0.3899900782108307,
+        -0.2838300323486328,
+        0.44795057058334353,
+        1.8810229921340942,
+        -2.886677579879761,
+        -1.1599004411697387,
+        0.002066459748893976,
+        -0.04001387819647789
+      ],
+      "q99": [
+        0.1530261474847791,
+        0.32915401458740223,
+        1.2546923208236693,
+        3.303542451858519,
+        2.7496529006957933,
+        0.6893712210655194,
+        0.040048558115959164,
+        -0.0017598449345678235
+      ]
+    },
+    "num_transitions": 101469,
+    "num_trajectories": 379
+  }
+}

run_lerobot_datasets.sh

@@ -0,0 +1,64 @@
+export HF_HOME=/mnt/petrelfs/share/yejinhui/Models/huggingface_cache
+
+export NCCL_SOCKET_IFNAME=bond0
+export NCCL_IB_HCA=mlx5_2,mlx5_3
+
+# 用于check save 的时候的通信
+export NCCL_BLOCKING_WAIT=1
+export NCCL_ASYNC_ERROR_HANDLING=1
+export NCCL_TIMEOUT=1000  # 超时时间设为 1 小时（单位：秒）
+
+cd /mnt/petrelfs/yujunqiu/code/vla-baseline/llavavla-00hf1
+
+# MODEL_PATH=/mnt/petrelfs/yejinhui/Projects/llavavla/playground/Pretrained_models/Qwen2.5-VL-3B-Instruct # must be a local path, due to simpler will run in other where
+# data_root_dir=./playground/Datasets/OXE_LEROBOT_DATASET
+run_root_dir=./playground/Checkpoints
+task_name=libero_10
+run_id=0903_${task_name}_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm
+
+
+export WANDB_MODE=disabled
+
+output_dir=${run_root_dir}/${run_id}
+mkdir -p ${output_dir}
+# mv this script to the output dir
+cp $0 ${output_dir}/
+
+  # --pretrained_checkpoint ${MODEL_PATH} \
+# export CUDA_VISIBLE_DEVICES=4,5,6,7
+
+  # --datasets.vla_data.data_mix libero_goal \
+  # --framework.framework_py qwenpi \
+
+DEBUG=False
+# DEBUG=True
+
+if [ "$DEBUG" = True ]; then
+  num_processes=1
+  run_id=debug
+else
+  num_processes=8
+fi
+
+
+accelerate launch \
+  --config_file scripts/run_scripts/deepspeed_zero2.yaml \
+  --num_processes ${num_processes} \
+  llavavla/training/train_qwenvla.py \
+  --config_yaml ./llavavla/config/lerobot_data/qwenvla_cotrain_libero.yaml \
+  --datasets.vla_data.per_device_batch_size 16 \
+  --datasets.vla_data.data_mix ${task_name} \
+  --framework.action_model.future_action_window_size 7 \
+  --trainer.max_train_steps 100_000 \
+  --trainer.save_interval 10_000 \
+  --run_root_dir ${run_root_dir} \
+  --run_id ${run_id} \
+  --wandb_project Internvla \
+  --wandb_entity michaelyu-1101-fudanuniversity \
+  --is_debug ${DEBUG} \
+  --framework.qwenvl.base_vlm /mnt/phwfile/efm_t/zhuyangkun_tmp_need_del/exp/exp_08_09/manip_sys2_qwen25_3b_onevision_molmo_a0all_refsp20/checkpoint-20000
+
+#   --framework.qwenvl.base_vlm ${MODEL_PATH} \
+#   --data_root_dir ${data_root_dir} \
+
+  # --is_debug True

summary.jsonl

@@ -0,0 +1,3 @@
+{"steps": 10000}
+{"steps": 20000}
+{"steps": 30000}