Request examples for the Dataset files

[kinredon]

Thanks for your great work.

When I was reproducing this work, I found that I did not know how to obtain the dataset file, such as:

wo_system_image_180k_filter_w_image_size_filter_wo_some_code.jsonl

I have downloaded the dataset from the Hugging Face website, but how to convert it to a JSONL file? Could you please provide an example or any instructions for this?

Big thanks!

[yfzhang114]

Based on your dataset structure with image, question, and response fields, here's a simple solution to convert your Hugging Face dataset to JSONL format while saving images locally.

[yfzhang114]

#!/usr/bin/env python3
import os, json, base64
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor, as_completed
from datasets import load_dataset
from tqdm import tqdm
import io
from PIL import Image

HF_DATA_DIR = "/mnt/datasets_vlm02/yifan.zhang/data/Thyme-SFT"
ROOT_OUT    = Path("Thyme_sft_data")
IMG_ROOT    = '/mnt/datasets_vlm02/yifan.zhang/data/Thyme_sft_data/img'
JSONL_ROOT  = ROOT_OUT / "jsonl"
SPLITS      = ["wo_thinking_thyme_single_round", "2round",] #,  "computation"
MAX_WORKERS = os.cpu_count()      # 可根据机器调整

# IMG_ROOT.mkdir(parents=True, exist_ok=True)
JSONL_ROOT.mkdir(parents=True, exist_ok=True)

# ----------- 线程池任务 -----------
def save_one_image(args):
    """
    解码Base64字符串，处理透明度并将图像保存为JPEG。

    Args:
        args (tuple): 包含 (b64_str, save_path) 的元组。
    """
    b64_str, save_path = args
    if os.path.exists(save_path):
        return save_path

    try:
        # 1. 解码Base64得到原始二进制数据
        image_bytes = base64.b64decode(b64_str)

        # 2. 使用Pillow从二进制数据中打开图像
        with Image.open(io.BytesIO(image_bytes)) as img:
            # 3. 处理透明度 (关键步骤)
            # 检查图像模式是否需要处理透明度。
            # 'P' 模式可能含有透明度，'LA' 是灰度+透明度。
            # 'RGBA' 是最常见的带透明度的模式。
            if img.mode in ("RGBA", "LA", "P"):
                # 为了统一处理所有透明情况，先将图像转换为RGBA模式。
                # 如果图像是 'P' 模式并带有透明度，转换后会得到正确的RGBA图像。
                img = img.convert("RGBA")

                # 创建一个白色背景的底图
                background = Image.new("RGB", img.size, (255, 255, 255))

                # 将原图粘贴到底图上。
                # 此时的 img 已经是 RGBA 模式，所以它自身可以作为 mask，
                # Pillow会自动使用其Alpha通道。
                background.paste(img, (0, 0), img)
                img = background # 现在img是合并后的RGB图像了

            # 如果图像模式不是RGB（例如 'L', 'CMYK' 等），则转换为RGB
            elif img.mode != "RGB":
                img = img.convert("RGB")

            # 4. 以JPEG格式保存图像
            # JPEG不支持透明度，所以背景填充是必要的。
            img.save(save_path, "jpeg", quality=95) # 推荐加上 quality 参数

        return str(save_path)

    except Exception as e:
        # 增加异常捕获，方便调试是哪个图片出了问题
        print(f"Error processing image for {save_path}: {e}")
        return None

# ----------- 主处理 -----------
for split in SPLITS:
    print(f"\n>>> Processing split : {split}  (max_workers={MAX_WORKERS})")
    # 3. 写 jsonl, 检查是否已经存在
    jsonl_path = JSONL_ROOT / f"{split}.jsonl"
    if not jsonl_path.exists():  # 如果 jsonl 文件不存在，才写入
        print(f"  JSONL  -> {jsonl_path}")
    else:
        print(f"  JSONL already exists: {jsonl_path}")

    ds = load_dataset(HF_DATA_DIR, split=split)

    img_dir = IMG_ROOT + '/' + split
    # img_dir.mkdir(exist_ok=True)

    # 1. 先收集所有待保存任务
    tasks = []               # (b64_str, save_path)
    records = []             # 用于写 jsonl
    for sample_idx, sample in enumerate(ds):
        img_paths = []
        for img_idx, b64_img in enumerate(sample["image"], start=1):
            img_name = f"{sample_idx+1:08d}_{img_idx:02d}.jpg"
            img_path = img_dir + '/' + img_name
            tasks.append((b64_img, img_path))
            img_paths.append(str(img_path))
        records.append({
            "image": img_paths,
            "question": sample["question"],
            "response": sample["response"]
        })

    # 2. 多线程执行
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
        # 只保存尚未存在的图像
        saved_images = list(tqdm(pool.map(save_one_image, tasks),
                                 total=len(tasks), desc="Saving images"))
    
    # 过滤掉返回值为 None 的项（即文件已经存在的）
    saved_images = [img for img in saved_images if img is not None]

    with open(jsonl_path, "w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec, ensure_ascii=False) + "\n")
    
    print(f"  Images -> {img_dir}  ({len(saved_images)} files)")

print("\nAll done (multi-threaded)!")

[kinredon]

@yfzhang114 Thanks for your response. I use the code you first provide to generate the jsonl. I note that you have modified above code and remove the split of "computation".

[yfzhang114]

The second code snippet is clearer, and computation can also be constructed using a similar approach.