Чанковая передача файлов .safetensors

export TB_ENDPOINT="https://<your-workspace-host>"   # см. Настройки → API
export TB_TOKEN="tb_live_…"                           # не коммитьте это значение

{
  "repo": "tb-labs/RuVision-7B-Instruct",
  "revision": "a91f3c0d7e1b4f62",
  "chunk_size": 67108864,
  "files": [
    {
      "path": "model-00001-of-00003.safetensors",
      "size": 4983451184,
      "sha256": "0f6c2d…e91a",
      "chunks": ["5b9e04…71cd", "c013aa…0b42", "…"]      // 75 хэшей
    }
  ]
}

FIRST=$(curl -s -H "Authorization: Bearer $TB_TOKEN" \
  "$TB_ENDPOINT/api/v1/repos/tb-labs/RuVision-7B-Instruct/manifest" | jq -r '.files[0].chunks[0]')

N=$(curl -s -L -r 0-7 -H "Authorization: Bearer $TB_TOKEN" "$TB_ENDPOINT/media/chunks/$FIRST" \
  | od -An -t u8 --endian=little | tr -d ' ')

curl -s -L -r 8-$((8 + N - 1)) -H "Authorization: Bearer $TB_TOKEN" \
  "$TB_ENDPOINT/media/chunks/$FIRST" | jq 'del(."__metadata__") | to_entries[:3]'

import hashlib
import os
from concurrent.futures import ThreadPoolExecutor

import requests

ENDPOINT = os.environ["TB_ENDPOINT"]
CHUNK_SIZE = 64 * 1024 * 1024

session = requests.Session()
session.headers["Authorization"] = f"Bearer {os.environ['TB_TOKEN']}"


def read_chunk(path, index):
    with open(path, "rb") as f:
        f.seek(index * CHUNK_SIZE)
        return f.read(CHUNK_SIZE)


def chunk_digests(path):
    digests = []
    with open(path, "rb") as f:
        while block := f.read(CHUNK_SIZE):
            digests.append(hashlib.sha256(block).hexdigest())
    return digests


def upload(path, repo, target_path):
    digests = chunk_digests(path)
    r = session.post(f"{ENDPOINT}/api/v1/repos/{repo}/uploads", json={
        "path": target_path,
        "size": os.path.getsize(path),
        "chunks": digests,
    })
    r.raise_for_status()
    plan = r.json()
    missing = set(plan["missing"])
    # первый индекс каждого отсутствующего хэша (одинаковые чанки загружаются один раз)
    todo = {d: i for i, d in reversed(list(enumerate(digests))) if d in missing}

    def put(item):
        digest, index = item
        resp = session.put(
            f"{ENDPOINT}/media/chunks/{digest}",
            data=read_chunk(path, index),
            headers={"Content-Type": "application/octet-stream",
                     "X-TB-Upload-Id": plan["upload_id"]},
            timeout=300,
        )
        resp.raise_for_status()

    with ThreadPoolExecutor(max_workers=8) as pool:
        list(pool.map(put, todo.items()))

    commit = session.post(f"{ENDPOINT}/api/v1/repos/{repo}/uploads/{plan['upload_id']}/commit")
    commit.raise_for_status()
    return commit.json()["revision"]


def download(repo, file_path, dest, revision="main"):
    r = session.get(f"{ENDPOINT}/api/v1/repos/{repo}/manifest", params={"revision": revision})
    r.raise_for_status()
    manifest = r.json()
    entry = next(f for f in manifest["files"] if f["path"] == file_path)
    size = manifest["chunk_size"]

    with open(dest, "ab") as f:          # выделяем место, сохраняя уже скачанное
        f.truncate(entry["size"])

    def fetch(item):
        index, digest = item
        with open(dest, "r+b") as f:
            f.seek(index * size)
            if hashlib.sha256(f.read(size)).hexdigest() == digest:
                return                          # уже скачан: докачка
        resp = session.get(f"{ENDPOINT}/media/chunks/{digest}", timeout=300)
        resp.raise_for_status()
        if hashlib.sha256(resp.content).hexdigest() != digest:
            raise IOError(f"chunk {index}: digest mismatch")
        with open(dest, "r+b") as f:
            f.seek(index * size)
            f.write(resp.content)

    with ThreadPoolExecutor(max_workers=8) as pool:
        list(pool.map(fetch, enumerate(entry["chunks"])))

    h = hashlib.sha256()
    with open(dest, "rb") as f:
        while block := f.read(CHUNK_SIZE):
            h.update(block)
    if h.hexdigest() != entry["sha256"]:
        raise IOError(f"{file_path}: file digest mismatch")


if __name__ == "__main__":
    download("tb-labs/RuVision-7B-Instruct",
             "model-00001-of-00003.safetensors",
             "model-00001-of-00003.safetensors")

# 1. Есть ли этот чанк на сервере? (200 — да, 404 — нужно загрузить)
curl -I -H "Authorization: Bearer $TB_TOKEN" "$TB_ENDPOINT/media/chunks/$SHA"

# 2. Загрузка чанка №0 файла (байты 0 … 64 МиБ − 1)
dd if=model-00001-of-00003.safetensors bs=64M skip=0 count=1 status=none \
  | curl --fail -X PUT --data-binary @- \
      -H "Authorization: Bearer $TB_TOKEN" \
      -H "Content-Type: application/octet-stream" \
      -H "X-TB-Upload-Id: $UPLOAD_ID" \
      "$TB_ENDPOINT/media/chunks/$SHA"

# 3. Скачивание чанка; -C - докачивает прерванную передачу через Range
curl --fail -L -C - -o "chunks/$SHA" \
  -H "Authorization: Bearer $TB_TOKEN" \
  "$TB_ENDPOINT/media/chunks/$SHA"

# 4. Проверка
echo "$SHA  chunks/$SHA" | sha256sum -c -