contentm_agent/执行会话/目标-调研5个开源内容工作台项目并生成分析文档-5199a6/取证/_fetch_opencreator_docs.py

# -*- coding: utf-8 -*-
"""OpenCreator 正文取证拉取（raw 优先，失败回落 GitHub contents API）
通道经验：第 3 棒记 raw 不稳；本棒实测 raw 3/3 正常 ⇒ 先 raw，失败再走 contents。
输出：取证/opencreator/oc_<path 的 / 换成 _>
"""
import json, os, sys, time, base64, urllib.request, urllib.error

REPO = "krillinai/OpenCreator"
BRANCH = "master"
HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, "opencreator")
os.makedirs(OUT, exist_ok=True)

FILES = [
    # 根
    "AGENTS.md", "package.json", "tsconfig.base.json", ".env.example",
    "pnpm-workspace.yaml", "CONTRIBUTING.md",
    # docs 设计与契约
    "docs/2026-07-03-codex-native-agent-runtime-design.md",
    "docs/runtime-api-for-ui-v1.md",
    "docs/specs/2026-09-19-codex-native-image-provider-design.md",
    "docs/visual-component-guidelines.md",
    "docs/creator-artifact-versions.md",
    "docs/ui-prototype-runtime-review.md",
    "docs/opencreator-user-guide-and-troubleshooting.md",
    "docs/funasr-transcription.md",
    "docs/contributing/skills-contributing.md",
    "docs/contributing/templates-contributing.md",
    "docs/development/codex-image-generation.md",
    "docs/development/creator-error-diagnostics.md",
    "docs/development/remotion-components.md",
    "docs/development/runtime-connection-recovery.md",
    "docs/development/video-translation-components.md",
    "docs/operations/opencreator-desktop-release-runbook.md",
    "docs/operations/2026-07-15-scheduled-task-dedicated-thread-release-runbook.md",
    "docs/operations/pr-338-模板规范审查.md",
    "docs/test-reports/2026-07-15-scheduled-task-dedicated-thread-final-acceptance.md",
    "docs/superpowers/specs/2026-09-20-youtube-shorts-stickman-codex-native-design.md",
    "docs/zh/README.md",
    # packages/protocol（契约）
    "packages/protocol/package.json",
    "packages/protocol/src/index.ts",
    "packages/protocol/src/creator.ts",
    "packages/protocol/src/api.ts",
    "packages/protocol/src/creator-services.ts",
    "packages/protocol/src/creator-provider-catalog.ts",
    "packages/protocol/src/creator-download.ts",
    "packages/protocol/src/creator-visual-assets.ts",
    "packages/protocol/src/errors.ts",
    "packages/protocol/src/events.ts",
    "packages/protocol/src/issues.ts",
    "packages/protocol/src/media-generation.ts",
    "packages/protocol/src/settings.ts",
    "packages/protocol/src/smart-dubbing.ts",
    "packages/protocol/src/stickman.ts",
    "packages/protocol/src/video-metadata.ts",
    "packages/protocol/src/wechat-article.ts",
    # packages/skill-market
    "packages/skill-market/package.json",
    "packages/skill-market/src/catalog.ts",
    "packages/skill-market/src/types.ts",
    "packages/skill-market/src/source-categories.json",
    "packages/skill-market/src/source-skills.json",
    # skills（自带技能）
    "skills/krillinai-cli/SKILL.md",
    "skills/krillinai-cli/references/cli-contract.md",
    "skills/krillinai-cover/SKILL.md",
    "skills/krillinai-pipeline/SKILL.md",
    "skills/krillinai-render-horizontal/SKILL.md",
    "skills/krillinai-render-vertical/SKILL.md",
    "skills/krillinai-subtitle/SKILL.md",
    "skills/krillinai-tts/SKILL.md",
    # runtime（内嵌 Go CLI）
    "runtime/krillinai/go.mod",
    # apps 包清单
    "apps/daemon/package.json",
    "apps/web/package.json",
    "apps/desktop/package.json",
]


def raw_url(p):
    return "https://raw.githubusercontent.com/%s/%s/%s" % (REPO, BRANCH, p)


def api_url(p):
    from urllib.parse import quote
    return "https://api.github.com/repos/%s/contents/%s?ref=%s" % (REPO, quote(p), BRANCH)


def http(url, tries=3, timeout=40):
    last = None
    for i in range(tries):
        try:
            req = urllib.request.Request(url, headers={
                "User-Agent": "wb-research/1.0",
                "Accept": "application/vnd.github+json",
            })
            with urllib.request.urlopen(req, timeout=timeout) as r:
                return r.read()
        except Exception as e:
            last = e
            time.sleep(1.5)
    raise last


def main():
    log = []
    ok = 0
    for p in FILES:
        safe = p.replace("/", "_")
        dst = os.path.join(OUT, "oc_" + safe)
        chan = "raw"
        try:
            body = http(raw_url(p))
            # raw 对不存在的路径会返回 404 -> urllib 抛 HTTPError；此处已成功
        except Exception:
            chan = "api"
            try:
                j = json.loads(http(api_url(p)).decode("utf-8"))
                body = base64.b64decode(j["content"]) if j.get("encoding") == "base64" else j["content"].encode("utf-8")
            except Exception as e2:
                log.append({"path": p, "ok": False, "err": "%s | %s" % (chan, e2)})
                print("FAIL %-70s %s" % (p, e2))
                continue
        with open(dst, "wb") as f:
            f.write(body)
        ok += 1
        log.append({"path": p, "ok": True, "channel": chan, "bytes": len(body)})
        print("OK   %-70s %-4s %d" % (p, chan, len(body)))
    with open(os.path.join(OUT, "_content_fetch_log.json"), "w", encoding="utf-8") as f:
        json.dump({"repo": REPO, "branch": BRANCH, "total": len(FILES), "ok": ok, "items": log},
                  f, ensure_ascii=False, indent=1)
    print("--- ok %d / %d ---" % (ok, len(FILES)))


if __name__ == "__main__":
    main()