139 lines
4.7 KiB
Python
139 lines
4.7 KiB
Python
|
# -*- coding: utf-8 -*-
|
|||
|
|
"""OpenCreator 文件树统计（可复算）
|
|||
|
|
输入：取证/opencreator/tree_*.json（本棒分片拉取）＋ 取证/api/opencreator.tree.top.json（第 1 棒顶层）
|
|||
|
|
输出：取证/opencreator/opencreator.tree.stat.json ＋ 控制台摘要
|
|||
|
|
口径：只数 type=='blob'；顶层目录取 path 第一段。
|
|||
|
|
"""
|
|||
|
|
import json, os, collections
|
|||
|
|
|
|||
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|||
|
|
OC = os.path.join(HERE, "opencreator")
|
|||
|
|
API = os.path.join(HERE, "api")
|
|||
|
|
|
|||
|
|
# 切片树里的 path 是相对「该切片根」的，必须补回前缀才是仓根相对路径
|
|||
|
|
SLICES = {
|
|||
|
|
"tree_.github.json": ".github",
|
|||
|
|
"tree_apps.json": "apps",
|
|||
|
|
"tree_assets.json": "assets",
|
|||
|
|
"tree_packages.json": "packages",
|
|||
|
|
"tree_resources.json": "resources",
|
|||
|
|
"tree_runtime.json": "runtime",
|
|||
|
|
"tree_scripts.json": "scripts",
|
|||
|
|
"tree_skills.json": "skills",
|
|||
|
|
"tree_template.json": "template",
|
|||
|
|
"tree_docs.json": "docs",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
blobs = [] # (path, size)
|
|||
|
|
trees = 0
|
|||
|
|
for f, prefix in SLICES.items():
|
|||
|
|
d = json.load(open(os.path.join(OC, f), encoding="utf-8"))
|
|||
|
|
assert not d.get("truncated"), f + " truncated"
|
|||
|
|
for e in d["tree"]:
|
|||
|
|
if e["type"] == "blob":
|
|||
|
|
blobs.append((prefix + "/" + e["path"], e.get("size", 0)))
|
|||
|
|
else:
|
|||
|
|
trees += 1
|
|||
|
|
|
|||
|
|
root = json.load(open(os.path.join(API, "opencreator.tree.top.json"), encoding="utf-8"))
|
|||
|
|
for e in root["tree"]:
|
|||
|
|
if e["type"] == "blob":
|
|||
|
|
blobs.append((e["path"], e.get("size", 0)))
|
|||
|
|
else:
|
|||
|
|
trees += 1
|
|||
|
|
|
|||
|
|
# 去重（docs/specs 在 docs 整树里已含）
|
|||
|
|
seen, uniq = set(), []
|
|||
|
|
for p, s in blobs:
|
|||
|
|
if p in seen:
|
|||
|
|
continue
|
|||
|
|
seen.add(p)
|
|||
|
|
uniq.append((p, s))
|
|||
|
|
blobs = uniq
|
|||
|
|
|
|||
|
|
by_top = collections.Counter()
|
|||
|
|
for p, _ in blobs:
|
|||
|
|
by_top[p.split("/")[0]] += 1
|
|||
|
|
|
|||
|
|
def sub(prefix, n=2):
|
|||
|
|
c = collections.Counter()
|
|||
|
|
for p, _ in blobs:
|
|||
|
|
if p.startswith(prefix + "/"):
|
|||
|
|
parts = p.split("/")
|
|||
|
|
c["/".join(parts[:n])] += 1
|
|||
|
|
return c
|
|||
|
|
|
|||
|
|
ext = collections.Counter()
|
|||
|
|
for p, _ in blobs:
|
|||
|
|
ext[os.path.splitext(p)[1].lower() or "(noext)"] += 1
|
|||
|
|
|
|||
|
|
apps = sub("apps", 3)
|
|||
|
|
daemon_src = collections.Counter()
|
|||
|
|
for p, _ in blobs:
|
|||
|
|
if p.startswith("apps/daemon/src/"):
|
|||
|
|
parts = p.split("/")
|
|||
|
|
daemon_src["/".join(parts[:4]) if len(parts) > 3 else "/".join(parts[:3])] += 1
|
|||
|
|
web_src = collections.Counter()
|
|||
|
|
for p, _ in blobs:
|
|||
|
|
if p.startswith("apps/web/src/"):
|
|||
|
|
parts = p.split("/")
|
|||
|
|
web_src["/".join(parts[:4]) if len(parts) > 3 else "/".join(parts[:3])] += 1
|
|||
|
|
rt = sub("runtime/krillinai", 3)
|
|||
|
|
tpl = collections.Counter()
|
|||
|
|
for p, _ in blobs:
|
|||
|
|
if p.startswith("template/"):
|
|||
|
|
parts = p.split("/")
|
|||
|
|
tpl[parts[1]] += 1
|
|||
|
|
# 模板条目数（二级目录）
|
|||
|
|
tpl_entries = collections.Counter()
|
|||
|
|
for p, _ in blobs:
|
|||
|
|
if p.startswith("template/") and len(p.split("/")) >= 3:
|
|||
|
|
tpl_entries["/".join(p.split("/")[:2])] += 1
|
|||
|
|
# writing-templates 模板条目
|
|||
|
|
wt = collections.Counter()
|
|||
|
|
for p, _ in blobs:
|
|||
|
|
if p.startswith("packages/writing-templates/templates/"):
|
|||
|
|
wt["/".join(p.split("/")[:4])] += 1
|
|||
|
|
|
|||
|
|
test_files = [p for p, _ in blobs if ".test." in p or ".spec." in p]
|
|||
|
|
|
|||
|
|
out = {
|
|||
|
|
"repo": "krillinai/OpenCreator",
|
|||
|
|
"branch": "master",
|
|||
|
|
"total_blobs": len(blobs),
|
|||
|
|
"total_trees": trees,
|
|||
|
|
"total_bytes": sum(s for _, s in blobs),
|
|||
|
|
"blobs_by_top_dir": dict(by_top.most_common()),
|
|||
|
|
"ext_top20": dict(ext.most_common(20)),
|
|||
|
|
"apps_by_3": dict(apps.most_common()),
|
|||
|
|
"apps_daemon_src": dict(daemon_src.most_common()),
|
|||
|
|
"apps_web_src": dict(web_src.most_common()),
|
|||
|
|
"runtime_krillinai": dict(rt.most_common()),
|
|||
|
|
"template_by_category": dict(tpl.most_common()),
|
|||
|
|
"template_entries_by_category": {k: sum(1 for p, _ in blobs if p.startswith(k + "/"))
|
|||
|
|
for k in tpl},
|
|||
|
|
"writing_templates_entries": dict(wt.most_common()),
|
|||
|
|
"test_file_count": len(test_files),
|
|||
|
|
"test_files_sample": test_files[:15],
|
|||
|
|
}
|
|||
|
|
with open(os.path.join(OC, "opencreator.tree.stat.json"), "w", encoding="utf-8") as f:
|
|||
|
|
json.dump(out, f, ensure_ascii=False, indent=1)
|
|||
|
|
|
|||
|
|
print("total_blobs=%d total_trees=%d total_bytes=%d test_files=%d"
|
|||
|
|
% (len(blobs), trees, out["total_bytes"], len(test_files)))
|
|||
|
|
print("顶层目录 blob 数:", dict(by_top.most_common()))
|
|||
|
|
print()
|
|||
|
|
print("apps 三层:", dict(apps.most_common(12)))
|
|||
|
|
print()
|
|||
|
|
print("daemon/src 四层:", dict(daemon_src.most_common(15)))
|
|||
|
|
print()
|
|||
|
|
print("web/src 四层:", dict(web_src.most_common(15)))
|
|||
|
|
print()
|
|||
|
|
print("runtime/krillinai 三层:", dict(rt.most_common(20)))
|
|||
|
|
print()
|
|||
|
|
print("template 分类 blob 数:", dict(tpl.most_common()))
|
|||
|
|
print("template 条目数(二级目录):", dict(out["template_entries_by_category"]))
|
|||
|
|
print()
|
|||
|
|
print("writing-templates 条目:", dict(wt.most_common()))
|
|||
|
|
print()
|
|||
|
|
print("扩展名 top20:", dict(ext.most_common(20)))
|