Skip to content
Documentation

Inventory

FieldValue
TypeSkill Resource
Source~/.copilot/skills/architecture/scripts/inventory.py
DescriptionNot specified

Source Content

#!/usr/bin/env python3
"""Legacy code inventory — phase 1 of the discovery playbook.


Walks a legacy repository and classifies every artifact by platform and type
(RPG, DDS, CL, COBOL, copybook, JCL, VSAM/CICS, SQL, ...), separating genuine
legacy source from modern glue and docs/build/binary noise — so you know the
shape of the system before spending context reading any single program.


Classification matches the extension cheat-sheets in the skill's decoders:
  references/ibm-i-as400.md, references/mainframe-cobol.md, references/cics-online.md


Usage:
    python3 inventory.py <repo_dir> [--out <dir>]


Always prints a Markdown summary to stdout. With --out, also writes
<out>/inventory.md (summary + per-file table) and <out>/inventory.csv.


No third-party dependencies; Python 3.8+.
"""
from __future__ import annotations


import argparse
import csv
import os
import sys
from collections import defaultdict


# Legacy platform families (everything else is "Modern" or "Non-source").
LEGACY_PLATFORMS = {"IBM i", "Mainframe/COBOL", "Mainframe", "CICS", "Database"}


# extension -> (human type, platform family)
EXT_MAP = {
    # --- IBM i / AS400 ---
    ".rpg": ("RPG III (fixed / OPM)", "IBM i"),
    ".rpg36": ("RPG/36", "IBM i"),
    ".rpg38": ("RPG/38", "IBM i"),
    ".rpgle": ("RPG IV / free (ILE)", "IBM i"),
    ".sqlrpgle": ("SQLRPGLE (RPG + embedded SQL)", "IBM i"),
    ".clp": ("CL (OPM)", "IBM i"),
    ".cl": ("CL", "IBM i"),
    ".clle": ("CL (ILE)", "IBM i"),
    ".cblle": ("COBOL (ILE, IBM i)", "IBM i"),
    ".pf": ("DDS physical file (table)", "IBM i"),
    ".lf": ("DDS logical file (view/index)", "IBM i"),
    ".dspf": ("DDS display file (green screen)", "IBM i"),
    ".prtf": ("DDS printer file (report)", "IBM i"),
    ".dds": ("DDS (unspecified)", "IBM i"),
    ".bnd": ("Binder source", "IBM i"),
    ".bnddir": ("Binding directory", "IBM i"),
    # --- Mainframe COBOL / z/OS ---
    ".cbl": ("COBOL program", "Mainframe/COBOL"),
    ".cob": ("COBOL program", "Mainframe/COBOL"),
    ".cobol": ("COBOL program", "Mainframe/COBOL"),
    ".cpy": ("Copybook (record layout)", "Mainframe/COBOL"),
    ".cpybk": ("Copybook (record layout)", "Mainframe/COBOL"),
    ".copy": ("Copybook (record layout)", "Mainframe/COBOL"),
    ".jcl": ("JCL job", "Mainframe/COBOL"),
    ".job": ("JCL job", "Mainframe/COBOL"),
    ".prc": ("JCL procedure (PROC)", "Mainframe/COBOL"),
    ".proc": ("JCL procedure (PROC)", "Mainframe/COBOL"),
    ".pli": ("PL/I program", "Mainframe"),
    ".pl1": ("PL/I program", "Mainframe"),
    ".asm": ("Assembler (HLASM)", "Mainframe"),
    ".mlc": ("Assembler macro", "Mainframe"),
    ".rexx": ("REXX script", "Mainframe"),
    ".rex": ("REXX script", "Mainframe"),
    # --- CICS ---
    ".bms": ("BMS map (CICS screen)", "CICS"),
    ".csd": ("CICS resource definitions", "CICS"),
    # --- Cross-cutting ---
    ".sql": ("SQL DDL/DML", "Database"),
}


# Modern programming languages / scripts — relevant as glue/UI, not legacy.
MODERN_EXT = {
    ".java", ".kt", ".scala", ".groovy", ".js", ".mjs", ".cjs", ".ts", ".tsx",
    ".jsx", ".vue", ".py", ".go", ".rb", ".php", ".cs", ".fs", ".swift",
    ".c", ".cc", ".cpp", ".cxx", ".h", ".hpp", ".rs", ".sh", ".bash", ".zsh",
    ".ps1", ".bat", ".cmd_", ".pl", ".pm", ".lua", ".r",
}


# Docs / build / config / data / images / binaries — noise for discovery.
NON_SOURCE_EXT = {
    # docs
    ".md", ".markdown", ".txt", ".rst", ".adoc", ".pdf", ".rtf",
    # web/style (static assets)
    ".html", ".htm", ".css", ".scss", ".less",
    # config / build / project
    ".json", ".yaml", ".yml", ".xml", ".properties", ".toml", ".ini", ".cfg",
    ".conf", ".lock", ".gradle", ".sln", ".csproj", ".vbproj", ".mfdirset",
    ".mod", ".ctl", ".cmd", ".editorconfig", ".gitignore", ".gitattributes", ".dockerignore",
    ".makefile", ".mk", ".cmake", ".bnddir_", ".j2", ".tpl", ".template",
    # data
    ".csv", ".tsv", ".data", ".dat", ".dai", ".log", ".out", ".parquet",
    # images / media
    ".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico", ".bmp", ".tiff", ".webp",
    ".mp4", ".mov", ".woff", ".woff2", ".ttf", ".eot",
    # archives / binaries
    ".zip", ".tar", ".gz", ".tgz", ".bz2", ".7z", ".jar", ".war", ".ear",
    ".dll", ".so", ".dylib", ".exe", ".bin", ".class", ".o", ".a", ".obj",
    ".lib", ".pdb", ".pyc",
}


SKIP_DIRS = {".git", ".svn", ".hg", "node_modules", "__pycache__",
             ".idea", ".vscode", ".mvn", "target", "dist", "build", ".gradle"}




def read_bytes(path: str, n: int = 200_000) -> bytes:
    try:
        with open(path, "rb") as fh:
            return fh.read(n)
    except OSError:
        return b""




def is_binary(sample: bytes) -> bool:
    return b"\x00" in sample[:4096]




def sniff_type(text: str) -> tuple[str, str]:
    """Guess (type, platform) from content — thresholded to avoid false hits.


    A single stray character in column 6 must NOT classify a file; we require
    several matching lines (or a definitive banner) before committing.
    """
    lines = text.splitlines()
    head_upper = "\n".join(lines[:80]).upper()


    # Definitive banners first.
    if "IDENTIFICATION DIVISION" in head_upper:
        return ("COBOL program", "Mainframe/COBOL")
    if "DFHMSD" in head_upper or "DFHMDI" in head_upper or "DFHMDF" in head_upper:
        return ("BMS map (CICS screen)", "CICS")
    if "**FREE" in head_upper or "CTL-OPT" in head_upper or "DCL-PROC" in head_upper:
        return ("RPG IV / free (ILE)", "IBM i")


    # Thresholded fixed-format detection over the first 200 lines.
    dds = rpg = jcl = cl = 0
    for s in lines[:200]:
        if s.startswith("//") and " JOB " in (s.upper() + " "):
            jcl += 1
        if len(s) >= 6 and s[:5] == "     " and s[5] == "A":
            dds += 1
        if len(s) >= 6 and s[:5] == "     " and s[5].upper() in "HFDICOP":
            rpg += 1
        st = s.strip().upper()
        if st == "PGM" or st.startswith("PGM ") or st == "ENDPGM":
            cl += 1


    if jcl >= 1:
        return ("JCL job", "Mainframe/COBOL")
    if dds >= 3:
        return ("DDS (unspecified)", "IBM i")
    if rpg >= 3:
        return ("RPG (fixed-format)", "IBM i")
    if cl >= 2:
        return ("CL", "IBM i")
    if "CREATE TABLE" in head_upper or "EXEC SQL" in head_upper:
        return ("SQL DDL/DML", "Database")
    return ("Unknown / unclassified", "?")




def classify(path: str) -> tuple[str, str, bool]:
    """Return (type, platform, is_cics)."""
    ext = os.path.splitext(path)[1].lower()


    if ext in NON_SOURCE_EXT:
        return ("Non-source (docs/build/data/binary)", "Non-source", False)
    if ext in MODERN_EXT:
        return ("Modern source (UI/glue/scripts)", "Modern", False)


    if ext in EXT_MAP:
        typ, plat = EXT_MAP[ext]
        if typ.startswith("COBOL program"):
            up = read_bytes(path).decode("utf-8", "replace").upper()
            if "EXEC CICS" in up:
                return ("COBOL program (CICS online)", "CICS", True)
            return ("COBOL program (batch)", plat, False)
        return (typ, plat, False)


    # Unknown / no extension: skip binaries, else sniff by content.
    sample = read_bytes(path)
    if not sample:
        return ("Empty / unreadable", "Non-source", False)
    if is_binary(sample):
        return ("Binary (non-source)", "Non-source", False)
    text = sample.decode("utf-8", "replace")
    typ, plat = sniff_type(text)
    if typ.startswith("COBOL") and "EXEC CICS" in text.upper():
        return ("COBOL program (CICS online)", "CICS", True)
    return (typ, plat, False)




def count_lines(path: str) -> int:
    try:
        with open(path, "rb") as fh:
            return sum(1 for _ in fh)
    except OSError:
        return 0




def walk(repo: str):
    rows = []
    for root, dirs, files in os.walk(repo):
        dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
        for name in files:
            full = os.path.join(root, name)
            rel = os.path.relpath(full, repo)
            typ, plat, is_cics = classify(full)
            rows.append({
                "path": rel,
                "ext": os.path.splitext(name)[1].lower() or "(none)",
                "type": typ,
                "platform": plat,
                "lines": count_lines(full),
                "cics": "yes" if is_cics else "-",
            })
    return rows




def summarize_md(rows: list[dict], repo: str) -> str:
    by_type = defaultdict(lambda: {"count": 0, "lines": 0, "platform": "?"})
    by_platform = defaultdict(int)
    for r in rows:
        by_type[r["type"]]["count"] += 1
        by_type[r["type"]]["lines"] += r["lines"]
        by_type[r["type"]]["platform"] = r["platform"]
        by_platform[r["platform"]] += 1


    legacy = sum(n for p, n in by_platform.items() if p in LEGACY_PLATFORMS)
    modern = by_platform.get("Modern", 0)
    noise = by_platform.get("Non-source", 0) + by_platform.get("?", 0)


    out = [f"# Code Inventory — `{os.path.abspath(repo)}`", ""]
    out.append(f"**Total files:** {len(rows)}  ")
    out.append(f"**Legacy source artifacts:** {legacy}  ")
    out.append(f"**Modern glue/UI:** {modern}  &nbsp;•&nbsp; **Docs/build/binary/unclassified:** {noise}")
    out.append("")
    legacy_plats = ", ".join(f"{p} ({n})" for p, n in sorted(by_platform.items(), key=lambda x: -x[1]) if p in LEGACY_PLATFORMS)
    out.append(f"**Legacy platforms present:** {legacy_plats or '-'}")
    out.append("")
    out.append("## By artifact type")
    out.append("")
    out.append("| Type | Platform | Count | Lines |")
    out.append("|---|---|---:|---:|")
    for typ, agg in sorted(by_type.items(), key=lambda x: (x[1]["platform"] not in LEGACY_PLATFORMS, -x[1]["count"])):
        out.append(f"| {typ} | {agg['platform']} | {agg['count']} | {agg['lines']:,} |")
    out.append("")
    out.append("## Files")
    out.append("")
    out.append("| Path | Type | Platform | Lines | CICS |")
    out.append("|---|---|---|---:|:--:|")
    for r in sorted(rows, key=lambda x: (x["platform"] not in LEGACY_PLATFORMS, x["platform"], x["type"], x["path"])):
        out.append(f"| `{r['path']}` | {r['type']} | {r['platform']} | {r['lines']:,} | {r['cics']} |")
    out.append("")
    return "\n".join(out)




def write_csv(rows: list[dict], path: str) -> None:
    with open(path, "w", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=["path", "ext", "type", "platform", "lines", "cics"])
        w.writeheader()
        w.writerows(rows)




def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description="Classify legacy code artifacts (phase 1 inventory).")
    ap.add_argument("repo", help="path to the legacy repository")
    ap.add_argument("--out", help="directory to write inventory.md and inventory.csv")
    args = ap.parse_args(argv)


    if not os.path.isdir(args.repo):
        print(f"error: not a directory: {args.repo}", file=sys.stderr)
        return 2


    rows = walk(args.repo)
    md = summarize_md(rows, args.repo)
    print(md)


    if args.out:
        os.makedirs(args.out, exist_ok=True)
        with open(os.path.join(args.out, "inventory.md"), "w") as fh:
            fh.write(md + "\n")
        write_csv(rows, os.path.join(args.out, "inventory.csv"))
        print(f"\n<!-- wrote {args.out}/inventory.md and inventory.csv -->", file=sys.stderr)
    return 0




if __name__ == "__main__":
    raise SystemExit(main())