diff options
| author | Claude <noreply@anthropic.com> | 2026-11-06 00:54:48 +0000 |
|---|---|---|
| committer | Claude <noreply@anthropic.com> | 2026-11-06 00:54:48 +0000 |
| commit | 7aba3deae3ff17d032c37e07d2929d1426b34278 (patch) | |
| tree | 9c482f9acda1599bd2f9bed95a6a46b75e634e80 /plugins/code-modernization/hooks/reader | |
| parent | 5472f5f56e0070feeeb464318b316714fe3e9105 (diff) | |
| parent | b8331dd69ab3e132bbf8cd6e89b5d8c1747e21b1 (diff) | |
Merge remote-tracking branch 'origin/main' into add-plugin/xcodeadd-plugin/xcode
Diffstat (limited to 'plugins/code-modernization/hooks/reader')
| -rw-r--r-- | plugins/code-modernization/hooks/reader/brief.ts | 206 | ||||
| -rw-r--r-- | plugins/code-modernization/hooks/reader/discover.ts | 341 | ||||
| -rw-r--r-- | plugins/code-modernization/hooks/reader/estate-model.ts | 213 | ||||
| -rw-r--r-- | plugins/code-modernization/hooks/reader/fs.ts | 79 | ||||
| -rw-r--r-- | plugins/code-modernization/hooks/reader/modernized.ts | 409 | ||||
| -rw-r--r-- | plugins/code-modernization/hooks/reader/progress.ts | 985 | ||||
| -rw-r--r-- | plugins/code-modernization/hooks/reader/rules.ts | 438 | ||||
| -rw-r--r-- | plugins/code-modernization/hooks/reader/topology.ts | 243 | ||||
| -rw-r--r-- | plugins/code-modernization/hooks/reader/tracks.ts | 89 | ||||
| -rw-r--r-- | plugins/code-modernization/hooks/reader/uplift.ts | 285 | ||||
| -rw-r--r-- | plugins/code-modernization/hooks/reader/verification.ts | 164 |
11 files changed, 3452 insertions, 0 deletions
diff --git a/plugins/code-modernization/hooks/reader/brief.ts b/plugins/code-modernization/hooks/reader/brief.ts new file mode 100644 index 00000000..8ced2934 --- /dev/null +++ b/plugins/code-modernization/hooks/reader/brief.ts @@ -0,0 +1,206 @@ +import { linesOf } from '../text' + +/** + * `analysis/<system>/MODERNIZATION_BRIEF.md`, read without a model: the + * target stack, the phases, which modules each phase names, the exit + * criteria boxes, and the approval block. + * + * The brief is prose an agent wrote, so every field is best-effort and a + * field that cannot be read is absent, never guessed. + */ + +export type Criterion = { + /** The line as written, checkbox included. */ + line: string + /** The criterion's text: the line without its checkbox and tick annotation. */ + text: string + isTicked: boolean +} + +export type Phase = { + number: number + title: string + /** `S`, `M`, `L`, `XL`, when the heading says. */ + size?: string + risk?: string + /** Module ids (as the caller's `knownModules` names them) the phase's text mentions. */ + modules: string[] + criteria: Criterion[] +} + +export type Approval = { + isSigned: boolean + by?: string + date?: string + /** `phase-1`, `full`, or undefined when the block does not say. */ + covers?: 'phase-1' | 'full' +} + +export type Brief = { + target?: string + phases: Phase[] + approval: Approval + /** Unticked `- [ ]` boxes under the Open Questions section. */ + openQuestions: number +} + +const PHASE_HEADING = /^#{2,4}\s+Phase\s+(\d+)\s*[—–:.\-]\s*(.+?)\s*$/i +const CHECKBOX = /^\s*[-*]\s+\[( |x|X)\]\s+(.*)$/ +const TICK_NOTE = /\s*\((?:[^()]*\d{4}-\d{2}-\d{2}[^()]*)\)\s*$/ + +const strip = (text: string) => text.replace(/\*\*/g, '').replace(/`/g, '').trim() + +/** The criterion text of a checkbox line, its tick annotation `(who, date)` removed. */ +export function criterionTextOf(line: string): string | null { + const match = CHECKBOX.exec(line) + + if (match === null) { + return null + } + + return strip((match[2] ?? '').replace(TICK_NOTE, '')).replace(/\s+/g, ' ') +} + +/** + * Parses the brief. + * + * @param text the file + * @param knownModules module ids from topology.json; a phase lists the ones + * its text mentions as whole words + */ +export function parseBrief(text: string, knownModules: readonly string[] = []): Brief { + const lines = linesOf(text) + + const titleLine = lines.find(line => /^#\s+/.test(line)) ?? '' + const arrow = /[→>]\s*(.+?)\s*$/.exec(titleLine.replace(/->/g, '→')) + // The title may hold the arrow inside a parenthesis: `(COBOL/CICS → Java/Spring)`. The close is not part of the target. + const fromTitle = arrow?.[1] !== undefined ? strip(arrow[1]).replace(/\)$/, m => (arrow[1]?.includes('(') === true ? m : '')) : undefined + // The header's own `**Target stack:** `java-spring`` is the token the commands take: it wins when it is one plain token. + const stackToken = /\*\*Target stack:?\*\*:?\s*`([A-Za-z0-9][\w.+/-]{0,59})`/i.exec(text)?.[1] + const target = stackToken ?? fromTitle + + const phases: Phase[] = [] + let current: { phase: Phase; body: string[]; level: number } | null = null + let section = '' + let openQuestions = 0 + + const closePhase = () => { + if (current === null) { + return + } + + const body = current.body.join('\n') + + // The modules a phase names, the pilot's first and the rest in the order the phase + // names them: the first one open is what the next step runs, and topology order + // would put an identity screen ahead of a batch job the brief chose as its pilot. + const wholeWord = (id: string) => + new RegExp(`(?<![A-Za-z0-9_])${id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![A-Za-z0-9_])`) + + const pilotLine = /^.*\bpilot unit\b.*$/im.exec(body)?.[0] ?? '' + + current.phase.modules = knownModules + .map(id => ({ id, at: body.search(wholeWord(id)), isPilot: wholeWord(id).test(pilotLine) })) + .filter(entry => entry.at >= 0) + .sort((a, b) => Number(b.isPilot) - Number(a.isPilot) || a.at - b.at) + .map(entry => entry.id) + + phases.push(current.phase) + current = null + } + + for (const line of lines) { + const heading = /^(#{1,6})\s+(.+?)\s*$/.exec(line) + + if (heading !== null) { + const level = heading[1]?.length ?? 1 + const phase = PHASE_HEADING.exec(line) + + if (phase !== null) { + closePhase() + + const parts = (phase[2] ?? '').split('·').map(part => strip(part)) + const size = parts.map(part => /^Size\s+(\w+)/i.exec(part)?.[1]).find(Boolean) + const risk = parts.map(part => /^Risk\s+(\w+)/i.exec(part)?.[1]).find(Boolean) + + current = { + level, + body: [line], + phase: { + number: Number(phase[1]), + title: parts[0] ?? '', + ...(size !== undefined && { size: size.toUpperCase() }), + ...(risk !== undefined && { risk }), + modules: [], + criteria: [], + }, + } + + continue + } + + if (current !== null && level <= current.level) { + closePhase() + } + + if (level <= 2) { + section = strip(heading[2] ?? '').toLowerCase() + } + } + + if (current !== null) { + current.body.push(line) + + const box = CHECKBOX.exec(line) + const textOf = criterionTextOf(line) + + if (box !== null && textOf !== null) { + current.phase.criteria.push({ + line, + text: textOf, + isTicked: box[1] !== ' ', + }) + } + + continue + } + + if (/open questions/.test(section) && /^\s*[-*]\s+\[ \]/.test(line)) { + openQuestions += 1 + } + } + + closePhase() + + const approvedLine = lines.find(line => /^\s*Approved by\s*:/i.test(line)) + const by = approvedLine?.replace(/^\s*Approved by\s*:/i, '').replace(/Date\s*:.*$/i, '').trim() + const isSigned = by !== undefined && by !== '' && !/^_+$/.test(by.replace(/\s/g, '')) + + const dateLine = lines.find(line => /^\s*(?:Approved by\s*:.*)?Date\s*:\s*\S/i.test(line)) + const date = /Date\s*:\s*(\d{4}-\d{2}-\d{2}|[^\s_][^_\n]*?)\s*$/i.exec(dateLine ?? '')?.[1] + + const coversLine = lines.find(line => /Approval covers\s*:/i.test(line)) ?? '' + const ticked = [...coversLine.matchAll(/\[\s*[xX]\s*\]\s*([^[\]|]+)/g)].map(m => (m[1] ?? '').trim()) + + const plain = coversLine.replace(/^.*Approval covers\s*:/i, '').trim() + + const coversText = ticked[0] ?? (/\[/.test(coversLine) || /\|/.test(plain) ? '' : plain) + + const covers = /full/i.test(coversText) + ? ('full' as const) + : /phase\s*1/i.test(coversText) + ? ('phase-1' as const) + : undefined + + return { + ...(target !== undefined && target !== '' && { target }), + phases, + approval: { + isSigned, + ...(isSigned && by !== undefined && { by }), + ...(isSigned && date !== undefined && { date: date.trim() }), + ...(isSigned && covers !== undefined && { covers }), + }, + openQuestions, + } +} diff --git a/plugins/code-modernization/hooks/reader/discover.ts b/plugins/code-modernization/hooks/reader/discover.ts new file mode 100644 index 00000000..b6ab6117 --- /dev/null +++ b/plugins/code-modernization/hooks/reader/discover.ts @@ -0,0 +1,341 @@ +import { baseName, join, stemOf } from '../paths' +import { + groupOfDir, + languageOf, + weightOf, + type EstateModel, + type EstateUnit, +} from './estate-model' +import { listOrEmpty, type ReaderFs } from './fs' + +/** + * Reads the estate off the legacy tree when no map exists: any language, any + * layout. A directory with a build file of its own is one unit (a Maven or Gradle + * module, an npm package, a .NET project, a Go module, a Python package); a tree + * with no build files is one unit per source file when it is small enough to show, + * else one per directory. Sizes are bytes of source: what the file system knows + * without reading a file. + */ + +/** Directories that hold what a build made or what someone else wrote, never the system's own code. */ +const SKIP_DIRS = new Set([ + '.git', '.hg', '.svn', 'node_modules', 'target', 'dist', 'obj', 'build', 'out', '.gradle', '.idea', '.vscode', + '__pycache__', '.venv', 'venv', '.tox', '.mypy_cache', '.pytest_cache', 'coverage', '.next', '.nuxt', 'Pods', + 'vendor', 'bower_components', 'site-packages', +]) + +const MANIFEST_NAMES = new Set([ + 'pom.xml', 'build.gradle', 'build.gradle.kts', 'package.json', 'go.mod', 'Cargo.toml', 'pyproject.toml', + 'setup.py', 'composer.json', 'Gemfile', 'mix.exs', 'CMakeLists.txt', 'pubspec.yaml', 'Package.swift', + 'build.sbt', 'project.clj', 'deps.edn', 'cpanfile', 'Makefile.PL', 'BUILD.bazel', 'Project.toml', +]) + +const MANIFEST_SUFFIXES = ['.csproj', '.fsproj', '.vbproj', '.vcxproj'] + +const isManifest = (name: string): boolean => MANIFEST_NAMES.has(name) || MANIFEST_SUFFIXES.some(suffix => name.endsWith(suffix)) + +/** What a walk may spend: directories read, and entries seen. */ +export type Budget = { dirs: number; entries: number } + +export const DEFAULT_BUDGET: Budget = { dirs: 4000, entries: 80_000 } + +/** A file-per-unit estate is shown only up to this many source files. */ +const MAX_FILE_UNITS = 500 + +/** The most units an estate keeps; the rest are too small to draw. */ +const MAX_UNITS = 1500 + +type DirInfo = { + /** Source bytes in files directly in this directory. */ + bytes: number + /** The same, counting files of a kind no list knows: the tree may be written in a language this one does not name. */ + lenientBytes: number + files: { name: string; size: number; lenient: number; raw: number }[] + dirs: string[] + hasManifest: boolean +} + +/** Reads every directory under `root` that the budget allows, keyed by path relative to it (`''` is the root). */ +export async function walk( + fs: ReaderFs, + root: string, + budget: Budget, +): Promise<{ dirs: Map<string, DirInfo>; isPartial: boolean; files: number; bytesByLanguage: Map<string, number> }> { + const dirs = new Map<string, DirInfo>() + const bytesByLanguage = new Map<string, number>() + let queue: string[] = [''] + let readDirs = 0 + let entries = 0 + let files = 0 + let isPartial = false + + while (queue.length > 0) { + const room = budget.dirs - readDirs + + if (room <= 0 || entries > budget.entries) { + isPartial = true + + break + } + + const level = queue.slice(0, Math.min(16, room)) + + queue = queue.slice(level.length) + readDirs += level.length + + const listed = await Promise.all(level.map(rel => listOrEmpty(fs, rel === '' ? root : join(root, rel)))) + + level.forEach((rel, index) => { + const list = listed[index] ?? [] + const info: DirInfo = { bytes: 0, lenientBytes: 0, files: [], dirs: [], hasManifest: false } + + entries += list.length + + for (const entry of list) { + if (entry.kind === 'dir') { + if (!SKIP_DIRS.has(entry.name) && !entry.name.startsWith('.')) { + info.dirs.push(entry.name) + queue.push(rel === '' ? entry.name : `${rel}/${entry.name}`) + } + + continue + } + + if (entry.kind !== 'file') { + continue + } + + if (isManifest(entry.name)) { + info.hasManifest = true + } + + const strict = weightOf(entry.name, entry.size) + const lenient = weightOf(entry.name, entry.size, true) + + if (lenient > 0) { + info.files.push({ name: entry.name, size: strict, lenient, raw: entry.size }) + info.bytes += strict + info.lenientBytes += lenient + files += 1 + } + + const language = strict > 0 ? languageOf(entry.name) : undefined + + if (language !== undefined) { + bytesByLanguage.set(language, (bytesByLanguage.get(language) ?? 0) + strict) + } + } + + dirs.set(rel, info) + }) + } + + if (queue.length > 0) { + isPartial = true + } + + return { dirs, isPartial, files, bytesByLanguage } +} + +/** Every file a walk counted, by path relative to its root, with its size in bytes. */ +export function sizesOf(dirs: Map<string, DirInfo>): Map<string, number> { + const sizes = new Map<string, number>() + + for (const [rel, info] of dirs) { + for (const file of info.files) { + sizes.set(rel === '' ? file.name : `${rel}/${file.name}`, file.raw) + } + } + + return sizes +} + +/** + * The files under `root` and their sizes, read with the file API alone: nothing is run inside the tree. + * Null when the walk hit its budget, since a partial listing would call every unread file changed. + */ +export async function sourceSizes(fs: ReaderFs, root: string, budget: Budget = DEFAULT_BUDGET): Promise<Map<string, number> | null> { + const walked = await walk(fs, root, budget) + + return walked.isPartial ? null : sizesOf(walked.dirs) +} + +/** Bytes under `rel`, every directory beneath included. */ +function totalBytes(dirs: Map<string, DirInfo>, rel: string): number { + const info = dirs.get(rel) + + if (info === undefined) { + return 0 + } + + return info.dirs.reduce( + (sum, name) => sum + totalBytes(dirs, rel === '' ? name : `${rel}/${name}`), + info.bytes, + ) +} + +const under = (path: string, dir: string): boolean => dir === '' || path === dir || path.startsWith(`${dir}/`) + +/** Units where each directory with a build file is one, sized by the bytes not inside a unit beneath it. */ +function buildModules(dirs: Map<string, DirInfo>, rootLabel: string): EstateUnit[] { + const manifestDirs = [...dirs.entries()].filter(([, info]) => info.hasManifest).map(([rel]) => rel) + const set = new Set(manifestDirs) + + const ownBytes = (rel: string): number => { + const info = dirs.get(rel) + + if (info === undefined) { + return 0 + } + + return info.dirs.reduce((sum, name) => { + const child = rel === '' ? name : `${rel}/${name}` + + return set.has(child) ? sum : sum + ownBytes(child) + }, info.bytes) + } + + return manifestDirs + .map(rel => ({ + id: rel === '' ? '.' : rel, + name: rel === '' ? rootLabel : baseName(rel), + group: rel === '' ? rootLabel : groupOfDir(rel, rootLabel), + size: ownBytes(rel), + dir: rel, + })) + .filter(unit => unit.size > 0) +} + +/** One unit per source file, grouped by the directory it sits in. */ +function fileUnits(dirs: Map<string, DirInfo>, rootLabel: string): EstateUnit[] { + const units: EstateUnit[] = [] + + for (const [rel, info] of dirs) { + for (const file of info.files) { + if (file.size <= 0) { + continue + } + + const path = rel === '' ? file.name : `${rel}/${file.name}` + + units.push({ + id: path, + name: stemOf(file.name), + group: rel === '' ? rootLabel : rel, + size: Math.max(1, file.size), + file: path, + }) + } + } + + return units +} + +/** One unit per directory: the top-level ones, and the children of a lone directory that holds nearly everything. */ +function directoryUnits(dirs: Map<string, DirInfo>, rootLabel: string): EstateUnit[] { + const total = totalBytes(dirs, '') + let parent = '' + + for (let guard = 0; guard < 4; guard += 1) { + const info = dirs.get(parent) + const children = info?.dirs ?? [] + const sizes = children.map(name => ({ name, bytes: totalBytes(dirs, parent === '' ? name : `${parent}/${name}`) })) + const largest = sizes.sort((a, b) => b.bytes - a.bytes)[0] + + if (largest === undefined || total === 0 || largest.bytes / total < 0.85 || (info?.bytes ?? 0) / total > 0.05) { + break + } + + parent = parent === '' ? largest.name : `${parent}/${largest.name}` + } + + const info = dirs.get(parent) + const units: EstateUnit[] = [] + + for (const name of info?.dirs ?? []) { + const rel = parent === '' ? name : `${parent}/${name}` + const size = totalBytes(dirs, rel) + + if (size > 0) { + units.push({ id: rel, name, group: parent === '' ? rootLabel : parent, size, dir: rel }) + } + } + + if ((info?.bytes ?? 0) > 0) { + units.push({ id: parent === '' ? '.' : parent, name: parent === '' ? `${rootLabel} (files)` : `${baseName(parent)} (files)`, group: parent === '' ? rootLabel : parent, size: info?.bytes ?? 0, dir: parent }) + } + + return units +} + +/** + * The estate of one system, read off `<root>` (its directory under the legacy tree). Null when the + * directory holds no source. + */ +export async function discoverEstate( + fs: ReaderFs, + root: string, + rootLabel: string, + budget: Budget = DEFAULT_BUDGET, +): Promise<EstateModel | null> { + const walked = await walk(fs, root, budget) + + if (walked.files === 0) { + return null + } + + // A tree written in a language nobody listed has almost no weight on a strict count: count every file of an unknown + // kind as source then, so the estate is the tree and not the few configuration files beside it. + const strictTotal = [...walked.dirs.values()].reduce((sum, info) => sum + info.bytes, 0) + const lenientTotal = [...walked.dirs.values()].reduce((sum, info) => sum + info.lenientBytes, 0) + + if (lenientTotal > 0 && strictTotal < lenientTotal * 0.35) { + for (const info of walked.dirs.values()) { + info.bytes = info.lenientBytes + + for (const file of info.files) { + file.size = file.lenient + } + } + } + + const counted = [...walked.dirs.values()].reduce((sum, info) => sum + info.files.filter(file => file.size > 0).length, 0) + const manifestCount = [...walked.dirs.entries()].filter(([rel, info]) => info.hasManifest && rel !== '').length + + let units: EstateUnit[] + let granularity: EstateModel['granularity'] + + if (manifestCount >= 2) { + units = buildModules(walked.dirs, rootLabel) + granularity = 'build module' + } else if (counted <= MAX_FILE_UNITS) { + units = fileUnits(walked.dirs, rootLabel) + granularity = 'file' + } else { + units = directoryUnits(walked.dirs, rootLabel) + granularity = 'directory' + } + + if (units.length === 0) { + return null + } + + const total = [...walked.bytesByLanguage.values()].reduce((sum, bytes) => sum + bytes, 0) + + const languages = [...walked.bytesByLanguage.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 3) + .map(([name, bytes]) => ({ name, share: total > 0 ? bytes / total : 0 })) + + return { + source: 'tree', + measure: 'bytes', + granularity, + units: units.sort((a, b) => b.size - a.size).slice(0, MAX_UNITS), + files: counted, + languages, + isPartial: walked.isPartial, + fileSizes: sizesOf(walked.dirs), + } +} + diff --git a/plugins/code-modernization/hooks/reader/estate-model.ts b/plugins/code-modernization/hooks/reader/estate-model.ts new file mode 100644 index 00000000..fb8204a3 --- /dev/null +++ b/plugins/code-modernization/hooks/reader/estate-model.ts @@ -0,0 +1,213 @@ +import { baseName, dirName, norm, stemOf } from '../paths' +import { nodeOfFile, type Topology } from './topology' + +/** + * The estate: the units a system's code is divided into, whatever its language. + * They come from the map's `topology.json` when there is one and from the legacy + * tree itself when there is not, so the pane has a map from the first minute of + * a session in any workspace. + */ + +export type EstateUnit = { + id: string + name: string + /** What the tile sits under: a domain, or the directory that holds the unit. */ + group: string + /** Lines when the map gave them, bytes of source when the unit was read off the tree. */ + size: number + /** The unit as a directory, relative to `legacy/<system>/` (`''` is the system root). */ + dir?: string + /** The unit as one file, relative to `legacy/<system>/`. */ + file?: string +} + +/** What one unit is: a module the map named, a source file, a build module (a directory with its own build file), or a directory. */ +export type Granularity = 'module' | 'file' | 'build module' | 'directory' + +export type EstateModel = { + source: 'map' | 'tree' + /** What `size` counts. */ + measure: 'lines' | 'bytes' + /** What a unit is, for the pane's words. */ + granularity: Granularity + units: EstateUnit[] + /** Source files seen under the system root. */ + files: number + /** The languages the files are written in, largest first, as shares of the source bytes. */ + languages: { name: string; share: number }[] + /** True when the walk stopped at its budget: the sizes are a floor. */ + isPartial: boolean + /** Every source or configuration file the walk saw, relative to the system root, with its size in bytes. Kept to compare a working copy with the tree it was copied from; absent for an estate the map gave. */ + fileSizes?: ReadonlyMap<string, number> +} + +/** A language's name by file extension: the common ones, and the mainframe and scripting ones legacy work meets. */ +const LANGUAGES: Record<string, string> = { + java: 'Java', kt: 'Kotlin', kts: 'Kotlin', scala: 'Scala', groovy: 'Groovy', clj: 'Clojure', + cs: 'C#', vb: 'VB.NET', fs: 'F#', aspx: 'ASP.NET', ascx: 'ASP.NET', cshtml: 'Razor', asp: 'Classic ASP', + py: 'Python', rb: 'Ruby', php: 'PHP', pl: 'Perl', pm: 'Perl', lua: 'Lua', r: 'R', + js: 'JavaScript', mjs: 'JavaScript', cjs: 'JavaScript', jsx: 'JavaScript', ts: 'TypeScript', tsx: 'TypeScript', + go: 'Go', rs: 'Rust', swift: 'Swift', dart: 'Dart', ex: 'Elixir', exs: 'Elixir', erl: 'Erlang', hs: 'Haskell', + c: 'C', h: 'C', cc: 'C++', cpp: 'C++', cxx: 'C++', hpp: 'C++', m: 'Objective-C', + cbl: 'COBOL', cob: 'COBOL', cpy: 'COBOL', cobol: 'COBOL', pco: 'COBOL', jcl: 'JCL', bms: 'BMS', prc: 'JCL', + rpg: 'RPG', rpgle: 'RPG', sqlrpgle: 'RPG', clle: 'CL', pli: 'PL/I', pl1: 'PL/I', nat: 'Natural', asm: 'Assembler', + pas: 'Pascal', dpr: 'Delphi', bas: 'Basic', frm: 'VB6', cls: 'VB6', vbs: 'VBScript', + sql: 'SQL', sh: 'Shell', bash: 'Shell', ps1: 'PowerShell', bat: 'Batch', cmd: 'Batch', + jsp: 'JSP', xhtml: 'JSF', cfm: 'ColdFusion', tcl: 'Tcl', f: 'Fortran', f90: 'Fortran', for: 'Fortran', +} + +/** Extensions that are not source: assets, binaries, archives, lock files, data and prose. */ +const NOT_SOURCE = new Set([ + 'png', 'jpg', 'jpeg', 'gif', 'svg', 'ico', 'bmp', 'webp', 'pdf', 'zip', 'gz', 'tgz', 'tar', 'jar', 'war', 'ear', + 'class', 'dll', 'exe', 'so', 'dylib', 'a', 'o', 'lib', 'pyc', 'woff', 'woff2', 'ttf', 'eot', 'otf', 'mp3', 'mp4', + 'mov', 'avi', 'wav', 'ogg', 'bin', 'dat', 'db', 'sqlite', 'log', 'lock', 'md', 'txt', 'rst', 'adoc', 'csv', 'tsv', + 'map', 'tmp', 'bak', 'orig', 'swp', 'ds_store', 'jks', 'p12', 'pem', 'crt', 'key', 'der', 'iso', 'img', 'rpm', 'deb', +]) + +/** Configuration and markup a build reads: part of the system, but never the bulk of its code. */ +const CONFIG_EXT = new Set([ + 'xml', 'json', 'yaml', 'yml', 'toml', 'properties', 'gradle', 'cfg', 'conf', 'ini', 'html', 'htm', 'css', 'scss', + 'less', 'vue', 'svelte', 'xsl', 'xslt', 'xsd', 'wsdl', 'proto', 'graphql', 'ftl', 'vm', 'tpl', 'twig', 'erb', + 'jspx', 'tag', 'tld', 'dtd', 'mustache', 'hbs', +]) + +const CONFIG_NAMES = new Set(['Makefile', 'Dockerfile', 'Jenkinsfile', 'Rakefile', 'Gemfile', 'Vagrantfile', 'Procfile']) + +const extOf = (name: string): string => { + const dot = name.lastIndexOf('.') + + return dot > 0 && dot < name.length - 1 ? name.slice(dot + 1).toLowerCase() : '' +} + +/** A single file counts for at most this much: one binary blob or one generated file must not be a system's bulk. */ +const CAP_SOURCE = 400_000 +const CAP_CONFIG = 100_000 + +/** + * What a file adds to the estate's size: its bytes (capped) when it is source or configuration, nothing when it is + * an asset, an archive, a document or a file of no known kind. `isLenient` counts unknown kinds as source too, for + * a language this list does not know. + */ +export function weightOf(name: string, size: number, isLenient = false): number { + if (name.startsWith('.')) { + return 0 + } + + const ext = extOf(name) + + if (NOT_SOURCE.has(ext)) { + return 0 + } + + if (LANGUAGES[ext] !== undefined) { + return Math.min(size, CAP_SOURCE) + } + + if (CONFIG_EXT.has(ext) || CONFIG_NAMES.has(name)) { + return Math.min(size, CAP_CONFIG) + } + + return isLenient ? Math.min(size, CAP_SOURCE) : 0 +} + +/** Whether a file is one the estate counts (source or configuration). */ +export const isSourceFile = (name: string): boolean => weightOf(name, 1) > 0 + +export const languageOf = (name: string): string | undefined => LANGUAGES[extOf(name)] + +/** A language name as a person writes it, from what a map recorded (`cobol` -> `COBOL`, `java` -> `Java`). */ +const nameOfLanguage = (raw: string): string => { + const known = Object.values(LANGUAGES).find(name => name.toLowerCase() === raw.toLowerCase()) + + return known ?? (raw.length <= 4 ? raw.toUpperCase() : raw.charAt(0).toUpperCase() + raw.slice(1)) +} + +/** The estate as a topology's modules, sized in lines. */ +export function estateOfTopology(topology: Topology, files: number): EstateModel { + const byLanguage = new Map<string, number>() + + for (const node of topology.modules) { + if (node.language !== undefined) { + byLanguage.set(nameOfLanguage(node.language), (byLanguage.get(nameOfLanguage(node.language)) ?? 0) + Math.max(1, node.loc)) + } + } + + const total = [...byLanguage.values()].reduce((sum, size) => sum + size, 0) + + return { + source: 'map', + measure: 'lines', + granularity: 'module', + units: topology.modules.map(node => ({ + id: node.id, + name: node.name, + group: node.domain ?? 'Other', + size: Math.max(1, node.loc), + ...(node.file !== undefined && { file: node.file }), + })), + files, + languages: [...byLanguage.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 3) + .map(([name, size]) => ({ name, share: total > 0 ? size / total : 0 })), + isPartial: false, + } +} + +/** + * The unit a path under `legacy/<system>/` belongs to: by the map's own file + * where there is a map, else by the file itself or the nearest directory that is a unit. + */ +export function unitOfPath( + estate: EstateModel, + topology: Topology | null, + rel: string, +): EstateUnit | null { + const wanted = norm(rel) + + if (estate.source === 'map' && topology !== null) { + const node = nodeOfFile(topology, wanted) + + return node === null ? null : (estate.units.find(unit => unit.id === node.id) ?? null) + } + + const exact = estate.units.find(unit => unit.file === wanted) + + if (exact !== undefined) { + return exact + } + + let best: EstateUnit | null = null + let bestLength = -1 + + for (const unit of estate.units) { + const dir = unit.dir + + if (dir === undefined) { + continue + } + + const isInside = dir === '' || wanted === dir || wanted.startsWith(`${dir}/`) + + if (isInside && dir.length > bestLength) { + best = unit + bestLength = dir.length + } + } + + return best +} + +/** The names a transformed directory may be matched to a unit by: the id, the name, and the last path segment. */ +export function keysOfUnit(unit: EstateUnit): string[] { + const own = unit.dir !== undefined && unit.dir !== '' ? baseName(unit.dir) : unit.file !== undefined ? stemOf(unit.file) : '' + + return [unit.id, unit.name, own].filter(key => key !== '') +} + +/** The group a unit's directory sits in: its parent directory, or the root's own label. */ +export const groupOfDir = (dir: string, rootLabel: string): string => { + const parent = dirName(dir) + + return parent === '' ? rootLabel : parent +} diff --git a/plugins/code-modernization/hooks/reader/fs.ts b/plugins/code-modernization/hooks/reader/fs.ts new file mode 100644 index 00000000..1d8e9f6d --- /dev/null +++ b/plugins/code-modernization/hooks/reader/fs.ts @@ -0,0 +1,79 @@ +/** + * The file access the reader needs, narrow enough that `$.fs` and a test's + * in-memory tree both satisfy it. Paths are relative to the session's + * working directory or absolute, as `$.fs` takes them. + */ +export type ReaderFs = { + read: (path: string) => Promise<string> + /** An entry as it stands: a symbolic link is `other` (with `isLink`), whatever it leads to. `stat` says what that is. */ + list: (path: string) => Promise<{ name: string; kind: 'file' | 'dir' | 'other'; size: number; isLink?: boolean }[]> + exists: (path: string) => Promise<boolean> + stat: (path: string) => Promise<{ kind: 'file' | 'dir' | 'other'; size: number; mtimeMs: number }> +} + +/** `read`, or null when the file is missing, unreadable or over the size cap. */ +export async function readOrNull(fs: ReaderFs, path: string): Promise<string | null> { + try { + // `exists` never rejects; asking first keeps a missing optional file out of the error log. + if (!(await fs.exists(path))) { + return null + } + + return await fs.read(path) + } catch { + return null + } +} + +/** `list`, or an empty list when the directory is missing. */ +export async function listOrEmpty( + fs: ReaderFs, + path: string, +): Promise<{ name: string; kind: 'file' | 'dir' | 'other'; size: number; isLink?: boolean }[]> { + try { + if (!(await fs.exists(path))) { + return [] + } + + return await fs.list(path) + } catch { + return [] + } +} + +/** `stat`'s mtime, or null when the path is missing. */ +export async function mtimeOrNull(fs: ReaderFs, path: string): Promise<number | null> { + try { + if (!(await fs.exists(path))) { + return null + } + + return (await fs.stat(path)).mtimeMs + } catch { + return null + } +} + +/** + * The directories under `path`, by name. A symbolic link to a directory counts: the engine lists a link as `other`, + * and a legacy tree is often a link to where the code really lives. + */ +export async function dirNamesOf(fs: ReaderFs, path: string): Promise<string[]> { + const names: string[] = [] + + for (const entry of await listOrEmpty(fs, path)) { + if (entry.kind === 'dir') { + names.push(entry.name) + } else if (entry.kind === 'other') { + try { + if ((await fs.stat(`${path}/${entry.name}`)).kind === 'dir') { + names.push(entry.name) + } + } catch { + // a link that leads nowhere is not a system + } + } + } + + return names +} diff --git a/plugins/code-modernization/hooks/reader/modernized.ts b/plugins/code-modernization/hooks/reader/modernized.ts new file mode 100644 index 00000000..d60b8f66 --- /dev/null +++ b/plugins/code-modernization/hooks/reader/modernized.ts @@ -0,0 +1,409 @@ +import { join } from '../paths' +import { linesOf } from '../text' +import { listOrEmpty, readOrNull, type ReaderFs } from './fs' + +/** + * The state of each transformed module, read from `modernized/<system>/`: + * what exists on disk, what the test reports say, what the notes record. + * No inference: a fact that is not on disk is absent. + */ + +export type TestTotals = { + tests: number + failures: number + errors: number + skipped: number + /** How many report files were read. */ + reports: number +} + +export type ModuleState = + | 'scaffolded' + | 'tests-written' + | 'tests-red' + /** An uplift's tests fail and there is no baseline row to say whether the source runtime failed them too. */ + | 'tests-failing' + | 'tests-green' + | 'reviewed' + | 'ported' + | 'switched' + +export type ModernizedModule = { + /** The directory's name under `modernized/<system>/`. */ + dir: string + /** Path relative to the working directory. */ + path: string + hasMain: boolean + hasTests: boolean + hasNotes: boolean + /** The date in the notes' "Architecture review" section; absent when none. */ + reviewDate?: string + hasReviewSection: boolean + /** The notes say "ported, not switched". */ + isPortedNotSwitched: boolean + /** The notes say the route was switched. */ + isSwitched: boolean + /** Open follow-ups the notes list (bullets under a Follow-ups heading). */ + followUps: number + tests: TestTotals | null + state: ModuleState + /** Newest mtime among the notes and test reports; 0 when unknown. */ + mtimeMs: number +} + +/** States a unit of work counts as finished in: the review is done, or the module is ported or switched. */ +const DONE_STATES: ReadonlySet<ModuleState> = new Set(['reviewed', 'ported', 'switched']) + +/** + * Whether a module's work is finished. Green tests with notes written count: the notes are where the plugin's + * review step lists what the critic found, and how they headline it varies, so a missing headline is not unfinished work. + */ +export const isDone = (module: Pick<ModernizedModule, 'state' | 'hasNotes'>): boolean => + DONE_STATES.has(module.state) || (module.state === 'tests-green' && module.hasNotes) + +const EMPTY: TestTotals = { tests: 0, failures: 0, errors: 0, skipped: 0, reports: 0 } + +/** Totals of one Visual Studio test result file (`.trx`), the report `dotnet test --logger trx` leaves. */ +export function totalsOfTrx(xml: string): TestTotals | null { + const counters = /<Counters\b[^>]*>/.exec(xml)?.[0] + + if (counters === undefined || !/\btotal="\d+"/.test(counters)) { + return null + } + + return { + tests: attr(counters, 'total'), + failures: attr(counters, 'failed'), + errors: attr(counters, 'error') + attr(counters, 'timeout') + attr(counters, 'aborted'), + skipped: attr(counters, 'notExecuted') + attr(counters, 'inconclusive'), + reports: 1, + } +} + +const attr = (tag: string, name: string): number => { + const match = new RegExp(`\\b${name}="(\\d+)"`).exec(tag) + + return match?.[1] !== undefined ? Number(match[1]) : 0 +} + +/** Totals of one JUnit-style XML report (`<testsuite ...>` / `<testsuites ...>`). */ +export function totalsOfJunitXml(xml: string): TestTotals | null { + const suites = /<testsuites\b[^>]*>/.exec(xml)?.[0] + + if (suites !== undefined && /\btests="\d+"/.test(suites)) { + return { + tests: attr(suites, 'tests'), + failures: attr(suites, 'failures'), + errors: attr(suites, 'errors'), + skipped: attr(suites, 'skipped') + attr(suites, 'disabled'), + reports: 1, + } + } + + const all = [...xml.matchAll(/<testsuite\b[^>]*>/g)].map(match => match[0]) + + if (all.length === 0) { + return null + } + + return all.reduce<TestTotals>( + (sum, tag) => ({ + tests: sum.tests + attr(tag, 'tests'), + failures: sum.failures + attr(tag, 'failures'), + errors: sum.errors + attr(tag, 'errors'), + skipped: sum.skipped + attr(tag, 'skipped') + attr(tag, 'disabled'), + reports: 1, + }), + { ...EMPTY, reports: 1 }, + ) +} + +const add = (a: TestTotals, b: TestTotals): TestTotals => ({ + tests: a.tests + b.tests, + failures: a.failures + b.failures, + errors: a.errors + b.errors, + skipped: a.skipped + b.skipped, + reports: a.reports + b.reports, +}) + +/** Where test runners leave JUnit XML, relative to a module's directory. */ +const REPORT_DIRS = [ + 'target/surefire-reports', + 'target/failsafe-reports', + 'build/test-results/test', + 'build/test-results', + 'build/logs', + 'test-results', + 'test-reports', + 'reports/junit', + 'TestResults', +] + +const REPORT_FILES = ['junit.xml', 'report.xml', 'test-report.xml', 'pytest.xml'] + +/** When a file last changed; 0 when it cannot be said. */ +async function changedAt(fs: ReaderFs, path: string): Promise<number> { + try { + return (await fs.stat(path)).mtimeMs + } catch { + return 0 + } +} + +/** + * The test totals a directory's report files hold: a custom JSON report, JUnit XML, or a `.trx`, and when the newest of + * the files read last changed (what a verdict about the module is compared with). + */ +export async function readTestsAt(fs: ReaderFs, dir: string, withTime = true): Promise<{ totals: TestTotals; atMs: number } | null> { + const stamp = (path: string) => (withTime ? changedAt(fs, path) : Promise.resolve(0)) + const customPath = join(dir, '.modernize/test-report.json') + const custom = await readOrNull(fs, customPath) + + if (custom !== null) { + try { + const raw = JSON.parse(custom) as Record<string, unknown> + const num = (key: string) => (typeof raw[key] === 'number' ? (raw[key] as number) : 0) + + return { + totals: { + tests: num('tests'), + failures: num('failures'), + errors: num('errors'), + skipped: num('skipped'), + reports: 1, + }, + atMs: await stamp(customPath), + } + } catch { + // fall through to the XML reports + } + } + + let total: TestTotals | null = null + let atMs = 0 + + for (const sub of REPORT_DIRS) { + const entries = await listOrEmpty(fs, join(dir, sub)) + + for (const entry of entries) { + const isTrx = entry.name.endsWith('.trx') + + if (entry.kind !== 'file' || !(entry.name.endsWith('.xml') || isTrx) || entry.size > 3_500_000) { + continue + } + + const xml = await readOrNull(fs, join(dir, sub, entry.name)) + const totals = xml === null ? null : isTrx ? totalsOfTrx(xml) : totalsOfJunitXml(xml) + + if (totals !== null) { + total = total === null ? totals : add(total, totals) + atMs = Math.max(atMs, await stamp(join(dir, sub, entry.name))) + } + } + + if (total !== null) { + return { totals: total, atMs } + } + } + + for (const name of REPORT_FILES) { + const xml = await readOrNull(fs, join(dir, name)) + const totals = xml !== null ? totalsOfJunitXml(xml) : null + + if (totals !== null) { + return { totals, atMs: await stamp(join(dir, name)) } + } + } + + return null +} + +/** The test totals a directory's report files hold, without asking when each file changed. */ +export async function readTests(fs: ReaderFs, dir: string): Promise<TestTotals | null> { + return (await readTestsAt(fs, dir, false))?.totals ?? null +} + +const TEST_DIRS = ['src/test', 'tests', 'test', 'spec', '__tests__'] +const MAIN_DIRS = ['src/main', 'src', 'app', 'lib'] + +async function anyDir(fs: ReaderFs, dir: string, names: readonly string[]): Promise<boolean> { + for (const name of names) { + if ((await listOrEmpty(fs, join(dir, name))).length > 0) { + return true + } + } + + return false +} + +/** What the notes record, read by heading and phrase. */ +export function readNotes(notes: string): { + hasReviewSection: boolean + reviewDate?: string + isPortedNotSwitched: boolean + isSwitched: boolean + followUps: number +} { + const lines = linesOf(notes) + // The plugin's review step spawns the architecture critic and lists what it found in the notes; how the notes + // headline that varies, so any heading that says review or critic counts, and so does naming the critic. + const named = lines.findIndex(line => /^#{1,4}\s+.*\b(?:review|critic)/i.test(line)) + const start = named >= 0 ? named : -1 + let reviewDate: string | undefined + + if (start >= 0) { + const level = /^(#+)/.exec(lines[start] ?? '')?.[1]?.length ?? 2 + + const end = lines.findIndex( + (line, index) => index > start && new RegExp(`^#{1,${level}}\\s`).test(line), + ) + + const section = lines.slice(start, end < 0 ? undefined : end).join('\n') + + reviewDate = /\b(\d{4}-\d{2}-\d{2})\b/.exec(section)?.[1] + } + + const followStart = lines.findIndex(line => /^#{1,4}\s+.*follow-?ups?/i.test(line)) + let followUps = 0 + + if (followStart >= 0) { + for (const line of lines.slice(followStart + 1)) { + if (/^#{1,4}\s/.test(line)) { + break + } + + if (/^\s*[-*]\s+(?!\[[xX]\])/.test(line) || /^\s*\d+\.\s+/.test(line)) { + followUps += 1 + } + } + } + + const isPortedNotSwitched = /ported,?\s+not\s+switched/i.test(notes) + + return { + hasReviewSection: start >= 0 || /architecture[- ]critic/i.test(notes), + ...(reviewDate !== undefined && { reviewDate }), + isPortedNotSwitched, + isSwitched: + !isPortedNotSwitched && + /\b(?:route|traffic)\b[^\n]{0,40}\bswitched\b|\bswitched\b[^\n]{0,20}\b\d{4}-\d{2}-\d{2}\b/i.test( + notes, + ), + followUps, + } +} + +/** The state a module's facts add up to, furthest first. */ +export function stateOf(module: Omit<ModernizedModule, 'state'>): ModuleState { + const tests = module.tests + const isRed = tests !== null && tests.failures + tests.errors > 0 + const isGreen = tests !== null && tests.tests > 0 && !isRed + + if (module.isSwitched && isGreen && module.hasReviewSection) { + return 'switched' + } + + if (module.isPortedNotSwitched && isGreen) { + return 'ported' + } + + if (isGreen && module.hasReviewSection) { + return 'reviewed' + } + + if (isRed) { + return 'tests-red' + } + + if (isGreen) { + return 'tests-green' + } + + if (module.hasTests) { + return 'tests-written' + } + + return 'scaffolded' +} + +/** + * The state of one unit of an uplift's working copy: worse than its baseline, matching or exceeding it + * (every test that passed on the source runtime still passes, none is missing), tested but not comparable + * to a baseline row, changed and not yet tested, or not touched at all. + */ +export function stateOfUplift( + tests: TestTotals | null, + baseline: { pass: number; fail: number; error: number; skip: number } | null, + isChanged: boolean, +): ModuleState | 'untouched' { + if (tests !== null && tests.tests > 0) { + const bad = tests.failures + tests.errors + + // With no baseline row there is nothing to be worse than: the failures are said, not judged. + if (baseline === null && bad > 0) { + return 'tests-failing' + } + + if (baseline !== null && bad > baseline.fail + baseline.error) { + return 'tests-red' + } + + const passed = tests.tests - bad - tests.skipped + const isCovered = baseline !== null && tests.tests >= baseline.pass + baseline.fail + baseline.error + baseline.skip && passed >= baseline.pass + + return isCovered ? 'reviewed' : 'tests-green' + } + + return isChanged ? 'scaffolded' : 'untouched' +} + +/** Reads every code directory under `modernized/<system>/`. */ +export async function readModernized( + fs: ReaderFs, + system: string, +): Promise<ModernizedModule[]> { + return readModernizedIn(fs, join('modernized', system)) +} + +/** Reads every code directory directly under `root`: a transformed system's modules, or a reimagined system's services. */ +export async function readModernizedIn( + fs: ReaderFs, + root: string, + observed: ReadonlyMap<string, TestTotals> = new Map(), +): Promise<ModernizedModule[]> { + const entries = await listOrEmpty(fs, root) + const out: ModernizedModule[] = [] + + for (const entry of entries) { + if (entry.kind !== 'dir' || entry.name.startsWith('.')) { + continue + } + + const path = join(root, entry.name) + const notes = await readOrNull(fs, join(path, 'TRANSFORMATION_NOTES.md')) + const read = notes !== null ? readNotes(notes) : null + const reported = await readTestsAt(fs, path) + const tests = reported?.totals ?? observed.get(path) ?? null + // The newest of the notes and the test reports: what a verdict about the module is checked against. + const mtimeMs = Math.max(notes !== null ? await changedAt(fs, join(path, 'TRANSFORMATION_NOTES.md')) : 0, reported?.atMs ?? 0) + + const facts = { + dir: entry.name, + path, + hasMain: await anyDir(fs, path, MAIN_DIRS), + hasTests: await anyDir(fs, path, TEST_DIRS), + hasNotes: notes !== null, + hasReviewSection: read?.hasReviewSection ?? false, + ...(read?.reviewDate !== undefined && { reviewDate: read.reviewDate }), + isPortedNotSwitched: read?.isPortedNotSwitched ?? false, + isSwitched: read?.isSwitched ?? false, + followUps: read?.followUps ?? 0, + tests, + mtimeMs, + } + + out.push({ ...facts, state: stateOf(facts) }) + } + + return out +} diff --git a/plugins/code-modernization/hooks/reader/progress.ts b/plugins/code-modernization/hooks/reader/progress.ts new file mode 100644 index 00000000..04064cff --- /dev/null +++ b/plugins/code-modernization/hooks/reader/progress.ts @@ -0,0 +1,985 @@ +import { join } from '../paths' +import { isSystemName, isToken, plain } from '../text' +import { parseBrief, type Brief, type Phase } from './brief' +import { discoverEstate, sourceSizes } from './discover' +import { estateOfTopology, keysOfUnit, type EstateModel, type EstateUnit } from './estate-model' +import { dirNamesOf, mtimeOrNull, readOrNull, type ReaderFs } from './fs' +import { isDone, readModernizedIn, type ModernizedModule, type TestTotals } from './modernized' +import { needsReview, parseRules, type RuleSet } from './rules' +import { pickTrack, TRACK_LABELS, type TrackKey } from './tracks' +import { parseTopology, type Topology } from './topology' +import { parseVerification, proofOfModule, type ProofState, type Verification } from './verification' +import { parseLedger } from '../review/ledger' +import { + baselineRowOf, + changedUnitsOf, + parseBaseline, + parseCatalog, + changedPathsOf, + readUpliftUnits, + type Baseline, + type Catalog, +} from './uplift' + +/** + * One reading of a system's modernization state, from the artifacts on disk + * alone. Everything here is a fact of a file that exists; nothing is inferred + * by a model, and what cannot be read is absent. + */ + +export type StageKey = + | 'preflight' + | 'assess' + | 'map' + | 'rules' + | 'brief' + | 'transform' + | 'harden' + | 'deltas' + | 'baseline' + | 'pilot' + | 'migrate' + | 'compare' + | 'verify' + | 'spec' + | 'design' + | 'scaffold' + | 'tests' + +export type Stage = { + key: StageKey + label: string + isDone: boolean + mtimeMs: number | null + /** One short qualifier (`signed`, `319 rules`), when there is one. */ + detail?: string +} + +export type ReviewVerdict = 'confirmed' | 'wrong' | 'discuss' + +export type ReviewLedger = Record< + string, + { verdict: ReviewVerdict; at: string; title?: string; /** The reviewer's own words, when they gave any. */ note?: string } +> + +export type NextStep = { + /** The slash command to run, or what to do by hand. */ + text: string + isByHand: boolean + reason: string + /** A step the pane can do itself: `sign` opens the sign-off dialog. */ + action?: 'sign' +} + +/** What an uplift has left on disk, beyond the modules it changed. */ +export type UpliftFacts = { + catalog: Catalog | null + baseline: Baseline | null + hasPlaybook: boolean + hasNotes: boolean + hasCopy: boolean + /** Whether the working copy's changes could be read (a version-control status), or only its test reports. */ + isChangeKnown: boolean +} + +export type Snapshot = { + system: string + systems: string[] + /** Which way the plugin's commands are working on the system. */ + track: TrackKey + /** True once the system has any artifact under `analysis/`: modernization is under way, not only possible. */ + hasAnalysis: boolean + stages: Stage[] + /** The units the estate map draws: the map's modules, or what the legacy tree holds. */ + estate: EstateModel | null + topology: Topology | null + rules: RuleSet | null + brief: Brief | null + /** The units of work under way: transformed modules, uplifted modules, or reimagined services. */ + modules: ModernizedModule[] + /** Estate unit id to its unit of work, where one matches. */ + byNode: Map<string, ModernizedModule> + /** Units of work that match no unit of the estate. */ + extras: ModernizedModule[] + uplift: UpliftFacts | null + findings: { exists: boolean; isScan: boolean } + /** The proof pack's verdicts, when the verify command has run. */ + verification: Verification | null + /** + * Where each built module stands with the proof, by its folder name lower-cased. An uplift is one piece, so it has one + * entry, under the name of its working copy (`<system>-uplifted`). + */ + proofs: Map<string, ProofState> + reviews: ReviewLedger + /** Size of the units finished over the estate's total; null without an estate, or where units of work are not the estate's own. */ + percent: number | null + totals: { modules: number; done: number; loc: number; locDone: number } + attention: string[] + next: NextStep | null + readAtMs: number +} + +export const STAGE_LABELS: Record<StageKey, string> = { + preflight: 'preflight', + assess: 'assess', + map: 'map', + rules: 'rules', + brief: 'brief', + transform: 'transform', + harden: 'harden', + deltas: 'deltas', + baseline: 'baseline', + pilot: 'pilot', + migrate: 'migrate', + compare: 'compare', + verify: 'verify', + spec: 'spec', + design: 'design', + scaffold: 'scaffold', + tests: 'tests', +} + +const STAGE_FILES = { + preflight: 'PREFLIGHT.md', + assess: 'ASSESSMENT.md', + map: 'topology.json', + rules: 'BUSINESS_RULES.md', + brief: 'MODERNIZATION_BRIEF.md', + transform: 'TRANSFORMATION_NOTES.md', + harden: 'SECURITY_FINDINGS.md', + verify: 'VERIFICATION.json', + deltas: 'DELTA_CATALOG.md', + baseline: 'BASELINE.md', + pilot: 'PLAYBOOK.md', + spec: 'AI_NATIVE_SPEC.md', + design: 'REIMAGINED_ARCHITECTURE.md', +} as const + +export const REVIEWS_FILE = 'RULE_REVIEWS.json' + +/** Parsed files kept by path and mtime, so an unchanged file is parsed once. */ +export type ReaderCache = Map<string, { mtimeMs: number; value: unknown }> + +async function cached<T>( + fs: ReaderFs, + cache: ReaderCache, + path: string, + parse: (text: string) => T, +): Promise<{ value: T | null; mtimeMs: number | null }> { + const mtimeMs = await mtimeOrNull(fs, path) + + if (mtimeMs === null) { + cache.delete(path) + + return { value: null, mtimeMs: null } + } + + const hit = cache.get(path) + + if (hit !== undefined && hit.mtimeMs === mtimeMs) { + return { value: hit.value as T, mtimeMs } + } + + const text = await readOrNull(fs, path) + + if (text === null) { + return { value: null, mtimeMs } + } + + const value = parse(text) + + cache.set(path, { mtimeMs, value }) + + return { value, mtimeMs } +} + +/** The systems the workspace holds: each directory under `analysis/` and under the legacy tree. */ +export async function systemsOf(fs: ReaderFs, legacyDir = 'legacy'): Promise<string[]> { + const names = new Set<string>() + + for (const root of ['analysis', legacyDir]) { + for (const name of await dirNamesOf(fs, root)) { + // The names the plugin's workflows accept: anything else could not be run on, and is not put in a command or a note. + if (isSystemName(name)) { + names.add(name) + } + } + } + + return [...names].sort() +} + +const slugOf = (target: string) => + target + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + +/** How a transformed directory's name is matched to a unit's names. */ +const keyOf = (name: string) => name.toLowerCase().replace(/::/g, '-').replace(/[^a-z0-9]+/g, '-') + +function approvedPhases(brief: Brief): Phase[] { + if (!brief.approval.isSigned) { + return [] + } + + const ordered = [...brief.phases].sort((a, b) => a.number - b.number) + + return brief.approval.covers === 'full' ? ordered : ordered.slice(0, 1) +} + +/** Whether one unit of work is finished, by what the track counts as finished. */ +export function isUnitDone( + snapshot: Pick<Snapshot, 'track' | 'uplift'>, + module: Pick<ModernizedModule, 'state' | 'hasNotes'>, +): boolean { + if (snapshot.track !== 'uplift') { + return isDone(module) + } + + // An uplift is proven by reproducing the baseline; with no per-module baseline, passing is all there is to compare. + const baseline = snapshot.uplift?.baseline + const hasRows = (baseline?.rows.size ?? 0) > 0 || (baseline?.headline ?? null) !== null + + return module.state === 'reviewed' || (module.state === 'tests-green' && !hasRows) +} + +/** The command that opens the whole process: the prefix without its verb dash (`/code-modernization:modernize`). */ +export const frontDoorOf = (prefix: string): string => prefix.replace(/-$/, '') + +/** + * The verify step a built module calls for: none yet, changed since it was verified, or failed. A module that is + * proven, or partly proven (which a person decides on), calls for nothing; nor does one with no notes and no verdict. + */ +function verifyStepOf(prefix: string, system: string, module: Pick<ModernizedModule, 'dir'>, proof: ProofState | undefined): NextStep | null { + if (proof === undefined || proof.state === 'proven' || proof.state === 'partly') { + return null + } + + const name = plain(module.dir, 60) + const text = `${prefix}verify ${system}${isToken(module.dir) ? ` ${module.dir}` : ''}` + + if (proof.state === 'none') { + return { text, isByHand: false, reason: `${name} is built but not yet checked against the old code` } + } + + if (proof.state === 'changed') { + return { text, isByHand: false, reason: `${name} changed after it was verified: check it again` } + } + + return { text, isByHand: false, reason: `${name} is NOT PROVEN${proof.reason === '' ? '' : `: ${proof.reason}`}. Fix that, then check again` } +} + +function nextOfTransform( + prefix: string, + system: string, + stages: Stage[], + brief: Brief | null, + byNode: Map<string, ModernizedModule>, + hasAnalysis: boolean, + proofs: ReadonlyMap<string, ProofState>, +): NextStep | null { + const done = (key: StageKey) => stages.find(stage => stage.key === key)?.isDone === true + const cmd = (name: string, rest = '') => `${prefix}${name} ${system}${rest === '' ? '' : ` ${rest}`}` + + // Nothing has been written for this system yet: the front door asks what the person wants, once, and gives the first step. + if (!hasAnalysis) { + return { text: `${frontDoorOf(prefix)} ${system}`, isByHand: false, reason: 'start here: it asks what you want done, then gives you the first step' } + } + + if (!done('preflight') && !done('assess')) { + return { text: cmd('preflight'), isByHand: false, reason: 'first, check the environment and that all the code is there' } + } + + if (!done('assess')) { + return { text: cmd('assess'), isByHand: false, reason: 'what you have: size, complexity, risks and the recommended approach' } + } + + if (!done('map')) { + return { text: cmd('map'), isByHand: false, reason: 'how the parts of the code connect: calls, data and business flows' } + } + + if (!done('rules')) { + return { text: cmd('extract-rules'), isByHand: false, reason: 'what the code does, written as rules a business person can check' } + } + + if (!done('brief') || brief === null) { + return { text: cmd('brief'), isByHand: false, reason: 'the phased plan you approve before anything is built' } + } + + if (!brief.approval.isSigned) { + return { + text: 'approve the brief', + isByHand: true, + action: 'sign', + reason: 'nothing is built until a person approves the plan: sign it here, or tell Claude which phases you approve', + } + } + + const target = brief.target !== undefined ? slugOf(brief.target) : '' + + for (const phase of approvedPhases(brief)) { + // In the order the phase names them: a module that is built and not yet verified is checked before the next one starts. + for (const id of phase.modules) { + const module = byNode.get(id) + + if (module === undefined || !isDone(module)) { + // A module id is text from the map: it goes in a command only when it is one plain token. + const isNamed = isToken(id) + + return { + text: cmd('transform', `${isNamed ? id : '<module>'}${target === '' ? '' : ` ${target}`}`), + isByHand: !isNamed, + reason: `Phase ${phase.number} names ${plain(id, 60)} and it is not reviewed yet`, + } + } + + const step = verifyStepOf(prefix, system, module, proofs.get(module.dir.toLowerCase())) + + if (step !== null) { + return step + } + } + + if (phase.modules.length === 0) { + return { + text: `pick Phase ${phase.number}'s first module and run ${prefix}transform ${system} <module>${target === '' ? '' : ` ${target}`}`, + isByHand: true, + reason: `Phase ${phase.number} names no module the map knows`, + } + } + } + + return { + text: cmd('status'), + isByHand: false, + reason: + brief.approval.covers === 'full' + ? 'every approved phase is built and checked' + : 'Phase 1 is built and checked; its retrospective revises the brief before Phase 2 is approved', + } +} + +/** The uplift command takes its two versions from what the person said at the start (`INTENT.md`, the brief), and asks when neither has them. */ +function nextOfUplift(prefix: string, system: string, stages: Stage[], proofs: ReadonlyMap<string, ProofState>): NextStep { + const done = (key: StageKey) => stages.find(stage => stage.key === key)?.isDone === true + const command = `${prefix}uplift ${system}` + + if (!done('deltas')) { + return { text: command, isByHand: false, reason: 'first it lists what the newer version breaks in this code' } + } + + if (!done('baseline')) { + return { text: command, isByHand: false, reason: 'next it records what the tests do today, to compare against' } + } + + if (!done('pilot')) { + return { text: command, isByHand: false, reason: 'next it migrates one module first and writes down what it learned' } + } + + if (!done('compare')) { + return { text: command, isByHand: false, reason: 'next it migrates the rest in batches, each compared with the baseline' } + } + + // The whole upgraded copy is one thing to prove, under the name of its working copy. + const step = verifyStepOf(prefix, system, { dir: `${system}-uplifted` }, proofs.get(`${system}-uplifted`)) + + if (step !== null) { + return { ...step, text: `${prefix}verify ${system}` } + } + + return { text: `${prefix}status ${system}`, isByHand: false, reason: 'the upgrade is built and checked; status says what is stale' } +} + +function nextOfReimagine(prefix: string, system: string, stages: Stage[], modules: readonly ModernizedModule[], proofs: ReadonlyMap<string, ProofState>): NextStep { + const done = (key: StageKey) => stages.find(stage => stage.key === key)?.isDone === true + const command = `${prefix}reimagine ${system}` + + if (!done('spec')) { + return { text: command, isByHand: false, reason: 'first it writes down what the old system does, from its code' } + } + + if (!done('design')) { + return { text: command, isByHand: false, reason: 'next it designs the new architecture and checks it against that spec' } + } + + if (!done('scaffold')) { + return { text: command, isByHand: false, reason: 'next it builds each service of the approved design, with tests' } + } + + for (const module of modules) { + const step = verifyStepOf(prefix, system, module, proofs.get(module.dir.toLowerCase())) + + if (step !== null) { + return step + } + } + + return { text: `${prefix}status ${system}`, isByHand: false, reason: 'the services are built and checked; status says what is stale' } +} + +export type ReadOptions = { + /** The system to read; the first one found when absent or unknown. */ + system?: string + /** What a command is written with, up to its verb (`/code-modernization:modernize-`). */ + commandPrefix: string + nowMs: number + /** The read-only tree holding the systems, relative to the workspace; `legacy` when absent. */ + legacyDir?: string + /** Which way to follow the system; `auto` when absent: whichever one's artifacts are newest. */ + track?: TrackKey | 'auto' + /** Units of an uplift's working copy that were written during this session: they count as changed even when the file kept its size. */ + writtenUnits?: ReadonlySet<string> + /** How long a listing of the working copy is reused, in milliseconds; 15 seconds when absent. */ + changedTtlMs?: number + /** Test totals read off commands this session, by module path, for modules that left no report file. */ + observed?: ReadonlyMap<string, TestTotals> +} + +const newest = (...times: (number | null)[]): number | null => { + const known = times.filter((time): time is number => time !== null) + + return known.length === 0 ? null : Math.max(...known) +} + +/** The files of an uplift's working copy and their sizes, reused for a short while so an idle pane does not walk the tree again and again. */ +async function workingSizesOf( + fs: ReaderFs, + cache: ReaderCache, + root: string, + nowMs: number, + ttlMs: number, +): Promise<Map<string, number> | null> { + const key = `changed:${root}` + const hit = cache.get(key) + + if (hit !== undefined && nowMs - hit.mtimeMs < ttlMs) { + return hit.value as Map<string, number> | null + } + + const value = await sourceSizes(fs, root) + + cache.set(key, { mtimeMs: nowMs, value }) + + return value +} + +/** The estate to draw: the map's modules, or what the legacy tree holds. Read once per system, then kept. */ +async function estateOf( + fs: ReaderFs, + cache: ReaderCache, + track: TrackKey, + topology: Topology | null, + legacyRoot: string, + system: string, +): Promise<EstateModel | null> { + if (topology !== null && track !== 'uplift') { + return estateOfTopology(topology, 0) + } + + const key = `estate:${legacyRoot}` + const at = (await mtimeOrNull(fs, legacyRoot)) ?? 0 + const hit = cache.get(key) + + if (hit !== undefined && hit.mtimeMs === at) { + return hit.value as EstateModel | null + } + + const found = await discoverEstate(fs, legacyRoot, system) + + cache.set(key, { mtimeMs: at, value: found }) + + return found ?? (topology !== null ? estateOfTopology(topology, 0) : null) +} + +/** Reads one system's state. Null when the workspace holds no system. */ +export async function readSnapshot( + fs: ReaderFs, + cache: ReaderCache, + options: ReadOptions, +): Promise<Snapshot | null> { + const legacyDir = options.legacyDir ?? 'legacy' + const systems = await systemsOf(fs, legacyDir) + + const system = + options.system !== undefined && systems.includes(options.system) + ? options.system + : systems[0] + + if (system === undefined) { + return null + } + + const dir = join('analysis', system) + const legacyRoot = join(legacyDir, system) + const transformRoot = join('modernized', system) + const upliftRoot = join('modernized', `${system}-uplifted`) + const reimagineRoot = join('modernized', `${system}-reimagined`) + const observed = options.observed ?? new Map<string, TestTotals>() + + const [topo, rules, catalog, baseline, verificationRead, reviewsRaw, findingsHead, marks] = await Promise.all([ + cached(fs, cache, join(dir, STAGE_FILES.map), parseTopology), + cached(fs, cache, join(dir, STAGE_FILES.rules), parseRules), + cached(fs, cache, join(dir, STAGE_FILES.deltas), parseCatalog), + cached(fs, cache, join(dir, STAGE_FILES.baseline), parseBaseline), + cached(fs, cache, join(dir, STAGE_FILES.verify), parseVerification), + readOrNull(fs, join(dir, REVIEWS_FILE)), + readOrNull(fs, join(dir, STAGE_FILES.harden)), + Promise.all( + [ + mtimeOrNull(fs, dir), + mtimeOrNull(fs, join(dir, STAGE_FILES.preflight)), + mtimeOrNull(fs, join(dir, STAGE_FILES.assess)), + mtimeOrNull(fs, join(dir, STAGE_FILES.pilot)), + mtimeOrNull(fs, join(upliftRoot, 'UPLIFT_NOTES.md')), + mtimeOrNull(fs, upliftRoot), + mtimeOrNull(fs, join(dir, STAGE_FILES.spec)), + mtimeOrNull(fs, join(dir, STAGE_FILES.design)), + mtimeOrNull(fs, reimagineRoot), + mtimeOrNull(fs, transformRoot), + ], + ), + ]) + + const [analysisAt, preflightAt, assessAt, playbookAt, upliftNotesAt, upliftCopyAt, specAt, designAt, reimagineDirAt, transformDirAt] = marks + + const topology = topo.value + const moduleIds = topology?.modules.map(node => node.id) ?? [] + + const brief = await cached(fs, cache, join(dir, STAGE_FILES.brief), text => + parseBrief(text, moduleIds), + ) + + // A brief parsed before the map existed named no modules: parse it again. + if ( + brief.value !== null && + moduleIds.length > 0 && + brief.value.phases.length > 0 && + brief.value.phases.every(phase => phase.modules.length === 0) + ) { + const text = await readOrNull(fs, join(dir, STAGE_FILES.brief)) + + if (text !== null) { + brief.value = parseBrief(text, moduleIds) + cache.set(join(dir, STAGE_FILES.brief), { mtimeMs: brief.mtimeMs ?? 0, value: brief.value }) + } + } + + const reviews = parseLedger(reviewsRaw) + + const isScan = findingsHead !== null && /generated-by:\s*modernize-harden/.test(findingsHead.split('\n')[0] ?? '') + + const transformed = await readModernizedIn(fs, transformRoot, observed) + + const track = pickTrack(options.track ?? 'auto', { + transformAt: newest(transformDirAt, ...transformed.map(module => module.mtimeMs)), + // A baseline says nothing of the track: a rewrite records one too, for the legacy behavior it must keep. + upliftAt: newest(catalog.mtimeMs, playbookAt, upliftNotesAt, upliftCopyAt), + reimagineAt: newest(specAt, designAt, reimagineDirAt), + }) + + const estate = await estateOf(fs, cache, track, topology, legacyRoot, system) + + let modules: ModernizedModule[] = [] + const byNode = new Map<string, ModernizedModule>() + const extras: ModernizedModule[] = [] + let uplift: UpliftFacts | null = null + + if (track === 'uplift') { + const hasCopy = upliftCopyAt !== null + const legacySizes = estate?.isPartial === false ? estate.fileSizes : undefined + const workingSizes = hasCopy && legacySizes !== undefined ? await workingSizesOf(fs, cache, upliftRoot, options.nowMs, options.changedTtlMs ?? 15_000) : null + const changedPaths = legacySizes !== undefined && workingSizes !== null ? changedPathsOf(legacySizes, workingSizes) : null + const changed = estate !== null && changedPaths !== null ? changedUnitsOf(estate, changedPaths) : new Set<string>() + + for (const id of options.writtenUnits ?? []) { + changed.add(id) + } + + const pairs = + estate !== null && hasCopy + ? await readUpliftUnits(fs, upliftRoot, estate, changed, baseline.value, observed) + : [] + + modules = pairs.map(pair => pair.module) + + for (const pair of pairs) { + byNode.set(pair.unit.id, pair.module) + } + + uplift = { + catalog: catalog.value, + baseline: baseline.value, + hasPlaybook: playbookAt !== null, + hasNotes: upliftNotesAt !== null, + hasCopy, + isChangeKnown: changedPaths !== null, + } + } else { + modules = track === 'reimagine' ? await readModernizedIn(fs, reimagineRoot, observed) : transformed + + if (track === 'transform') { + const byKey = new Map<string, EstateUnit>() + + for (const unit of estate?.units ?? []) { + for (const key of keysOfUnit(unit)) { + byKey.set(keyOf(key), unit) + } + } + + for (const module of modules) { + const unit = byKey.get(keyOf(module.dir)) + + if (unit !== undefined) { + byNode.set(unit.id, module) + } else { + extras.push(module) + } + } + } else { + extras.push(...modules) + } + } + + const snapshotFacts = { track, uplift } + const done = modules.filter(module => isUnitDone(snapshotFacts, module)) + + // Where each built module stands with the proof. An uplift is one piece: the whole working copy has the one verdict. + const verification = verificationRead.value + const proofs = new Map<string, ProofState>() + + if (track === 'uplift') { + const name = `${system}-uplifted` + const proof = proofOfModule(verification, 'uplift', name, upliftNotesAt ?? 0, upliftNotesAt !== null) + + if (proof !== null) { + proofs.set(name.toLowerCase(), proof) + } + } else { + for (const module of modules) { + const proof = proofOfModule(verification, track, module.dir, module.mtimeMs, module.hasNotes) + + if (proof !== null) { + proofs.set(module.dir.toLowerCase(), proof) + } + } + } + + const proofList = [...proofs.values()] + const isProofDone = (proof: ProofState) => proof.state === 'proven' || proof.state === 'partly' + + const verifyStage: Stage[] = + proofList.length === 0 && track !== 'uplift' && modules.length === 0 + ? [] + : [ + { + key: 'verify', + label: STAGE_LABELS.verify, + isDone: proofList.length > 0 && proofList.every(isProofDone), + mtimeMs: null, + ...(proofList.length > 0 && { + detail: + track === 'uplift' + ? (proofList[0]?.verdict ?? 'not verified yet').toLowerCase() + : `${proofList.filter(proof => proof.state === 'proven').length} of ${proofList.length} proven`, + }), + }, + ] + + const green = modules.filter( + module => module.state === 'tests-green' || module.state === 'reviewed' || isUnitDone(snapshotFacts, module), + ) + + const passing = green.reduce( + (sum, module) => sum + Math.max(0, (module.tests?.tests ?? 0) - (module.tests?.failures ?? 0) - (module.tests?.errors ?? 0)), + 0, + ) + + const withFindings: Stage[] = + findingsHead !== null || track === 'transform' + ? [ + { + key: 'harden', + label: STAGE_LABELS.harden, + // Any findings file counts. A harden marker on its first line says the command wrote it, and older or + // newer versions of the command may or may not stamp one, so its absence is not a reason to doubt the file. + isDone: findingsHead !== null, + mtimeMs: null, + }, + ] + : [] + + const stages: Stage[] = [ + ...(track === 'transform' + ? [ + { key: 'preflight' as const, label: STAGE_LABELS.preflight, isDone: preflightAt !== null, mtimeMs: preflightAt }, + { key: 'assess' as const, label: STAGE_LABELS.assess, isDone: assessAt !== null, mtimeMs: assessAt }, + { + key: 'map' as const, + label: STAGE_LABELS.map, + isDone: topology !== null, + mtimeMs: topo.mtimeMs, + ...(topology !== null && { detail: `${topology.modules.length} modules` }), + }, + { + key: 'rules' as const, + label: STAGE_LABELS.rules, + isDone: rules.value !== null && rules.value.rules.length > 0, + mtimeMs: rules.mtimeMs, + ...(rules.value !== null && { detail: `${rules.value.rules.length} rules` }), + }, + { + key: 'brief' as const, + label: STAGE_LABELS.brief, + isDone: brief.value !== null, + mtimeMs: brief.mtimeMs, + ...(brief.value !== null && { + detail: brief.value.approval.isSigned ? 'approved' : 'unapproved', + }), + }, + { + // Transform writes under modernized/, not analysis/: this stage is done once a module's tests are green. + key: 'transform' as const, + label: STAGE_LABELS.transform, + isDone: green.length > 0, + mtimeMs: null, + ...(modules.length > 0 && { + detail: green.length === 0 ? 'in progress' : passing > 0 ? `${passing} green` : `${green.length} module${green.length === 1 ? '' : 's'}`, + }), + }, + ...verifyStage, + ] + : track === 'uplift' + ? [ + { key: 'preflight' as const, label: STAGE_LABELS.preflight, isDone: preflightAt !== null, mtimeMs: preflightAt }, + { + key: 'deltas' as const, + label: STAGE_LABELS.deltas, + isDone: catalog.mtimeMs !== null, + mtimeMs: catalog.mtimeMs, + ...(catalog.value !== null && { detail: `${catalog.value.count} deltas` }), + }, + { + key: 'baseline' as const, + label: STAGE_LABELS.baseline, + isDone: baseline.mtimeMs !== null, + mtimeMs: baseline.mtimeMs, + ...(baseline.value !== null && { + detail: baseline.value.isTargetOnly + ? 'target-only' + : baseline.value.results !== null + ? `${baseline.value.results.toLocaleString('en-US')} tests` + : undefined, + }), + }, + { key: 'pilot' as const, label: STAGE_LABELS.pilot, isDone: playbookAt !== null, mtimeMs: playbookAt }, + { + key: 'migrate' as const, + label: STAGE_LABELS.migrate, + isDone: modules.length > 0 && playbookAt !== null, + mtimeMs: null, + ...(modules.length > 0 && { detail: `${modules.length} module${modules.length === 1 ? '' : 's'}` }), + }, + { + key: 'compare' as const, + label: STAGE_LABELS.compare, + isDone: upliftNotesAt !== null, + mtimeMs: upliftNotesAt, + ...(modules.length > 0 && { detail: `${done.length}/${modules.length} match` }), + }, + ...verifyStage, + ] + : [ + { key: 'preflight' as const, label: STAGE_LABELS.preflight, isDone: preflightAt !== null, mtimeMs: preflightAt }, + { key: 'spec' as const, label: STAGE_LABELS.spec, isDone: specAt !== null, mtimeMs: specAt }, + { key: 'design' as const, label: STAGE_LABELS.design, isDone: designAt !== null, mtimeMs: designAt }, + { + key: 'scaffold' as const, + label: STAGE_LABELS.scaffold, + isDone: modules.length > 0, + mtimeMs: null, + ...(modules.length > 0 && { detail: `${modules.length} service${modules.length === 1 ? '' : 's'}` }), + }, + { + key: 'tests' as const, + label: STAGE_LABELS.tests, + isDone: modules.length > 0 && green.length === modules.length, + mtimeMs: null, + ...(modules.length > 0 && { detail: `${green.length}/${modules.length} green` }), + }, + ...verifyStage, + ]), + ...withFindings, + ] + + const loc = (estate?.units ?? []).reduce((sum, unit) => sum + Math.max(1, unit.size), 0) + + const locDone = (estate?.units ?? []).reduce((sum, unit) => { + const module = byNode.get(unit.id) + + return module !== undefined && isUnitDone(snapshotFacts, module) ? sum + Math.max(1, unit.size) : sum + }, 0) + + const attention: string[] = [] + + for (const module of modules) { + const row = track === 'uplift' ? baselineRowOf(baseline.value, module.dir) : null + + if ((module.state === 'tests-red' || module.state === 'tests-failing') && module.tests !== null) { + const bad = module.tests.failures + module.tests.errors + + attention.push( + track === 'uplift' + ? row === null + ? `${module.dir}: ${bad} tests failing, and the baseline has no row for it to compare with` + : `${module.dir}: ${bad} failing, the baseline had ${row.fail + row.error}` + : `${module.dir}: ${bad} of ${module.tests.tests} tests red`, + ) + } + + if (module.tests !== null && module.tests.reports > 0 && module.tests.tests === 0) { + attention.push(`${module.dir}: test reports exist but 0 cases executed`) + } + + if (track === 'transform' && module.state === 'tests-green' && module.hasNotes && !module.hasReviewSection) { + attention.push(`${module.dir}: tests green; the notes do not show the architecture review`) + } + } + + // A check that failed or could not be completed says why, in the check's own first reason. PARTLY PROVEN lets the work go on: a person decides. + for (const [name, proof] of proofs) { + if (proof.state === 'not' || proof.state === 'partly') { + attention.push(`${track === 'uplift' ? 'the upgrade' : (modules.find(module => module.dir.toLowerCase() === name)?.dir ?? name)}: ${proof.verdict ?? ''}${proof.reason === '' ? '' : `: ${proof.reason}`}`) + } + } + + for (const extra of extras) { + if (track === 'transform' && extra.hasMain && (!extra.hasNotes || !extra.hasTests)) { + attention.push( + `${extra.dir}: shared code without ${[!extra.hasTests && 'tests', !extra.hasNotes && 'notes'].filter(Boolean).join(' or ')}; an unfinished transform`, + ) + } + + if (track === 'reimagine' && extra.hasMain && !extra.hasTests) { + attention.push(`${extra.dir}: scaffolded with no acceptance tests`) + } + } + + if (track === 'uplift' && uplift !== null) { + if (uplift.baseline?.isTargetOnly === true) { + attention.push('the baseline is target-only: the source runtime could not run, so nothing is compared') + } + + if (modules.length > 0 && !uplift.hasPlaybook) { + attention.push(`${modules.length} module${modules.length === 1 ? '' : 's'} touched before a playbook was written: pilot one first`) + } + + if (modules.length > 0 && uplift.baseline === null) { + attention.push('modules touched before a baseline was recorded: nothing to compare them with') + } + } + + if ( + brief.mtimeMs !== null && + rules.mtimeMs !== null && + brief.mtimeMs < rules.mtimeMs + ) { + attention.push('the brief is older than the rule set it was planned from') + } + + if (rules.value !== null) { + const pending = rules.value.rules.filter( + rule => rule.priority === 'P0' && needsReview(rule) && reviews[rule.id] === undefined, + ).length + + const wrong = Object.values(reviews).filter(entry => entry.verdict === 'wrong').length + + // A P0 rule a reviewer sent to discussion stops the build commands until it is settled. + const discussing = rules.value.rules.filter(rule => rule.priority === 'P0' && reviews[rule.id]?.verdict === 'discuss').length + + if (discussing > 0) { + attention.push(`${discussing} high-priority rule${discussing === 1 ? '' : 's'} under discussion: the build waits`) + } + + if (wrong > 0) { + attention.push(`${wrong} rule${wrong === 1 ? '' : 's'} marked wrong by a reviewer: no test is built on ${wrong === 1 ? 'it' : 'them'}`) + } + + if (pending > 0) { + attention.push(`${pending} high-priority rule${pending === 1 ? '' : 's'} still need${pending === 1 ? 's' : ''} a person's review`) + } + } + + const next: NextStep | null = + track === 'uplift' + ? nextOfUplift(options.commandPrefix, system, stages, proofs) + : track === 'reimagine' + ? nextOfReimagine(options.commandPrefix, system, stages, modules, proofs) + : nextOfTransform(options.commandPrefix, system, stages, brief.value, byNode, analysisAt !== null, proofs) + + return { + system, + systems, + track, + hasAnalysis: analysisAt !== null, + stages, + estate, + topology, + rules: rules.value, + brief: brief.value, + modules, + byNode, + extras, + uplift, + findings: { exists: findingsHead !== null, isScan }, + verification, + proofs, + reviews, + percent: track === 'reimagine' || loc === 0 ? null : locDone / loc, + totals: { + modules: estate?.units.length ?? 0, + done: [...byNode.values()].filter(module => isUnitDone(snapshotFacts, module)).length, + loc, + locDone, + }, + attention, + next, + readAtMs: options.nowMs, + } +} + +/** One line for the status bar and the prompt's hidden context. */ +export function oneLineOf(snapshot: Snapshot): string { + // A rewrite's five analysis steps are what "analysis 5/5" counts: its build, its proof and its scan are not among them. + const countable = snapshot.stages.filter(stage => stage.key !== 'harden' && stage.key !== 'transform' && !(snapshot.track === 'transform' && stage.key === 'verify')) + const done = countable.filter(stage => stage.isDone).length + + const parts = + snapshot.track === 'transform' + ? [`${snapshot.system}: analysis ${done}/5`] + : [`${snapshot.system}: ${TRACK_LABELS[snapshot.track]} ${done}/${countable.length}`] + + if (snapshot.brief !== null) { + parts.push(snapshot.brief.approval.isSigned ? 'brief approved' : 'brief waiting for approval') + } + + if (snapshot.totals.modules > 0 && snapshot.modules.length > 0) { + parts.push( + snapshot.track === 'reimagine' + ? `${snapshot.modules.length} service${snapshot.modules.length === 1 ? '' : 's'} built` + : `${snapshot.totals.done}/${snapshot.totals.modules} modules ${snapshot.track === 'uplift' ? 'match the baseline' : 'reviewed'}`, + ) + } + + const proven = [...snapshot.proofs.values()].filter(proof => proof.state === 'proven').length + + if (proven > 0) { + parts.push(snapshot.track === 'uplift' ? 'proven to behave the same' : `${proven} proven`) + } + + if (snapshot.attention.length > 0) { + parts.push(`${snapshot.attention.length} need${snapshot.attention.length === 1 ? 's' : ''} attention`) + } + + return parts.join(' · ') +} diff --git a/plugins/code-modernization/hooks/reader/rules.ts b/plugins/code-modernization/hooks/reader/rules.ts new file mode 100644 index 00000000..080edab8 --- /dev/null +++ b/plugins/code-modernization/hooks/reader/rules.ts @@ -0,0 +1,438 @@ +import { baseName } from '../paths' +import { linesOf } from '../text' + +/** + * `analysis/<system>/BUSINESS_RULES.md`, read into rule cards. + * + * The file is written by an agent, and the agent's layout varies between models and + * runs. Three shapes are read: a card per rule under a `### <ID> · <title>` heading with + * a header line (`P0 · Calculation · confidence High · file:line`), the same heading with + * labelled fields (`**Category:**`, `**Priority:**`, `**Plain English:**`, a fenced + * Given/When/Then block, `**Edge cases handled:**`, `**Suspected defect:**`), and table + * rows for the lower priorities. Anything else is skipped, never thrown on. + */ + +export type Citation = { + /** The path as written (`legacy/x/app/cbl/A.cbl`, `A.cbl`, `app/cpy/B.cpy`). */ + path: string + /** Lower-cased base name, the key the x-ray matches a read file by. */ + base: string + from: number + to: number +} + +export type Rule = { + /** `P0-052`, or `R-<hash>` of the title for a table row that carries no id. */ + id: string + title: string + /** `P0`, `P1`, `P2`, or `''` when the file does not say. */ + priority: string + category?: string + confidence?: string + /** The rule's one-paragraph statement. */ + statement: string + given?: string + when?: string + then?: string + edgeCases: string[] + defect?: string + sme?: string + /** Every `file:line` the card names; the first is the rule's own source. */ + citations: Citation[] + /** The domain heading (`## D6 Interest & Fees`) the rule sits under. */ + domain?: string +} + +export type RuleSet = { + rules: Rule[] + byId: Map<string, Rule> + /** Lower-cased base name of a cited file, to the rules citing it. */ + byFileBase: Map<string, Rule[]> +} + +const CITATION = + /([A-Za-z0-9_][A-Za-z0-9_./-]*\.[A-Za-z][A-Za-z0-9]{0,7}):(\d+)(?:\s*[-–]\s*(\d+))?/g + +/** `P0-052`, `RULE-001`, `BR-D7-09`: letters, optional segments (a domain), then the number. */ +const ID = '[A-Z][A-Z0-9]*(?:-[A-Z][A-Z0-9]*)*-\\d+[a-z]?' + +const CARD_HEADING = new RegExp(`^#{3,4}\\s+(${ID})\\s*[·:\\-–—]\\s*(.+?)\\s*$`) + +/** Sections that describe or summarise the rules rather than hold them: a legend, a panel summary, appendices, provenance. */ +const SKIPPED_SECTION = + /\b(?:index|appendix|open questions|rejected|refuted|how to read|p0 confirmation|provenance)\b/i + +const DOMAIN_HEADING = /^##\s+(.+?)\s*$/ + +const clean = (text: string) => + text + .replace(/\*\*/g, '') + .replace(/<br\s*\/?>(\s*)/gi, ' ') + .replace(/\s+/g, ' ') + .trim() + +/** Every `file:line[-line]` in `text`, in order, without duplicates. */ +export function citationsIn(text: string): Citation[] { + const seen = new Set<string>() + const out: Citation[] = [] + + for (const match of text.matchAll(CITATION)) { + const path = match[1] ?? '' + const from = Number(match[2]) + const to = match[3] !== undefined ? Number(match[3]) : from + const key = `${path}:${from}-${to}` + + if (path === '' || !Number.isFinite(from) || seen.has(key)) { + continue + } + + seen.add(key) + + out.push({ + path, + base: baseName(path).toLowerCase(), + from, + to: Math.max(from, to), + }) + } + + return out +} + +const priorityOf = (id: string, body: string): string => { + const fromId = /^(P\d)\b/.exec(id)?.[1] + + if (fromId !== undefined) { + return fromId + } + + const fromLabel = /\bPriority:?\s*\**\s*(P\d)\b/i.exec(body)?.[1]?.toUpperCase() + + if (fromLabel !== undefined) { + return fromLabel + } + + // `P0` · Calculation · confidence High · `app/cbl/A.cbl:10-20` + const header = linesOf(body).find(line => line.trim() !== '') ?? '' + + return /^\s*`?\**(P\d)\**`?\s*(?:·|$)/.exec(header)?.[1] ?? '' +} + +const gwtOf = (body: string, word: string): string | undefined => { + const match = new RegExp( + `\\*\\*${word}\\*\\*\\s*([\\s\\S]*?)(?=\\n[ \\t]*(?:[-*][ \\t]+)?>?\\s*\\*\\*(?:Given|When|Then)\\*\\*|\\n\\s*\\n|$)`, + 'i', + ).exec(body) + + const text = + match?.[1] !== undefined + ? clean(match[1].replace(/\n>\s?/g, ' ')).replace(/^[—–:-]\s*/, '') + : '' + + return text === '' ? undefined : text +} + +const markedOf = (body: string, marker: RegExp): string | undefined => { + for (const line of linesOf(body)) { + if (marker.test(line)) { + const text = clean(line.replace(/^>\s*/, '').replace(marker, '')) + .replace(/^[—–-]\s*/, '') + .trim() + + return text === '' ? undefined : text + } + } + + return undefined +} + +/** The text after `**Name:**` (or `**Name**:`) on its own line, for the labelled-fields layout. */ +const labelledOf = (body: string, name: string): string | undefined => { + const match = new RegExp(`^[ \\t]*\\*\\*${name}:?\\*\\*:?[ \\t]*(.*)$`, 'im').exec(body) + const text = match?.[1] !== undefined ? clean(match[1]) : '' + + return text === '' ? undefined : text +} + +/** + * Given/When/Then written as lines (`Given …`, `When …`, `Then …`, `And …`), usually inside a + * fenced block under `**Specification:**`. `And` lines continue the clause before them. + */ +const specOf = (body: string): { given?: string; when?: string; then?: string } => { + const block = /\*\*Specification:?\*\*:?\s*([\s\S]*?)(?=\n[ \t]*\*\*[A-Za-z][^*\n]*\*\*|$)/i.exec(body)?.[1] ?? '' + const clauses: Record<'given' | 'when' | 'then', string[]> = { given: [], when: [], then: [] } + let current: 'given' | 'when' | 'then' | undefined + + for (const raw of linesOf(block)) { + const line = raw.trim().replace(/^```\w*$/, '').replace(/^[-*]\s+/, '') + const match = /^(Given|When|Then|And)\s+(.*)$/.exec(line) + + if (match === null) { + continue + } + + const word = match[1] ?? '' + const rest = clean(match[2] ?? '') + + if (word === 'And') { + if (current !== undefined) { + clauses[current].push(rest) + } + + continue + } + + current = word.toLowerCase() as 'given' | 'when' | 'then' + clauses[current].push(rest) + } + + const join = (parts: string[]) => (parts.length === 0 ? undefined : parts.join(' · ')) + + return { given: join(clauses.given), when: join(clauses.when), then: join(clauses.then) } +} + +function cardOf( + id: string, + title: string, + body: string, + domain: string | undefined, +): Rule { + const lines = linesOf(body) + const header = lines.find(line => line.trim() !== '') ?? '' + const headerParts = header.split('·').map(part => clean(part)) + + const category = + headerParts + .slice(1) + .find(part => part !== '' && !/confidence/i.test(part) && !/`/.test(part)) ?? labelledOf(body, 'Category') + + const confidence = + /confidence\s*:?\s*\**\s*(High|Medium|Low)\b/i.exec(body)?.[1] ?? undefined + + const statement = + labelledOf(body, 'Plain English') ?? + lines + .slice(lines.indexOf(header) + 1) + .map(line => line.trim()) + .find( + line => + line !== '' && + !line.startsWith('>') && + !line.startsWith('|') && + !line.startsWith('**') && + !line.startsWith('-') && + !line.startsWith('#') && + !line.startsWith('`'), + ) ?? '' + + const edgeStart = lines.findIndex(line => /^\*\*Edge cases?[^*]*\*\*/i.test(line.trim())) + + const edgeCases: string[] = [] + + if (edgeStart >= 0) { + for (const line of lines.slice(edgeStart + 1)) { + const trimmed = line.trim() + + if (trimmed.startsWith('- ')) { + edgeCases.push(clean(trimmed.slice(2))) + } else if (trimmed !== '' && edgeCases.length > 0) { + break + } + } + } + + const spec = specOf(body) + const given = gwtOf(body, 'Given') ?? spec.given + const when = gwtOf(body, 'When') ?? spec.when + const then = gwtOf(body, 'Then') ?? spec.then + const defect = markedOf(body, /(?:⚠️?\s*\**\s*Suspected defect\**|^\s*\*\*Suspected defect:?\*\*:?)/i) + + // `❓ **SME question** — …`, or `**Confidence:** Medium — SME question: …` on the confidence line. + const smeInline = /\bSME questions?\s*:\s*(.+)$/im.exec(body)?.[1] + + const sme = + markedOf(body, /❓\s*\**\s*SME(?:\s+questions?)?\**/i) ?? + (smeInline !== undefined && clean(smeInline) !== '' ? clean(smeInline) : undefined) + + return { + id, + title: clean(title), + priority: priorityOf(id, body), + ...(category !== undefined && { category }), + ...(confidence !== undefined && { + confidence: confidence[0]?.toUpperCase() + confidence.slice(1).toLowerCase(), + }), + statement: clean(statement), + ...(given !== undefined && { given }), + ...(when !== undefined && { when }), + ...(then !== undefined && { then }), + edgeCases, + ...(defect !== undefined && { defect }), + ...(sme !== undefined && { sme }), + citations: citationsIn(body), + ...(domain !== undefined && { domain }), + } +} + +/** A short stable id for a rule that carries none: a hash of its title, so a verdict survives the table being rewritten in another order. */ +export function idOfTitle(title: string): string { + let hash = 2166136261 + + for (const char of title.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim()) { + hash = Math.imul(hash ^ char.charCodeAt(0), 16777619) + } + + return `R-${(hash >>> 0).toString(16).padStart(8, '0').slice(0, 6)}` +} + +function rowOf( + line: string, + domain: string | undefined, +): Rule | null { + const cells = line + .split('|') + .slice(1, -1) + .map(cell => cell.trim()) + + const first = cells[0] ?? '' + const titleMatch = /\*\*(.+?)\*\*/.exec(first) + + if (cells.length < 2 || titleMatch === null || /^-+$/.test(first)) { + return null + } + + const title = clean(titleMatch[1] ?? '') + + if (title === '' || /^rule$/i.test(title)) { + return null + } + + const idInTitle = new RegExp(`^(${ID})\\b`).exec(title)?.[1] + const category = clean(first.replace(titleMatch[0], '').replace(/<br\s*\/?>/gi, ' ')) + const gwt = cells[1] ?? '' + const part = (tag: string) => + clean(new RegExp(`\\*\\*${tag}\\*\\*\\s*(.*?)(?=\\*\\*[GWT]\\*\\*|$)`).exec(gwt)?.[1] ?? '') + + const given = part('G') + const when = part('W') + const then = part('T') + const flags = cells.slice(3).join(' ') + const priority = /\b(P\d)\b/.exec(`${flags} ${first}`)?.[1] ?? '' + + return { + id: idInTitle ?? idOfTitle(title), + title, + priority, + ...(category !== '' && { category }), + statement: '', + ...(given !== '' && { given }), + ...(when !== '' && { when }), + ...(then !== '' && { then }), + edgeCases: [], + ...(/⚠|defect/i.test(flags) && { defect: clean(flags) }), + ...(/❓|SME/i.test(flags) && { sme: clean(flags) }), + citations: citationsIn(line), + ...(domain !== undefined && { domain }), + } +} + +/** Parses BUSINESS_RULES.md. Never throws; an unreadable file is no rules. */ +export function parseRules(text: string): RuleSet { + const lines = linesOf(text) + const rules: Rule[] = [] + let domain: string | undefined + /** Set by a top-level heading: `# P1 rules` gives its rows a priority, an index or appendix is skipped. */ + let sectionPriority = '' + let isTopSkipped = false + let isSectionSkipped = false + + let open: { id: string; title: string; start: number; domain: string | undefined } | null = + null + + const close = (end: number) => { + if (open !== null) { + rules.push( + cardOf(open.id, open.title, lines.slice(open.start, end).join('\n'), open.domain), + ) + + open = null + } + } + + lines.forEach((line, index) => { + const isSkipped = isTopSkipped || isSectionSkipped + const card = isSkipped ? null : CARD_HEADING.exec(line) + + if (card !== null) { + close(index) + open = { id: card[1] ?? '', title: card[2] ?? '', start: index + 1, domain } + + return + } + + const top = /^#\s+(.+?)\s*$/.exec(line) + + if (top !== null) { + close(index) + sectionPriority = /\b(P\d)\b/.exec(top[1] ?? '')?.[1] ?? '' + isTopSkipped = SKIPPED_SECTION.test(top[1] ?? '') + isSectionSkipped = false + domain = undefined + + return + } + + const heading = DOMAIN_HEADING.exec(line) + + if (heading !== null && !line.startsWith('###')) { + close(index) + domain = clean(heading[1] ?? '') + isSectionSkipped = SKIPPED_SECTION.test(heading[1] ?? '') + + return + } + + if (/^#{1,4}\s/.test(line)) { + close(index) + + return + } + + if (open === null && !(isTopSkipped || isSectionSkipped) && line.startsWith('|')) { + const row = rowOf(line, domain) + + if (row !== null) { + rules.push(row.priority === '' ? { ...row, priority: sectionPriority } : row) + } + } + }) + + close(lines.length) + + const byId = new Map<string, Rule>() + const byFileBase = new Map<string, Rule[]>() + + for (const rule of rules) { + if (!byId.has(rule.id)) { + byId.set(rule.id, rule) + } + + for (const base of new Set(rule.citations.map(citation => citation.base))) { + const bucket = byFileBase.get(base) ?? [] + + bucket.push(rule) + byFileBase.set(base, bucket) + } + } + + return { rules, byId, byFileBase } +} + +/** The rules that need a person: a suspected defect, an SME note, or less than High confidence. */ +export function needsReview(rule: Rule): boolean { + return ( + rule.sme !== undefined || + rule.defect !== undefined || + (rule.confidence !== undefined && rule.confidence !== 'High') + ) +} diff --git a/plugins/code-modernization/hooks/reader/topology.ts b/plugins/code-modernization/hooks/reader/topology.ts new file mode 100644 index 00000000..b0f5adb5 --- /dev/null +++ b/plugins/code-modernization/hooks/reader/topology.ts @@ -0,0 +1,243 @@ +import { baseName, norm } from '../paths' + +/** + * `analysis/<system>/topology.json`, read into the few shapes the live pane uses. + * + * The file is written by an agent, so every field is treated as optional and + * of unknown shape: anything that does not fit is dropped, never thrown on. + */ + +export type TopoNode = { + id: string + name: string + /** `module`, `datastore`, `screen`, or whatever the map stage wrote. */ + kind: string + /** Lines of code; 0 when the map gave none. */ + loc: number + /** Source file relative to `legacy/<system>/`, when the node is one file. */ + file?: string + /** The name of the domain (first ancestor of kind `domain`) it sits under. */ + domain?: string + /** The language the map recorded for it, as written (`cobol`, `java`, `php`). */ + language?: string +} + +export type TopoEdge = { source: string; target: string; kind: string } + +export type TopoFlow = { + name: string + persona?: string + steps: { label: string; nodes: string[] }[] +} + +export type Topology = { + system: string + nodes: TopoNode[] + /** Nodes that are code (`kind: module`, or any leaf with a file and loc). */ + modules: TopoNode[] + domains: { name: string; modules: TopoNode[] }[] + edges: TopoEdge[] + entryPoints: Set<string> + deadEnds: Set<string> + flows: TopoFlow[] + byId: Map<string, TopoNode> + /** Lower-cased base name of a node's file, to the nodes that have it. */ + byFileBase: Map<string, TopoNode[]> +} + +const str = (value: unknown): string | undefined => + typeof value === 'string' && value !== '' ? value : undefined + +const isRecord = (value: unknown): value is Record<string, unknown> => + typeof value === 'object' && value !== null && !Array.isArray(value) + +const list = (value: unknown): unknown[] => (Array.isArray(value) ? value : []) + +function walk( + raw: unknown, + domain: string | undefined, + out: TopoNode[], + depth: number, +): void { + if (!isRecord(raw) || depth > 12) { + return + } + + const id = str(raw.id) ?? str(raw.name) + + if (id === undefined) { + return + } + + const kind = str(raw.kind) ?? 'node' + const name = str(raw.name) ?? id + const children = list(raw.children) + const here = kind === 'domain' ? name : domain + const loc = typeof raw.loc === 'number' && raw.loc > 0 ? raw.loc : 0 + const file = str(raw.file) + const language = str(raw.language) + + if (children.length === 0 || kind === 'module') { + out.push({ + id, + name, + kind, + loc, + ...(file !== undefined && { file: norm(file) }), + ...(here !== undefined && { domain: here }), + ...(language !== undefined && { language }), + }) + } + + for (const child of children) { + walk(child, here, out, depth + 1) + } +} + +/** Parses topology.json; null when the text is not a usable topology. */ +export function parseTopology(text: string): Topology | null { + let raw: unknown + + try { + raw = JSON.parse(text) + } catch { + return null + } + + if (!isRecord(raw)) { + return null + } + + const nodes: TopoNode[] = [] + + if (isRecord(raw.root)) { + walk(raw.root, undefined, nodes, 0) + } + + for (const flat of list(raw.nodes)) { + walk(flat, undefined, nodes, 0) + } + + if (nodes.length === 0) { + return null + } + + const byId = new Map<string, TopoNode>() + + for (const node of nodes) { + if (!byId.has(node.id)) { + byId.set(node.id, node) + } + } + + const unique = [...byId.values()] + + const modules = unique.filter( + node => + node.kind === 'module' || + (node.file !== undefined && node.loc > 0 && node.kind !== 'datastore'), + ) + + const byDomain = new Map<string, TopoNode[]>() + + for (const node of modules) { + const key = node.domain ?? 'Other' + const bucket = byDomain.get(key) ?? [] + + bucket.push(node) + byDomain.set(key, bucket) + } + + const byFileBase = new Map<string, TopoNode[]>() + + for (const node of unique) { + if (node.file === undefined) { + continue + } + + const key = baseName(node.file).toLowerCase() + const bucket = byFileBase.get(key) ?? [] + + bucket.push(node) + byFileBase.set(key, bucket) + } + + const edges: TopoEdge[] = [] + + for (const edge of list(raw.edges)) { + if (!isRecord(edge)) { + continue + } + + const source = str(edge.source) ?? str(edge.from) + const target = str(edge.target) ?? str(edge.to) + + if (source !== undefined && target !== undefined) { + edges.push({ source, target, kind: str(edge.kind) ?? 'uses' }) + } + } + + const flows: TopoFlow[] = [] + + for (const flow of list(raw.flows)) { + if (!isRecord(flow)) { + continue + } + + const name = str(flow.name) + + if (name === undefined) { + continue + } + + const persona = str(flow.persona) + + flows.push({ + name, + ...(persona !== undefined && { persona }), + steps: list(flow.steps) + .filter(isRecord) + .map(step => ({ + label: str(step.label) ?? '', + nodes: list(step.nodes).filter( + (node): node is string => typeof node === 'string', + ), + })), + }) + } + + const ids = (value: unknown) => + new Set(list(value).filter((id): id is string => typeof id === 'string')) + + return { + system: str(raw.system) ?? '', + nodes: unique, + modules, + domains: [...byDomain.entries()].map(([name, members]) => ({ + name, + modules: members, + })), + edges, + entryPoints: ids(raw.entryPoints), + deadEnds: ids(raw.deadEnds), + flows, + byId, + byFileBase, + } +} + +/** + * The topology node a legacy file belongs to: by its path relative to + * `legacy/<system>/` when the map recorded one, else by base name alone. + */ +export function nodeOfFile(topo: Topology, fileRel: string): TopoNode | null { + const wanted = norm(fileRel) + const candidates = topo.byFileBase.get(baseName(wanted).toLowerCase()) ?? [] + + return ( + candidates.find(node => node.file === wanted) ?? + candidates.find(node => wanted.endsWith(node.file ?? '\0')) ?? + candidates[0] ?? + null + ) +} diff --git a/plugins/code-modernization/hooks/reader/tracks.ts b/plugins/code-modernization/hooks/reader/tracks.ts new file mode 100644 index 00000000..5dfd8e14 --- /dev/null +++ b/plugins/code-modernization/hooks/reader/tracks.ts @@ -0,0 +1,89 @@ +/** + * The three ways the plugin's commands work on a system, and what each leaves behind. The pane follows + * whichever one the artifacts on disk say is under way, so a same-stack version uplift or a greenfield + * rebuild is followed as closely as a rewrite is. + */ + +export type TrackKey = 'transform' | 'uplift' | 'reimagine' + +export const TRACK_LABELS: Record<TrackKey, string> = { + transform: 'rewrite', + uplift: 'uplift', + reimagine: 'reimagine', +} + +/** The unit of work in each track, as the pane names a list of them. */ +export const TRACK_UNITS: Record<TrackKey, { one: string; many: string }> = { + transform: { one: 'module', many: 'Modules' }, + uplift: { one: 'module', many: 'Modules' }, + reimagine: { one: 'service', many: 'Services' }, +} + +/** What each state of a unit of work is called, by track: the same colors, worded for what the work is. */ +export const STATE_WORDS_BY_TRACK: Record<TrackKey, Record<string, string>> = { + transform: { + untouched: 'untouched', + scaffolded: 'scaffolded', + 'tests-written': 'tests written', + 'tests-red': 'tests red', + 'tests-failing': 'tests failing', + 'tests-green': 'tests green', + reviewed: 'reviewed', + ported: 'ported', + switched: 'switched', + }, + uplift: { + untouched: 'not migrated', + scaffolded: 'changed', + 'tests-written': 'tests written', + 'tests-red': 'worse than baseline', + 'tests-failing': 'tests failing', + 'tests-green': 'tests pass', + reviewed: 'matches baseline', + ported: 'ported', + switched: 'switched', + }, + reimagine: { + untouched: 'legacy', + scaffolded: 'scaffolded', + 'tests-written': 'tests written', + 'tests-red': 'tests red', + 'tests-failing': 'tests failing', + 'tests-green': 'tests green', + reviewed: 'reviewed', + ported: 'ported', + switched: 'switched', + }, +} + +/** Times (ms) of the artifacts that mark each track as under way; null where the artifact is absent. */ +export type TrackFacts = { + transformAt: number | null + upliftAt: number | null + reimagineAt: number | null +} + +/** + * The track under way: the one whose own artifacts were written last. The discovery artifacts (preflight, + * assess, map, rules, brief) feed all three and decide nothing. With no track's own artifacts a system + * is at the start of the rewrite path, which is where the plugin's commands begin. + */ +export function pickTrack(option: TrackKey | 'auto', facts: TrackFacts): TrackKey { + if (option !== 'auto') { + return option + } + + const ranked = ( + [ + ['transform', facts.transformAt], + ['reimagine', facts.reimagineAt], + ['uplift', facts.upliftAt], + ] as const + ).filter((entry): entry is readonly [TrackKey, number] => entry[1] !== null) + + if (ranked.length === 0) { + return 'transform' + } + + return ranked.reduce((best, entry) => (entry[1] > best[1] ? entry : best))[0] +} diff --git a/plugins/code-modernization/hooks/reader/uplift.ts b/plugins/code-modernization/hooks/reader/uplift.ts new file mode 100644 index 00000000..7cff7d36 --- /dev/null +++ b/plugins/code-modernization/hooks/reader/uplift.ts @@ -0,0 +1,285 @@ +import { join } from '../paths' +import { linesOf } from '../text' +import { unitOfPath, type EstateModel, type EstateUnit } from './estate-model' +import type { ReaderFs } from './fs' +import { readTests, stateOfUplift, type ModernizedModule, type ModuleState, type TestTotals } from './modernized' + +/** + * What a same-stack uplift leaves on disk, read without knowing the stack: the + * baseline's per-module results, the delta catalog's size, and which modules of the + * working copy differ from the legacy tree. + */ + +export type BaselineRow = { pass: number; fail: number; error: number; skip: number } + +export type Baseline = { + /** The file says the source runtime could not run here, so there is no per-test table. */ + isTargetOnly: boolean + /** Test results in all, from the table or the file's own totals line. */ + results: number | null + /** One row per module, keyed by the module path lower-cased with no trailing slash. */ + rows: Map<string, BaselineRow> + /** + * A baseline of one module has no per-module table: its title names the module and a small table gives the totals + * (`| Passed | 945 |`). Those totals are that module's row. + */ + headline: { title: string; row: BaselineRow } | null +} + +const int = (text: string | undefined): number => { + const value = Number((text ?? '').replace(/[,\s_]/g, '')) + + return Number.isFinite(value) ? value : 0 +} + +const cellsOf = (line: string): string[] => + line + .trim() + .replace(/^\|/, '') + .replace(/\|$/, '') + .split('|') + .map(cell => cell.trim()) + +const keyOf = (text: string): string => + text + .replace(/[`*]/g, '') + .replace(/^\.\//, '') + .replace(/\/+$/, '') + .trim() + .toLowerCase() + +/** Reads a baseline file: any table with pass and fail columns gives a row per module. */ +export function parseBaseline(text: string): Baseline { + const rows = new Map<string, BaselineRow>() + const lines = linesOf(text) + + for (let index = 0; index < lines.length - 1; index += 1) { + const head = lines[index] ?? '' + + if (!head.trim().startsWith('|') || !/^\s*\|?\s*:?-{2,}/.test(lines[index + 1] ?? '')) { + continue + } + + const names = cellsOf(head).map(name => name.replace(/[`*]/g, '').toLowerCase()) + const passAt = names.findIndex(name => /^pass/.test(name)) + const failAt = names.findIndex(name => /^fail/.test(name)) + + if (passAt < 0 || failAt < 0) { + continue + } + + const errorAt = names.findIndex(name => /^err/.test(name)) + const skipAt = names.findIndex(name => /^(skip|ignor|pending)/.test(name)) + const found = names.findIndex(name => /^(module|project|unit|suite|package|assembly|component|name)/.test(name)) + const keyAt = found >= 0 ? found : 0 + + for (let row = index + 2; row < lines.length && (lines[row] ?? '').trim().startsWith('|'); row += 1) { + const cells = cellsOf(lines[row] ?? '') + const key = keyOf(cells[keyAt] ?? '') + + if (key === '' || /^(total|totals|sum|all|overall)\b/.test(key)) { + continue + } + + rows.set(key, { + pass: int(cells[passAt]), + fail: int(cells[failAt]), + error: errorAt >= 0 ? int(cells[errorAt]) : 0, + skip: skipAt >= 0 ? int(cells[skipAt]) : 0, + }) + } + } + + const totals = /(?<![\d,])(\d[\d,]{0,14})\s+test\s+results?/i.exec(text)?.[1] ?? /\btotal\b[^\n]{0,40}?([\d,]{2,15})\s+tests?\b/i.exec(text)?.[1] + const summed = [...rows.values()].reduce((sum, row) => sum + row.pass + row.fail + row.error + row.skip, 0) + + // The first `| Passed | 945 |` style lines, whatever else the file holds. + const counts: Partial<Record<keyof BaselineRow, number>> = {} + + for (const line of lines) { + const found = /^\s*\|\s*\**\s*(pass(?:ed)?|fail(?:ed|ures?)?|errors?|skipped|skip)\s*\**\s*\|\s*\**\s*([\d,]+)\s*\**\s*\|?\s*$/i.exec(line) + + if (found !== null) { + const word = (found[1] ?? '').toLowerCase() + const key: keyof BaselineRow = word.startsWith('pass') ? 'pass' : word.startsWith('fail') ? 'fail' : word.startsWith('err') ? 'error' : 'skip' + + counts[key] ??= int(found[2]) + } + } + + const title = (lines.find(line => /^#\s+/.test(line)) ?? '').replace(/^#\s+/, '').replace(/[`*]/g, '').trim() + + return { + isTargetOnly: /^\s*target-only:/im.test(text), + results: totals !== undefined ? int(totals) : rows.size > 0 ? summed : null, + rows, + headline: + counts.pass !== undefined && counts.fail !== undefined && title !== '' + ? { title, row: { pass: counts.pass, fail: counts.fail, error: counts.error ?? 0, skip: counts.skip ?? 0 } } + : null, + } +} + +/** + * The baseline's row for one module: its own row, else the file's headline totals when its title names the module + * (a pilot's baseline is one module's, and says so in its title). Null when there is nothing to compare with. + */ +export function baselineRowOf(baseline: Baseline | null, dir: string): BaselineRow | null { + if (baseline === null || dir === '') { + return null + } + + const own = baseline.rows.get(keyOf(dir)) + + if (own !== undefined) { + return own + } + + if (baseline.headline === null) { + return null + } + + const escaped = keyOf(dir).replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + + return new RegExp(`(?<![\\w./-])${escaped}(?![\\w./-])`, 'i').test(baseline.headline.title) ? baseline.headline.row : null +} + +export type Catalog = { + count: number + /** Lower-cased base name of a file the catalog cites, to the delta ids it is cited under. */ + byFileBase: Map<string, string[]> +} + +/** + * What the delta catalog holds: how many deltas (its own statement of the count, else its `D<n>` ids), and which + * delta each file it names is cited under: the delta a table row or a heading names last, before the file. + */ +export function parseCatalog(text: string): Catalog | null { + const stated = /\b(\d+)\s+(?:distinct\s+)?deltas\b/i.exec(text)?.[1] + const ids = new Set<string>() + const byFileBase = new Map<string, string[]>() + let current: string | null = null + + for (const line of linesOf(text)) { + const opens = /^\s*(?:\|\s*\**|#{1,6}\s+\**)(D-?\d{1,3})\b/.exec(line)?.[1] + + if (opens !== undefined) { + current = opens + ids.add(opens) + } + + for (const found of line.matchAll(/D-?\d{1,3}\b/g)) { + ids.add(found[0]) + } + + if (current === null) { + continue + } + + for (const cited of line.matchAll(/`([\w./@+-]+\.[A-Za-z0-9]{1,8})(?::\d+(?:[-–]\d+)?)?`/g)) { + const base = (cited[1] ?? '').split('/').at(-1)?.toLowerCase() ?? '' + const held = byFileBase.get(base) ?? [] + + if (base !== '' && !held.includes(current)) { + held.push(current) + byFileBase.set(base, held) + } + } + } + + const count = stated !== undefined && int(stated) > 0 ? int(stated) : ids.size + + return count > 0 ? { count, byFileBase } : null +} + +/** + * The paths that differ between a tree and the working copy made from it: a file present in both with another size, + * one only the copy has, and one only the tree has. It compares names and sizes, so an edit that keeps a file's size + * exactly is not seen here; a test run in that module, or an edit made during the session, shows it. + */ +export function changedPathsOf(before: ReadonlyMap<string, number>, after: ReadonlyMap<string, number>): string[] { + const out: string[] = [] + + for (const [path, size] of after) { + if (before.get(path) !== size) { + out.push(path) + } + } + + for (const path of before.keys()) { + if (!after.has(path)) { + out.push(path) + } + } + + return out +} + +/** The estate's unit ids that hold at least one of the changed paths. */ +export function changedUnitsOf(estate: EstateModel, changed: readonly string[]): Set<string> { + const ids = new Set<string>() + + for (const path of changed) { + const unit = unitOfPath(estate, null, path) + + if (unit !== null) { + ids.add(unit.id) + } + } + + return ids +} + +/** + * Each unit of the working copy that the uplift has touched or tested, with its state: + * changed, tested, matching the baseline, or worse than it. `observed` holds the totals of test + * runs seen this session, used where no report file was left. + */ +export async function readUpliftUnits( + fs: ReaderFs, + workingRoot: string, + estate: EstateModel, + changed: ReadonlySet<string>, + baseline: Baseline | null, + observed: ReadonlyMap<string, TestTotals>, +): Promise<{ unit: EstateUnit; module: ModernizedModule }[]> { + const out: { unit: EstateUnit; module: ModernizedModule }[] = [] + + for (const unit of estate.units) { + if (unit.dir === undefined) { + if (changed.has(unit.id)) { + out.push({ unit, module: moduleOf(unit.name, join(workingRoot, unit.file ?? unit.id), null, 'scaffolded') }) + } + + continue + } + + const path = unit.dir === '' ? workingRoot : join(workingRoot, unit.dir) + const tests = (await readTests(fs, path)) ?? observed.get(path) ?? null + const row = baselineRowOf(baseline, unit.dir) + const state = stateOfUplift(tests, row, changed.has(unit.id)) + + if (state !== 'untouched') { + out.push({ unit, module: moduleOf(unit.dir === '' ? unit.name : unit.dir, path, tests, state) }) + } + } + + return out +} + +function moduleOf(dir: string, path: string, tests: TestTotals | null, state: ModuleState): ModernizedModule { + return { + dir, + path, + hasMain: true, + hasTests: tests !== null, + hasNotes: false, + hasReviewSection: false, + isPortedNotSwitched: false, + isSwitched: false, + followUps: 0, + tests, + state, + mtimeMs: 0, + } +} diff --git a/plugins/code-modernization/hooks/reader/verification.ts b/plugins/code-modernization/hooks/reader/verification.ts new file mode 100644 index 00000000..94ef6666 --- /dev/null +++ b/plugins/code-modernization/hooks/reader/verification.ts @@ -0,0 +1,164 @@ +import { plain } from '../text' +import type { TrackKey } from './tracks' + +/** + * `analysis/<system>/VERIFICATION.json`, written by the proof pack (`scripts/proof_pack.py`, run by the verify + * command): one verdict per built module, computed by a script from the evidence files. The pane shows the verdicts + * as they are and never computes one. + * + * The file is text from a run on an untrusted tree, and its shape may move between releases, so it is read + * tolerantly: a field of the wrong type is ignored, a verdict is only ever one of the three exact words, a name or a + * reason is cut to one short plain line, and how much is read is capped. + */ + +export type Verdict = 'PROVEN' | 'PARTLY PROVEN' | 'NOT PROVEN' + +const VERDICTS: readonly string[] = ['PROVEN', 'PARTLY PROVEN', 'NOT PROVEN'] + +/** What one module's verdict is, as the file says it. */ +export type ProofEntry = { + /** The module: a folder under `modernized/<system>/`, or the uplift's whole working copy. */ + name: string + track: TrackKey + verdict: Verdict + /** The first reason the verdict is not PROVEN, as one plain line; empty when the file gives none. */ + reason: string + /** When the module was checked; null when the file gives no time this can read. */ + atMs: number | null +} + +export type Verification = { + overall: Verdict | null + entries: ProofEntry[] + /** Built folders that hold only test code and the files that build it: the pack lists them and judges none, so the pane says nothing of them. */ + tooling: { track: TrackKey; name: string }[] +} + +/** How the module is standing with the proof, for a chip and for the next step. */ +export type ProofState = { + /** `none`: built, and nothing checked it yet. `changed`: checked, and its code or tests are newer than that check. */ + state: 'proven' | 'partly' | 'not' | 'changed' | 'none' + verdict?: Verdict + reason: string +} + +const MAX_TEXT = 4_000_000 +const MAX_ENTRIES = 1_000 +const MAX_NAME = 80 +const MAX_REASON = 160 + +/** The proof pack's own spelling of a track: a rewrite is `rewrite` there. */ +const TRACK_OF: Record<string, TrackKey> = { rewrite: 'transform', transform: 'transform', uplift: 'uplift', reimagine: 'reimagine' } + +/** `2026-09-24 20:31 UTC` (the pack's stamp) as milliseconds since the epoch; null when it is anything else. */ +function timeOf(value: unknown): number | null { + if (typeof value !== 'string') { + return null + } + + const match = /^(\d{4})-(\d{2})-(\d{2})[ T](\d{2}):(\d{2})(?::(\d{2}))?\s*(?:UTC|Z)?$/.exec(value.trim()) + + if (match === null) { + return null + } + + const ms = Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3]), Number(match[4]), Number(match[5]), Number(match[6] ?? 0)) + + return Number.isFinite(ms) ? ms : null +} + +const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === 'object' && value !== null && !Array.isArray(value) + +/** The file's text as a Verification; null when it is not one. */ +export function parseVerification(text: string): Verification | null { + if (text.length > MAX_TEXT) { + return null + } + + let raw: unknown + + try { + raw = JSON.parse(text) + } catch { + return null + } + + if (!isRecord(raw) || !Array.isArray(raw.modules)) { + return null + } + + const entries: ProofEntry[] = [] + + for (const item of raw.modules.slice(0, MAX_ENTRIES) as unknown[]) { + if (!isRecord(item)) { + continue + } + + const name = typeof item.name === 'string' ? plain(item.name, MAX_NAME) : '' + const track = typeof item.track === 'string' ? TRACK_OF[item.track] : undefined + + // Only the exact words count: a verdict with anything added to it is not one. + if (name === '' || track === undefined || typeof item.verdict !== 'string' || !VERDICTS.includes(item.verdict)) { + continue + } + + const first = Array.isArray(item.reasons) ? (item.reasons as unknown[]).find(reason => typeof reason === 'string' && reason.trim() !== '') : undefined + + entries.push({ + name, + track, + verdict: item.verdict as Verdict, + reason: typeof first === 'string' ? plain(first, MAX_REASON) : '', + atMs: timeOf(item.verifiedAt), + }) + } + + const overall = isRecord(raw.overall) && typeof raw.overall.verdict === 'string' && VERDICTS.includes(raw.overall.verdict) ? (raw.overall.verdict as Verdict) : null + const tooling: Verification['tooling'] = [] + + for (const item of (Array.isArray(raw.toolingOnly) ? (raw.toolingOnly as unknown[]) : []).slice(0, MAX_ENTRIES)) { + const name = isRecord(item) && typeof item.name === 'string' ? plain(item.name, MAX_NAME) : '' + const track = isRecord(item) && typeof item.track === 'string' && Object.hasOwn(TRACK_OF, item.track) ? TRACK_OF[item.track] : undefined + + if (name !== '' && track !== undefined) { + tooling.push({ track, name }) + } + } + + return { overall, entries, tooling } +} + +/** The pack stamps minutes; a file written in the same minute as the check is not newer than it. */ +const SLACK_MS = 90_000 + +/** + * Where a built module stands with the proof. Null when it has no notes and no verdict, so nothing is claimed of a + * module still being built, and null for a folder the pack lists as test tooling (only test code and build files: nothing to prove). + * + * @param verification the file, or null when there is none + * @param track the track the pane follows + * @param name the module's folder under `modernized/` + * @param changedAtMs when the module's notes or test reports last changed (0 when unknown) + * @param hasNotes whether the module has notes: the sign a module was finished being built + */ +export function proofOfModule( + verification: Verification | null, + track: TrackKey, + name: string, + changedAtMs: number, + hasNotes: boolean, +): ProofState | null { + const entry = verification?.entries.find(candidate => candidate.track === track && candidate.name.toLowerCase() === plain(name, MAX_NAME).toLowerCase()) + + if (entry === undefined) { + const isTooling = verification?.tooling.some(item => item.track === track && item.name.toLowerCase() === plain(name, MAX_NAME).toLowerCase()) === true + + return hasNotes && !isTooling ? { state: 'none', reason: '' } : null + } + + if (entry.atMs !== null && changedAtMs > entry.atMs + SLACK_MS) { + return { state: 'changed', verdict: entry.verdict, reason: entry.reason } + } + + return { state: entry.verdict === 'PROVEN' ? 'proven' : entry.verdict === 'PARTLY PROVEN' ? 'partly' : 'not', verdict: entry.verdict, reason: entry.reason } +} |
