feat: add progressive pipeline runner for dfgraph · nonbinary.computer/or1-design@f021296

OR-1 dataflow CPU sketch

feat: add progressive pipeline runner for dfgraph

Implements dfgraph/pipeline.py with run_progressive() function that runs
the assembler pipeline (parse -> lower -> resolve -> place -> allocate)
individually, capturing the deepest successful IRGraph even when later
passes fail.

Key features:
- PipelineStage enum tracking progress through pipeline
- PipelineResult dataclass with graph, stage, errors, and parse_error
- Graceful error handling at parse stage (returns PARSE_ERROR)
- Error accumulation at resolve/place/allocate stages (stops but returns graph)

Verifies AC2.1, AC2.2, AC5.2, AC5.3

Orual 2 weeks ago f0212962 78c259c4

+490

2 changed files

expand all

dfgraph

pipeline.py

tests

test_dfgraph_pipeline.py

+119

dfgraph/pipeline.py

··· 1 + """Progressive pipeline runner for dfasm assembly. 2 + 3 + Runs assembler passes individually, capturing the deepest successful IRGraph 4 + even when later passes fail. This enables partial graph visualisation. 5 + """ 6 + 7 + from __future__ import annotations 8 + 9 + from dataclasses import dataclass 10 + from enum import Enum 11 + from typing import Optional 12 + 13 + from lark import Lark 14 + from pathlib import Path 15 + 16 + from asm.ir import IRGraph 17 + from asm.lower import lower 18 + from asm.resolve import resolve 19 + from asm.place import place 20 + from asm.allocate import allocate 21 + from asm.errors import AssemblyError 22 + 23 + 24 + _GRAMMAR_PATH = Path(__file__).parent.parent / "dfasm.lark" 25 + _parser: Optional[Lark] = None 26 + 27 + 28 + def _get_parser() -> Lark: 29 + """Lazily initialize and cache the Lark parser.""" 30 + global _parser 31 + if _parser is None: 32 + _parser = Lark( 33 + _GRAMMAR_PATH.read_text(), 34 + parser="earley", 35 + propagate_positions=True, 36 + ) 37 + return _parser 38 + 39 + 40 + class PipelineStage(Enum): 41 + """Enumeration of pipeline stages.""" 42 + PARSE_ERROR = "parse_error" 43 + LOWER = "lower" 44 + RESOLVE = "resolve" 45 + PLACE = "place" 46 + ALLOCATE = "allocate" 47 + 48 + 49 + @dataclass(frozen=True) 50 + class PipelineResult: 51 + """Result of running the progressive pipeline. 52 + 53 + Attributes: 54 + graph: The IRGraph at the deepest successful stage, or None if parse failed 55 + stage: The stage where progress stopped 56 + errors: All AssemblyErrors accumulated from the pipeline 57 + parse_error: String representation of parse error, if stage is PARSE_ERROR 58 + """ 59 + graph: Optional[IRGraph] 60 + stage: PipelineStage 61 + errors: list[AssemblyError] 62 + parse_error: Optional[str] = None 63 + 64 + 65 + def run_progressive(source: str) -> PipelineResult: 66 + """Run the assembly pipeline progressively, capturing errors at each stage. 67 + 68 + Unlike asm._run_pipeline() which raises on first error, this function runs 69 + each pass independently and accumulates errors, allowing partial graphs to 70 + be captured for visualization. 71 + 72 + Pipeline stages: 73 + 1. Parse: Convert source to Lark CST (may raise lark.exceptions.UnexpectedInput) 74 + 2. Lower: CST to IRGraph (may accumulate errors but doesn't stop) 75 + 3. Resolve: Validate edge endpoints and scope (may accumulate errors) 76 + 4. Place: Validate/auto-place nodes on PEs (stops before this if resolve failed) 77 + 5. Allocate: Assign IRAM offsets and context slots (stops before this if place failed) 78 + 79 + Args: 80 + source: dfasm source code as a string 81 + 82 + Returns: 83 + PipelineResult containing the deepest graph, stage reached, and all errors 84 + """ 85 + # Stage 1: Parse 86 + try: 87 + tree = _get_parser().parse(source) 88 + except Exception as exc: 89 + return PipelineResult( 90 + graph=None, 91 + stage=PipelineStage.PARSE_ERROR, 92 + errors=[], 93 + parse_error=str(exc), 94 + ) 95 + 96 + # Stage 2: Lower 97 + graph = lower(tree) 98 + stage = PipelineStage.LOWER 99 + 100 + # Stage 3: Resolve 101 + # Note: resolve() always runs after lower (matches asm._run_pipeline behaviour) 102 + graph = resolve(graph) 103 + stage = PipelineStage.RESOLVE 104 + 105 + # Stage 4: Place (skip if resolve accumulated errors) 106 + if not graph.errors: 107 + graph = place(graph) 108 + stage = PipelineStage.PLACE 109 + 110 + # Stage 5: Allocate (skip if place accumulated errors) 111 + if not graph.errors: 112 + graph = allocate(graph) 113 + stage = PipelineStage.ALLOCATE 114 + 115 + return PipelineResult( 116 + graph=graph, 117 + stage=stage, 118 + errors=list(graph.errors), 119 + )

+371

tests/test_dfgraph_pipeline.py

··· 1 + """Tests for the progressive pipeline runner (dfgraph/pipeline.py). 2 + 3 + Tests verify: 4 + - dataflow-renderer.AC2.1: Clean source produces allocate-stage graph with zero errors 5 + - dataflow-renderer.AC2.2: Source with errors at any stage still returns a graph with valid nodes 6 + - dataflow-renderer.AC5.2: Error results contain AssemblyError objects with loc.line, category, and message 7 + - dataflow-renderer.AC5.3: Name resolution errors include suggestions (Levenshtein) 8 + - Parse errors are handled gracefully and return PARSE_ERROR stage with graph=None 9 + """ 10 + 11 + from textwrap import dedent 12 + 13 + from dfgraph.pipeline import run_progressive, PipelineStage, PipelineResult 14 + from asm.errors import ErrorCategory 15 + 16 + 17 + class TestProgressivePipelineCleanSource: 18 + """Tests for AC2.1: Clean source produces allocate-stage graph with zero errors.""" 19 + 20 + def test_simple_const_add_program(self): 21 + """Clean program with const and add nodes produces ALLOCATE stage with no errors.""" 22 + source = dedent("""\ 23 + @system pe=2, sm=0 24 + &c1|pe0 <| const, 3 25 + &c2|pe0 <| const, 7 26 + &result|pe0 <| add 27 + &output|pe1 <| pass 28 + &c1|pe0 |> &result|pe0:L 29 + &c2|pe0 |> &result|pe0:R 30 + &result|pe0 |> &output|pe1:L 31 + """) 32 + 33 + result = run_progressive(source) 34 + 35 + assert result.stage == PipelineStage.ALLOCATE 36 + assert len(result.errors) == 0 37 + assert result.graph is not None 38 + assert len(result.graph.nodes) >= 4 39 + 40 + def test_program_with_functions(self): 41 + """Clean program with function-scoped labels produces ALLOCATE stage with no errors.""" 42 + source = dedent("""\ 43 + @system pe=2, sm=0 44 + $main |> { 45 + &input <| const, 42 46 + &process <| pass 47 + &input |> &process:L 48 + } 49 + """) 50 + 51 + result = run_progressive(source) 52 + 53 + assert result.stage == PipelineStage.ALLOCATE 54 + assert len(result.errors) == 0 55 + assert result.graph is not None 56 + 57 + def test_program_with_sm_operations(self): 58 + """Clean program with SM operations produces ALLOCATE stage with no errors.""" 59 + source = dedent("""\ 60 + @system pe=2, sm=1 61 + @val|sm0:5 = 0x42 62 + &trigger|pe0 <| const, 1 63 + &reader|pe0 <| read, 5 64 + &relay|pe1 <| pass 65 + &trigger|pe0 |> &reader|pe0:L 66 + &reader|pe0 |> &relay|pe1:L 67 + """) 68 + 69 + result = run_progressive(source) 70 + 71 + assert result.stage == PipelineStage.ALLOCATE 72 + assert len(result.errors) == 0 73 + assert result.graph is not None 74 + 75 + def test_program_with_cross_pe_routing(self): 76 + """Clean program with cross-PE routing produces ALLOCATE stage with no errors.""" 77 + source = dedent("""\ 78 + @system pe=3, sm=0 79 + &source|pe0 <| const, 99 80 + &dest|pe1 <| pass 81 + &output|pe2 <| pass 82 + &source|pe0 |> &dest|pe1:L 83 + &dest|pe1 |> &output|pe2:L 84 + """) 85 + 86 + result = run_progressive(source) 87 + 88 + assert result.stage == PipelineStage.ALLOCATE 89 + assert len(result.errors) == 0 90 + assert result.graph is not None 91 + 92 + 93 + class TestProgressivePipelineResolveErrors: 94 + """Tests for AC2.2: Source with resolve errors still returns graph at RESOLVE stage.""" 95 + 96 + def test_undefined_name_reference_stops_at_resolve(self): 97 + """Program with undefined node reference produces graph at RESOLVE stage.""" 98 + source = dedent("""\ 99 + @system pe=2, sm=0 100 + &a <| const, 42 101 + &b <| add 102 + &a |> &undefined:L 103 + """) 104 + 105 + result = run_progressive(source) 106 + 107 + assert result.stage == PipelineStage.RESOLVE 108 + assert len(result.errors) > 0 109 + assert result.graph is not None 110 + assert len(result.graph.nodes) >= 2 111 + 112 + def test_name_error_has_category(self): 113 + """NAME error contains ErrorCategory.NAME.""" 114 + source = dedent("""\ 115 + @system pe=2, sm=0 116 + &a <| pass 117 + &b <| add 118 + &a |> &undefined:L 119 + """) 120 + 121 + result = run_progressive(source) 122 + 123 + assert result.stage == PipelineStage.RESOLVE 124 + name_errors = [e for e in result.errors if e.category == ErrorCategory.NAME] 125 + assert len(name_errors) > 0 126 + 127 + def test_name_error_has_suggestions(self): 128 + """NAME error includes suggestions when available (AC5.3).""" 129 + source = dedent("""\ 130 + @system pe=2, sm=0 131 + &nonexistent <| pass 132 + &a <| add 133 + &a |> &nonexistant:L 134 + """) 135 + 136 + result = run_progressive(source) 137 + 138 + assert result.stage == PipelineStage.RESOLVE 139 + name_errors = [e for e in result.errors if e.category == ErrorCategory.NAME] 140 + assert len(name_errors) > 0 141 + error = name_errors[0] 142 + # Levenshtein suggestions should be present 143 + assert len(error.suggestions) > 0 144 + 145 + def test_scope_violation_stops_at_resolve(self): 146 + """Program with scope violation produces graph at RESOLVE stage.""" 147 + source = dedent("""\ 148 + @system pe=2, sm=0 149 + $foo |> { 150 + &private <| pass 151 + } 152 + &top <| add 153 + &top |> &private:L 154 + """) 155 + 156 + result = run_progressive(source) 157 + 158 + assert result.stage == PipelineStage.RESOLVE 159 + assert len(result.errors) > 0 160 + scope_errors = [e for e in result.errors if e.category == ErrorCategory.SCOPE] 161 + assert len(scope_errors) > 0 162 + 163 + 164 + class TestProgressivePipelinePlacementErrors: 165 + """Tests for AC2.2: Source with placement errors returns graph at PLACE stage.""" 166 + 167 + def test_explicit_placement_violation_stops_at_place(self): 168 + """Program with invalid explicit PE placement produces graph at PLACE stage.""" 169 + source = dedent("""\ 170 + @system pe=2, sm=0 171 + &a|pe5 <| const, 1 172 + &b <| add 173 + &a |> &b:L 174 + """) 175 + 176 + result = run_progressive(source) 177 + 178 + assert result.stage == PipelineStage.PLACE 179 + assert len(result.errors) > 0 180 + assert result.graph is not None 181 + # Should have placement errors 182 + placement_errors = [e for e in result.errors if e.category == ErrorCategory.PLACEMENT] 183 + assert len(placement_errors) > 0 184 + 185 + 186 + class TestProgressivePipelineErrorMetadata: 187 + """Tests for AC5.2: Errors contain loc.line, category, and message fields.""" 188 + 189 + def test_error_contains_source_location(self): 190 + """Error contains SourceLoc with line and column.""" 191 + source = dedent("""\ 192 + @system pe=2, sm=0 193 + &a <| pass 194 + &b <| add 195 + &a |> &undefined:L 196 + """) 197 + 198 + result = run_progressive(source) 199 + 200 + assert len(result.errors) > 0 201 + error = result.errors[0] 202 + assert error.loc.line > 0 203 + assert error.loc.column >= 0 204 + 205 + def test_error_contains_category(self): 206 + """Error contains an ErrorCategory enum value.""" 207 + source = dedent("""\ 208 + @system pe=2, sm=0 209 + &a <| pass 210 + &b <| add 211 + &a |> &undefined:L 212 + """) 213 + 214 + result = run_progressive(source) 215 + 216 + assert len(result.errors) > 0 217 + error = result.errors[0] 218 + assert isinstance(error.category, ErrorCategory) 219 + 220 + def test_error_contains_message(self): 221 + """Error contains a non-empty message string.""" 222 + source = dedent("""\ 223 + @system pe=2, sm=0 224 + &a <| pass 225 + &b <| add 226 + &a |> &undefined:L 227 + """) 228 + 229 + result = run_progressive(source) 230 + 231 + assert len(result.errors) > 0 232 + error = result.errors[0] 233 + assert len(error.message) > 0 234 + assert isinstance(error.message, str) 235 + 236 + 237 + class TestProgressivePipelineParseErrors: 238 + """Tests for parse errors: invalid syntax returns PARSE_ERROR stage.""" 239 + 240 + def test_invalid_syntax_returns_parse_error(self): 241 + """Syntactically invalid source returns PARSE_ERROR stage.""" 242 + source = "this is not valid dfasm @ @@ {{{ syntax" 243 + 244 + result = run_progressive(source) 245 + 246 + assert result.stage == PipelineStage.PARSE_ERROR 247 + assert result.graph is None 248 + assert result.parse_error is not None 249 + assert isinstance(result.parse_error, str) 250 + assert len(result.parse_error) > 0 251 + 252 + def test_parse_error_no_errors_list(self): 253 + """PARSE_ERROR result has empty errors list (error is in parse_error field).""" 254 + source = "@@@" 255 + 256 + result = run_progressive(source) 257 + 258 + assert result.stage == PipelineStage.PARSE_ERROR 259 + assert len(result.errors) == 0 260 + 261 + 262 + class TestProgressivePipelineResultDataclass: 263 + """Tests for PipelineResult dataclass structure.""" 264 + 265 + def test_result_is_frozen_dataclass(self): 266 + """PipelineResult is a frozen dataclass.""" 267 + source = "@system pe=1, sm=0\n&a <| pass" 268 + 269 + result = run_progressive(source) 270 + 271 + assert isinstance(result, PipelineResult) 272 + # Should be frozen (immutable) 273 + try: 274 + result.stage = PipelineStage.LOWER 275 + assert False, "PipelineResult should be frozen" 276 + except (AttributeError, ValueError): 277 + pass # Expected 278 + 279 + def test_result_stage_is_enum(self): 280 + """result.stage is a PipelineStage enum value.""" 281 + source = "@system pe=1, sm=0\n&a <| pass" 282 + 283 + result = run_progressive(source) 284 + 285 + assert isinstance(result.stage, PipelineStage) 286 + 287 + def test_result_errors_is_list(self): 288 + """result.errors is a list of AssemblyError objects.""" 289 + source = "@system pe=2, sm=0\n&a <| pass\n&b <| add\n&a |> &undefined:L" 290 + 291 + result = run_progressive(source) 292 + 293 + assert isinstance(result.errors, list) 294 + if len(result.errors) > 0: 295 + from asm.errors import AssemblyError 296 + assert all(isinstance(e, AssemblyError) for e in result.errors) 297 + 298 + 299 + class TestProgressivePipelineGraphStructure: 300 + """Tests for IRGraph structure returned at each stage.""" 301 + 302 + def test_graph_at_lower_stage_has_nodes(self): 303 + """IRGraph at LOWER stage contains IRNodes.""" 304 + source = dedent("""\ 305 + @system pe=1, sm=0 306 + &a <| const, 42 307 + &b <| pass 308 + """) 309 + 310 + result = run_progressive(source) 311 + 312 + assert result.graph is not None 313 + assert len(result.graph.nodes) > 0 314 + assert all(name for name in result.graph.nodes.keys()) 315 + 316 + def test_graph_at_resolve_stage_has_nodes(self): 317 + """IRGraph at RESOLVE stage contains IRNodes.""" 318 + source = dedent("""\ 319 + @system pe=2, sm=0 320 + &a <| pass 321 + &b <| add 322 + &a |> &undefined:L 323 + """) 324 + 325 + result = run_progressive(source) 326 + 327 + assert result.stage == PipelineStage.RESOLVE 328 + assert result.graph is not None 329 + assert len(result.graph.nodes) > 0 330 + 331 + def test_graph_at_allocate_stage_has_offsets(self): 332 + """IRGraph at ALLOCATE stage has IRAM offsets on nodes.""" 333 + source = dedent("""\ 334 + @system pe=2, sm=0 335 + &a <| const, 42 336 + &b <| pass 337 + &a |> &b:L 338 + """) 339 + 340 + result = run_progressive(source) 341 + 342 + assert result.stage == PipelineStage.ALLOCATE 343 + # After allocation, nodes should have iram_offset set 344 + for node in result.graph.nodes.values(): 345 + assert node.iram_offset is not None 346 + 347 + 348 + class TestProgressivePipelineEmptySource: 349 + """Tests for edge cases.""" 350 + 351 + def test_empty_source_parses_successfully(self): 352 + """Empty source parses and returns ALLOCATE stage with default @system.""" 353 + source = "" 354 + 355 + result = run_progressive(source) 356 + 357 + # Empty source parses successfully and creates a default @system pragma 358 + assert result.stage == PipelineStage.ALLOCATE 359 + assert result.graph is not None 360 + # Should have default system config (pe=1, sm=1) 361 + assert result.graph.system is not None 362 + 363 + def test_system_pragma_only(self): 364 + """Source with only @system pragma parses successfully.""" 365 + source = "@system pe=2, sm=0" 366 + 367 + result = run_progressive(source) 368 + 369 + assert result.stage == PipelineStage.ALLOCATE 370 + assert len(result.errors) == 0 371 + assert result.graph is not None