Coverage for graphqler / compiler / resolvers / llm / comparison.py: 24%

1"""ResolverComparison — aggregates and persists the side-by-side comparison of 

2LLM resolver output vs classic resolver output. 

3 

4The saved JSON file (`eval/resolver_comparison.json`) is structured to make 

5systematic analysis easy: 

6 

7{ 

8 "summary": { 

9 "total_mutations": int, 

10 "mutations_that_differ": int, 

11 "total_queries": int, 

12 "queries_that_differ": int 

13 }, 

14 "mutations": { 

15 "<name>": { 

16 "classic": {"mutationType": "...", "hardDependsOn": {...}, "softDependsOn": {...}}, 

17 "llm": {"mutationType": "...", "hardDependsOn": {...}, "softDependsOn": {...}}, 

18 "differs": bool, 

19 "diff": { "<field>": {"classic": ..., "llm": ...} } 

20 } 

21 }, 

22 "queries": { ... same shape without mutationType ... } 

23} 

24""" 

25 

26import json 

27import logging 

28from pathlib import Path 

29 

30logger = logging.getLogger(__name__) 

31 

32COMPARISON_FILE_NAME = "eval/resolver_comparison.json" 

33 

34 

35class ResolverComparison: 

36 """Holds and saves the LLM-vs-classic comparison for mutations and queries.""" 

37 

38 def __init__(self, mutation_comparison: dict, query_comparison: dict): 

39 """ 

40 Args: 

41 mutation_comparison (dict): Output of LLMMutationObjectResolver.comparison. 

42 query_comparison (dict): Output of LLMQueryObjectResolver.comparison. 

43 """ 

44 self.mutation_comparison = mutation_comparison 

45 self.query_comparison = query_comparison 

46 

47 def build(self) -> dict: 

48 """Build the full comparison document. 

49 

50 Returns: 

51 dict: The complete comparison document with summary + per-endpoint diffs. 

52 """ 

53 mutations_that_differ = sum(1 for v in self.mutation_comparison.values() if v.get("differs")) 

54 queries_that_differ = sum(1 for v in self.query_comparison.values() if v.get("differs")) 

55 

56 return { 

57 "summary": { 

58 "total_mutations": len(self.mutation_comparison), 

59 "mutations_that_differ": mutations_that_differ, 

60 "total_queries": len(self.query_comparison), 

61 "queries_that_differ": queries_that_differ, 

62 }, 

63 "mutations": self.mutation_comparison, 

64 "queries": self.query_comparison, 

65 } 

66 

67 def save(self, output_dir: str) -> str: 

68 """Serialize and write the comparison document to disk. 

69 

70 Args: 

71 output_dir (str): The run output directory (same as the compiled/ parent). 

72 

73 Returns: 

74 str: Absolute path of the written file. 

75 """ 

76 dest = Path(output_dir) / COMPARISON_FILE_NAME 

77 dest.parent.mkdir(parents=True, exist_ok=True) 

78 document = self.build() 

79 with open(dest, "w") as f: 

80 json.dump(document, f, indent=2) 

81 logger.info(f"Resolver comparison saved to {dest}") 

82 summary = document["summary"] 

83 print( 

84 f"(C) LLM resolver comparison: " 

85 f"{summary['mutations_that_differ']}/{summary['total_mutations']} mutations differ, " 

86 f"{summary['queries_that_differ']}/{summary['total_queries']} queries differ → {dest}" 

87 ) 

88 return str(dest) 

89 

90 def print_diff_summary(self): 

91 """Print a human-readable diff summary to stdout.""" 

92 doc = self.build() 

93 summary = doc["summary"] 

94 print(f"\n{'='*60}") 

95 print(" Resolver comparison: LLM vs Classic") 

96 print(f"{'='*60}") 

97 print(f" Mutations : {summary['mutations_that_differ']} / {summary['total_mutations']} differ") 

98 print(f" Queries : {summary['queries_that_differ']} / {summary['total_queries']} differ") 

99 

100 for section, label in (("mutations", "MUTATION"), ("queries", "QUERY")): 

101 for name, entry in doc[section].items(): 

102 if not entry.get("differs"): 

103 continue 

104 print(f"\n [{label}] {name}") 

105 for field, values in entry.get("diff", {}).items(): 

106 print(f" {field}:") 

107 print(f" classic: {values['classic']}") 

108 print(f" llm : {values['llm']}") 

109 print(f"{'='*60}\n")