Coverage for graphqler / compiler / resolvers / llm / llm_query_object_resolver.py: 28%

1"""LLM-backed query resolver. 

2 

3Resolves hardDependsOn + softDependsOn for every query by asking an LLM for its 

4interpretation of the schema, then merging that result with the classic ID-based 

5resolver output. 

6 

7The `.comparison` attribute (set after `resolve()`) holds the side-by-side diff 

8between the LLM result and the classic result for systematic analysis. 

9""" 

10 

11import json 

12import copy 

13import logging 

14 

15from graphqler import config 

16from graphqler.compiler.resolvers.query_object_resolver import QueryObjectResolver 

17from .llm_resolver import LLMResolver 

18from .prompt_templates import QUERY_SYSTEM_PROMPT, QUERY_USER_PROMPT_TEMPLATE 

19 

20logger = logging.getLogger(__name__) 

21 

22 

23class LLMQueryObjectResolver(LLMResolver): 

24 """Resolves queries using an LLM, with optional fallback to the classic resolver.""" 

25 

26 def __init__(self): 

27 super().__init__() 

28 self.comparison: dict = {} # populated by resolve(); keyed by query name 

29 

30 def resolve(self, objects: dict, queries: dict, input_objects: dict) -> dict: 

31 """Resolve queries via LLM, falling back to classic resolver on failure. 

32 

33 Args: 

34 objects (dict): Compiled objects. 

35 queries (dict): Raw parsed queries. 

36 input_objects (dict): Raw parsed input objects. 

37 

38 Returns: 

39 dict: Queries enriched with hardDependsOn, softDependsOn. 

40 """ 

41 # Always run classic resolver — used as fallback and for comparison 

42 classic_queries = QueryObjectResolver().resolve(objects, copy.deepcopy(queries), input_objects) 

43 

44 try: 

45 llm_raw = self._call_llm_for_queries(objects, queries) 

46 llm_validated = self.validate_llm_query_result(llm_raw, list(queries.keys()), objects) 

47 merged = self.merge_with_classic(llm_validated, classic_queries, list(queries.keys())) 

48 self.comparison = self._build_comparison(classic_queries, merged, queries.keys()) 

49 logger.info(f"LLM query resolver: resolved {len(llm_validated)}/{len(queries)} queries") 

50 return merged 

51 except Exception as exc: 

52 if config.LLM_RESOLVER_FALLBACK_TO_ID: 

53 logger.warning(f"LLM query resolver failed ({exc}), falling back to classic resolver") 

54 self.comparison = {} 

55 return classic_queries 

56 raise 

57 

58 def _call_llm_for_queries(self, objects: dict, queries: dict) -> dict: 

59 """Build the prompt and call the LLM. 

60 

61 Args: 

62 objects (dict): Compiled objects (for schema context). 

63 queries (dict): Raw parsed queries. 

64 

65 Returns: 

66 dict: Raw parsed JSON response from the LLM. 

67 """ 

68 schema_context = self.build_schema_context(objects) 

69 simplified = self.simplify_endpoints(queries) 

70 queries_json = json.dumps(simplified, indent=2) 

71 

72 user_prompt = QUERY_USER_PROMPT_TEMPLATE.format( 

73 schema_context=schema_context, 

74 queries_json=queries_json, 

75 ) 

76 return self.call_llm(QUERY_SYSTEM_PROMPT, user_prompt) 

77 

78 def _build_comparison(self, classic: dict, llm_merged: dict, query_names) -> dict: 

79 """Build a per-query comparison dict. 

80 

81 Args: 

82 classic (dict): Classic resolver output. 

83 llm_merged (dict): LLM resolver output (after merge). 

84 query_names: Iterable of query names. 

85 

86 Returns: 

87 dict: Keyed by query name; each entry has 'classic', 'llm', 'differs', 'diff'. 

88 """ 

89 comparison = {} 

90 for name in query_names: 

91 c = classic.get(name, {}) 

92 llm_entry = llm_merged.get(name, {}) 

93 

94 c_summary = { 

95 "hardDependsOn": c.get("hardDependsOn", {}), 

96 "softDependsOn": c.get("softDependsOn", {}), 

97 } 

98 l_summary = { 

99 "hardDependsOn": llm_entry.get("hardDependsOn", {}), 

100 "softDependsOn": llm_entry.get("softDependsOn", {}), 

101 } 

102 

103 diff = {} 

104 for key in ("hardDependsOn", "softDependsOn"): 

105 if c_summary[key] != l_summary[key]: 

106 diff[key] = {"classic": c_summary[key], "llm": l_summary[key]} 

107 

108 comparison[name] = { 

109 "classic": c_summary, 

110 "llm": l_summary, 

111 "differs": bool(diff), 

112 "diff": diff, 

113 } 

114 return comparison