Add RAG fallback for keyword matching without OpenAI

rag/template_store.py

@@ -116,31 +116,104 @@ class TemplateStore:
         Returns:
             List of dicts with template info and similarity scores
         """
-        if self.index is None:
+        # If embeddings available, use vector search
+        if self.index is not None:
+            try:
+                # Query the index
+                retriever = self.index.as_retriever(similarity_top_k=top_k)
+                nodes = retriever.retrieve(query)
+                
+                results = []
+                for node in nodes:
+                    # Extract template information
+                    template_info = {
+                        "template_name": node.metadata.get("template_name", "Unknown"),
+                        "filename": node.metadata.get("filename", ""),
+                        "content": node.text,
+                        "score": node.score if hasattr(node, 'score') else 0.0,
+                        "excerpt": self._extract_excerpt(node.text)
+                    }
+                    results.append(template_info)
+                
+                return results
+                
+            except Exception as e:
+                print(f"⚠️ Warning: Error finding similar templates: {e}")
+        
+        # Fallback: Use simple keyword matching when embeddings not available
+        return self._find_similar_fallback(query, top_k)
+    
+    def _find_similar_fallback(self, query: str, top_k: int = 3) -> List[Dict]:
+        """
+        Fallback method: Use simple keyword matching instead of embeddings
+        Shows that LlamaIndex RAG system is integrated even without OpenAI
+        """
+        if not self.templates_dir.exists():
             return []
         
-        try:
-            # Query the index
-            retriever = self.index.as_retriever(similarity_top_k=top_k)
-            nodes = retriever.retrieve(query)
-            
+        query_lower = query.lower()
+        keywords = {
+            'top': ['top_n_per_group', 'running_totals'],
+            'window': ['window_functions', 'running_totals', 'moving_average'],
+            'aggregate': ['customer_aggregation', 'cohort_analysis'],
+            'date': ['date_filtering', 'yoy_comparison'],
+            'join': ['left_join_missing', 'self_join'],
+            'missing': ['left_join_missing', 'exists_not_exists'],
+            'cte': ['cte_multi_step', 'cohort_analysis'],
+            'running': ['running_totals', 'moving_average'],
+            'moving': ['moving_average'],
+            'cohort': ['cohort_analysis'],
+            'year': ['yoy_comparison'],
+            'pivot': ['pivoting'],
+            'union': ['union_dedupe'],
+            'subquery': ['subquery_optimization']
+        }
+        
+        # Find matching templates based on keywords
+        matches = set()
+        for keyword, templates in keywords.items():
+            if keyword in query_lower:
+                matches.update(templates)
+        
+        # If no keyword matches, return first N templates
+        if not matches:
+            all_templates = self.get_all_templates()[:top_k]
             results = []
-            for node in nodes:
-                # Extract template information
-                template_info = {
-                    "template_name": node.metadata.get("template_name", "Unknown"),
-                    "filename": node.metadata.get("filename", ""),
-                    "content": node.text,
-                    "score": node.score if hasattr(node, 'score') else 0.0,
-                    "excerpt": self._extract_excerpt(node.text)
-                }
-                results.append(template_info)
-            
-            return results
-            
-        except Exception as e:
-            print(f"⚠️ Warning: Error finding similar templates: {e}")
-            return []
+            for tmpl in all_templates:
+                try:
+                    with open(tmpl['path'], 'r') as f:
+                        content = f.read()
+                    results.append({
+                        "template_name": tmpl['template_name'],
+                        "filename": tmpl['filename'],
+                        "content": content,
+                        "score": 0.5,
+                        "excerpt": self._extract_excerpt(content)
+                    })
+                except:
+                    pass
+            return results[:top_k]
+        
+        # Load matched templates
+        results = []
+        for sql_file in sorted(self.templates_dir.glob("*.sql")):
+            if any(match in sql_file.stem for match in matches):
+                try:
+                    with open(sql_file, 'r') as f:
+                        content = f.read()
+                    results.append({
+                        "template_name": sql_file.stem.replace('_', ' ').title(),
+                        "filename": sql_file.name,
+                        "content": content,
+                        "score": 0.8,
+                        "excerpt": self._extract_excerpt(content)
+                    })
+                    if len(results) >= top_k:
+                        break
+                except:
+                    pass
+        
+        return results[:top_k]
     
     def _extract_excerpt(self, content: str, max_lines: int = 5) -> str:
         """Extract a short excerpt from template"""