Update app.py

app.py

@@ -90,7 +90,18 @@ An asynchronous FastAPI microservice that optimizes input prompts using the Goog
 @app.get("/agents.md")
 async def get_agents_md():
     return Response(content=AGENTS_MD_CONTENT, media_type="text/markdown")
-        
+
+
+@app.get("/robots.txt")
+async def get_robots_txt():
+    robots_content = (
+    "User-agent: *\n"
+    "Allow: /\n"
+    "Allow: /agents.md\n"
+    "Disallow: /optimize\n"
+    )
+    return Response(content=robots_content, media_type="text/plain")
+
 
 @app.get("/", response_class=HTMLResponse)
 async def read_items():