Use a Open-WebUI defined openai API URL in a pipeline?

[bartmch]

I have an existing OpenAI API Connection "http://model:8000/v1" which is the default chat model in my open-webui.
I would like to use this same "http://model:8000/v1" LLM inside the pipeline to e.g. do nl2sql. Maybe I missed it but I can only see ollama and other "externally defined" LLM's.
How can I use the open-webui defined OpenAI API URL and use it in my pipeline?

[2237717720]

me too

[procoprobocop]

You need to replace Ollama with Openal in the code. For this:

1. Remove from llama_index.llms.ollama import Ollama.
2. Import from llama_index.llms.openai import OpenAI.
3. Replace the Ollama object with OpenAI with the required parameters.
   Here is an example of the corrected code:

from typing import List, Union, Generator, Iterator
import os 
from pydantic import BaseModel
from llama_index.llms.openai import OpenAI
from llama_index.core.query_engine import NLSQLTableQueryEngine
from llama_index.core import SQLDatabase, PromptTemplate
from sqlalchemy import create_engine

class Pipeline:
    class Valves(BaseModel):
        DB_HOST: str
        DB_PORT: str
        DB_USER: str
        DB_PASSWORD: str        
        DB_DATABASE: str
        DB_TABLE: str
        OPENAI_API_BASE_URL: str
        OPENAI_API_KEY: str
        TEXT_TO_SQL_MODEL: str 

    def __init__(self):
        self.name = "Database RAG Pipeline"
        self.engine = None
        self.nlsql_response = ""

		# Initialize
        self.valves = self.Valves(
            **{
                "pipelines": ["*"],                                                          
                "DB_HOST": os.getenv("DB_HOST", "http://localhost"),                     
                "DB_PORT": os.getenv("DB_PORT", 5432),                                     
                "DB_USER": os.getenv("DB_USER", "postgres"),                                
                "DB_PASSWORD": os.getenv("DB_PASSWORD", "password"),                         
                "DB_DATABASE": os.getenv("DB_DATABASE", "postgres"),                         
                "DB_TABLE": os.getenv("DB_TABLE", "table_name"),                           
                "OPENAI_API_BASE_URL": os.getenv("OPENAI_API_BASE_URL", "https://api.openai.ai/v1"),
                "OPENAI_API_KEY": os.getenv("OPENAI_API_KEY", "secret_key"), 
                "TEXT_TO_SQL_MODEL": os.getenv("TEXT_TO_SQL_MODEL", "gpt-4")               
            }
        )

    def init_db_connection(self):
        self.engine = create_engine(
            f"postgresql+psycopg2://{self.valves.DB_USER}:{self.valves.DB_PASSWORD}@{self.valves.DB_HOST}:{self.valves.DB_PORT}/{self.valves.DB_DATABASE}"
        )
        return self.engine

    async def on_startup(self):
        self.init_db_connection()

    async def on_shutdown(self):
        pass

    def pipe(
        self, user_message: str, model_id: str, messages: List[dict], body: dict
    ) -> Union[str, Generator, Iterator]:

        sql_database = SQLDatabase(self.engine, include_tables=[self.valves.DB_TABLE])

        llm = OpenAI(
            model=self.valves.TEXT_TO_SQL_MODEL,
            api_key=self.valves.OPENAI_API_KEY,
            base_url=self.valves.OPENAI_API_BASE_URL,
            request_timeout=180.0,
            context_window=30000
        )

        text_to_sql_prompt = """
        Given an input question, first create a syntactically correct {dialect} query to run, then look at the results of the query and return the answer. 
        You can order the results by a relevant column to return the most interesting examples in the database.
        Unless the user specifies in the question a specific number of examples to obtain, query for at most 5 results using the LIMIT clause as per Postgres. You can order the results to return the most informative data in the database.
        Never query for all the columns from a specific table, only ask for a few relevant columns given the question.
        You should use DISTINCT statements and avoid returning duplicates wherever possible.
        Pay attention to use only the column names that you can see in the schema description. Be careful to not query for columns that do not exist. Pay attention to which column is in which table. Also, qualify column names with the table name when needed. You are required to use the following format, each taking one line:

        Question: Question here
        SQLQuery: SQL Query to run
        SQLResult: Result of the SQLQuery
        Answer: Final answer here

        Only use tables listed below.
        {schema}

        Question: {query_str}
        SQLQuery: 
        """

        text_to_sql_template = PromptTemplate(text_to_sql_prompt)

        query_engine = NLSQLTableQueryEngine(
            sql_database=sql_database, 
            tables=[self.valves.DB_TABLE],
            llm=llm, 
            embed_model="local", 
            text_to_sql_prompt=text_to_sql_template, 
            streaming=True
        )

        response = query_engine.query(user_message)

        return response.response_gen