From b35afe1a514af76494c0bb46ee71918bb1d9dad8 Mon Sep 17 00:00:00 2001 From: jwatson Date: Tue, 12 Nov 2024 15:12:51 -0800 Subject: [PATCH 1/5] "add the caii env vars back" --- .project-metadata.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.project-metadata.yaml b/.project-metadata.yaml index c05a2ae41..2342ec7af 100644 --- a/.project-metadata.yaml +++ b/.project-metadata.yaml @@ -26,6 +26,18 @@ environment_variables: default: "" description: "AWS Secret Access Key" required: true + CAII_DOMAIN: + default: "" + description: "The domain of the CAII service. Setting this will enable CAII as the sole source for both inference and embedding models." + required: false + CAII_INFERENCE_ENDPOINT_NAME: + default: "" + description: "The name of the inference endpoint for the CAII service. Required if CAII_DOMAIN is set." + required: false + CAII_EMBEDDING_ENDPOINT_NAME: + default: "" + description: "The name of the embedding endpoint for the CAII service. Required if CAII_DOMAIN is set." + required: false DB_URL: default: "jdbc:h2:file:~/databases/rag" description: "Internal DB URL. Do not change." From 59810e7c7213fede4b1589c5ad4e0436ff073b08 Mon Sep 17 00:00:00 2001 From: jwatson Date: Tue, 12 Nov 2024 15:21:28 -0800 Subject: [PATCH 2/5] only release branch creates official releases --- .github/workflows/publish_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index a6a7bd240..9f04da59c 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -79,7 +79,7 @@ jobs: id: create_release with: draft: false - prerelease: ${{ github.event.inputs.BRANCH == 'mob/main' }} + prerelease: ${{ github.event.inputs.BRANCH != 'release/1' }} name: ${{ github.event.inputs.VERSION }} tag_name: ${{ github.event.inputs.VERSION }} files: | From c798e5af7c6dcdcae14d3a546bd92f182bdf0df8 Mon Sep 17 00:00:00 2001 From: jwatson Date: Tue, 12 Nov 2024 15:41:27 -0800 Subject: [PATCH 3/5] handle 404 on an endpoint name, notes on failures modes --- llm-service/app/services/caii.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/llm-service/app/services/caii.py b/llm-service/app/services/caii.py index 0cfd07ec0..e0a8244a3 100644 --- a/llm-service/app/services/caii.py +++ b/llm-service/app/services/caii.py @@ -39,6 +39,7 @@ import json import os +from fastapi import HTTPException from llama_index.core.base.embeddings.base import BaseEmbedding from llama_index.core.llms import LLM @@ -60,6 +61,9 @@ def describe_endpoint(domain: str, endpoint_name: str): } desc = requests.post(describe_url, headers=headers, json=desc_json) + if desc.status_code == 404: + raise HTTPException(status_code=404, detail = f"Endpoint '{endpoint_name}' not found") + print(desc.content) content = json.loads(desc.content) return content @@ -92,14 +96,20 @@ def get_embedding_model() -> BaseEmbedding: endpoint = describe_endpoint(domain=domain, endpoint_name=endpoint_name) return CaiiEmbeddingModel(endpoint=endpoint) +### metadata methods below here + def get_caii_llm_models(): domain = os.environ['CAII_DOMAIN'] endpoint_name = os.environ['CAII_INFERENCE_ENDPOINT_NAME'] models = describe_endpoint(domain=domain, endpoint_name=endpoint_name) - return [{ "model_id": models["name"], "name": models["name"] }] + return [{ "model_id": models["name"], "name": models["name"], "replica_count": models["replica_count"] }] def get_caii_embedding_models(): + # notes: + # NameResolutionError is we can't contact the CAII_DOMAIN + # HTTPException (404) is we can't find the endpoint by name + domain = os.environ['CAII_DOMAIN'] endpoint_name = os.environ['CAII_EMBEDDING_ENDPOINT_NAME'] models = describe_endpoint(domain=domain, endpoint_name=endpoint_name) - return [{ "model_id": models["name"], "name": models["name"] }] + return [{ "model_id": models["name"], "name": models["name"], "replica_count": models["replica_count"] }] From 4cccfc5904859057fb32a90f92d9846960258343 Mon Sep 17 00:00:00 2001 From: Elijah Williams Date: Tue, 12 Nov 2024 17:14:16 -0700 Subject: [PATCH 4/5] wip on new endpoint for status --- .../app/routers/index/models/__init__.py | 7 ++- llm-service/app/services/caii.py | 8 ++- llm-service/app/services/models.py | 9 ++++ ui/src/api/modelsApi.ts | 17 +++++- ui/src/api/utils.ts | 1 + ui/src/layout/Sidebar.tsx | 24 ++++++--- ui/src/routeTree.gen.ts | 2 +- ui/src/routes/_layout.lazy.tsx | 54 +++++++++++++++++++ ui/src/routes/_layout.tsx | 17 ++---- ui/src/routes/index.tsx | 2 +- 10 files changed, 115 insertions(+), 26 deletions(-) create mode 100644 ui/src/routes/_layout.lazy.tsx diff --git a/llm-service/app/routers/index/models/__init__.py b/llm-service/app/routers/index/models/__init__.py index 8eb5c7594..db9fadd5b 100644 --- a/llm-service/app/routers/index/models/__init__.py +++ b/llm-service/app/routers/index/models/__init__.py @@ -37,7 +37,7 @@ # from fastapi import APIRouter from .... import exceptions -from ....services.models import get_available_embedding_models, get_available_llm_models +from ....services.models import get_available_embedding_models, get_available_llm_models, get_model_source, ModelSource router = APIRouter(prefix="/models") @@ -50,3 +50,8 @@ def get_llm_models() -> list: @exceptions.propagates def get_llm_embedding_models() -> list: return get_available_embedding_models() + +@router.get("/model_source", summary="Model source enabled - Bedrock or CAII") +@exceptions.propagates +def get_model() -> ModelSource: + return get_model_source() \ No newline at end of file diff --git a/llm-service/app/services/caii.py b/llm-service/app/services/caii.py index e0a8244a3..c0b226649 100644 --- a/llm-service/app/services/caii.py +++ b/llm-service/app/services/caii.py @@ -102,7 +102,7 @@ def get_caii_llm_models(): domain = os.environ['CAII_DOMAIN'] endpoint_name = os.environ['CAII_INFERENCE_ENDPOINT_NAME'] models = describe_endpoint(domain=domain, endpoint_name=endpoint_name) - return [{ "model_id": models["name"], "name": models["name"], "replica_count": models["replica_count"] }] + return build_model_response(models) def get_caii_embedding_models(): # notes: @@ -112,4 +112,8 @@ def get_caii_embedding_models(): domain = os.environ['CAII_DOMAIN'] endpoint_name = os.environ['CAII_EMBEDDING_ENDPOINT_NAME'] models = describe_endpoint(domain=domain, endpoint_name=endpoint_name) - return [{ "model_id": models["name"], "name": models["name"], "replica_count": models["replica_count"] }] + return build_model_response(models) + +def build_model_response(models): + return [{"model_id": models["name"], "name": models["name"], "available": models['replica_count'] > 0, + "replica_count": models["replica_count"]}] diff --git a/llm-service/app/services/models.py b/llm-service/app/services/models.py index e06ab3495..0de2ab5d8 100644 --- a/llm-service/app/services/models.py +++ b/llm-service/app/services/models.py @@ -36,6 +36,7 @@ # DATA. # import os +from enum import Enum from llama_index.core.base.embeddings.base import BaseEmbedding from llama_index.core.llms import LLM @@ -97,3 +98,11 @@ def _get_bedrock_embedding_models(): "name": "cohere.embed-english-v3", }] +class ModelSource(str, Enum): + BEDROCK = "Bedrock" + CAII = "CAII" + +def get_model_source() -> ModelSource: + if "CAII_DOMAIN" in os.environ: + return ModelSource.CAII + return ModelSource.BEDROCK diff --git a/ui/src/api/modelsApi.ts b/ui/src/api/modelsApi.ts index 5e4546bf4..b0374f667 100644 --- a/ui/src/api/modelsApi.ts +++ b/ui/src/api/modelsApi.ts @@ -35,12 +35,14 @@ * BUSINESS ADVANTAGE OR UNAVAILABILITY, OR LOSS OR CORRUPTION OF * DATA. ******************************************************************************/ -import { useQuery } from "@tanstack/react-query"; +import { queryOptions, useQuery } from "@tanstack/react-query"; import { getRequest, llmServicePath, QueryKeys } from "src/api/utils.ts"; export interface Model { name: string; model_id: string; + available?: boolean; + replica_count?: number; } export const useGetLlmModels = () => { @@ -68,3 +70,16 @@ export const useGetEmbeddingModels = () => { const getEmbeddingModels = async (): Promise => { return await getRequest(`${llmServicePath}/index/models/embeddings`); }; + +type ModelSource = "CAII" | "Bedrock"; + +export const getModelSourceQueryOptions = queryOptions({ + queryKey: [QueryKeys.getModelSource], + queryFn: async () => { + return await getModelSource(); + }, +}); + +const getModelSource = async (): Promise => { + return await getRequest(`${llmServicePath}/index/models/model_source`); +}; diff --git a/ui/src/api/utils.ts b/ui/src/api/utils.ts index dee75a9b3..02f735139 100644 --- a/ui/src/api/utils.ts +++ b/ui/src/api/utils.ts @@ -80,6 +80,7 @@ export enum QueryKeys { "getAmpUpdateJobStatus" = "getAmpUpdateJobStatus", "getLlmModels" = "getLlmModels", "getEmbeddingModels" = "getEmbeddingModels", + "getModelSource" = "getModelSource", } export const commonHeaders = { diff --git a/ui/src/layout/Sidebar.tsx b/ui/src/layout/Sidebar.tsx index 3c8665868..a90dfac3d 100644 --- a/ui/src/layout/Sidebar.tsx +++ b/ui/src/layout/Sidebar.tsx @@ -92,7 +92,7 @@ const Sidebar: React.FC = () => { feedbackModal.setIsModalOpen(true); }; - const items: MenuItem[] = [ + const baseItems: MenuItem[] = [ { label: ( { navToData, , ), - getItem( -
Leave Feedback
, - "leave-feedback", - popupFeedback, - , - ), ]; + // const caiiModels = getItem( + //
CAII Models
, + // "caii-models", + // navToData, + // , + // ) + + const feedbackItem = getItem( +
Leave Feedback
, + "leave-feedback", + popupFeedback, + , + ); + + const items = [...baseItems, feedbackItem]; + function chooseRoute() { if (matchRoute({ to: "/data", fuzzy: true })) { return ["data"]; diff --git a/ui/src/routeTree.gen.ts b/ui/src/routeTree.gen.ts index d4847c3d2..f0cea16e0 100644 --- a/ui/src/routeTree.gen.ts +++ b/ui/src/routeTree.gen.ts @@ -30,7 +30,7 @@ const LayoutDataImport = createFileRoute('/_layout/data')() const LayoutRoute = LayoutImport.update({ id: '/_layout', getParentRoute: () => rootRoute, -} as any) +} as any).lazy(() => import('./routes/_layout.lazy').then((d) => d.Route)) const IndexRoute = IndexImport.update({ id: '/', diff --git a/ui/src/routes/_layout.lazy.tsx b/ui/src/routes/_layout.lazy.tsx new file mode 100644 index 000000000..148673838 --- /dev/null +++ b/ui/src/routes/_layout.lazy.tsx @@ -0,0 +1,54 @@ +/******************************************************************************* + * CLOUDERA APPLIED MACHINE LEARNING PROTOTYPE (AMP) + * (C) Cloudera, Inc. 2024 + * All rights reserved. + * + * Applicable Open Source License: Apache 2.0 + * + * NOTE: Cloudera open source products are modular software products + * made up of hundreds of individual components, each of which was + * individually copyrighted. Each Cloudera open source product is a + * collective work under U.S. Copyright Law. Your license to use the + * collective work is as provided in your written agreement with + * Cloudera. Used apart from the collective work, this file is + * licensed for your use pursuant to the open source license + * identified above. + * + * This code is provided to you pursuant a written agreement with + * (i) Cloudera, Inc. or (ii) a third-party authorized to distribute + * this code. If you do not have a written agreement with Cloudera nor + * with an authorized and properly licensed third party, you do not + * have any rights to access nor to use this code. + * + * Absent a written agreement with Cloudera, Inc. ("Cloudera") to the + * contrary, A) CLOUDERA PROVIDES THIS CODE TO YOU WITHOUT WARRANTIES OF ANY + * KIND; (B) CLOUDERA DISCLAIMS ANY AND ALL EXPRESS AND IMPLIED + * WARRANTIES WITH RESPECT TO THIS CODE, INCLUDING BUT NOT LIMITED TO + * IMPLIED WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE; (C) CLOUDERA IS NOT LIABLE TO YOU, + * AND WILL NOT DEFEND, INDEMNIFY, NOR HOLD YOU HARMLESS FOR ANY CLAIMS + * ARISING FROM OR RELATED TO THE CODE; AND (D)WITH RESPECT TO YOUR EXERCISE + * OF ANY RIGHTS GRANTED TO YOU FOR THE CODE, CLOUDERA IS NOT LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, PUNITIVE OR + * CONSEQUENTIAL DAMAGES INCLUDING, BUT NOT LIMITED TO, DAMAGES + * RELATED TO LOST REVENUE, LOST PROFITS, LOSS OF INCOME, LOSS OF + * BUSINESS ADVANTAGE OR UNAVAILABILITY, OR LOSS OR CORRUPTION OF + * DATA. + ******************************************************************************/ + +import { createLazyFileRoute, Outlet } from "@tanstack/react-router"; +import { Layout } from "antd"; +import Sidebar from "src/layout/Sidebar.tsx"; + +const { Content } = Layout; + +export const Route = createLazyFileRoute("/_layout")({ + component: () => ( + + + + + + + ), +}); diff --git a/ui/src/routes/_layout.tsx b/ui/src/routes/_layout.tsx index f905b5aa4..3466f3398 100644 --- a/ui/src/routes/_layout.tsx +++ b/ui/src/routes/_layout.tsx @@ -36,19 +36,10 @@ * DATA. ******************************************************************************/ -import { createFileRoute, Outlet } from "@tanstack/react-router"; -import { Layout } from "antd"; -import Sidebar from "src/layout/Sidebar.tsx"; - -const { Content } = Layout; +import { createFileRoute } from "@tanstack/react-router"; +import { getModelSourceQueryOptions } from "src/api/modelsApi.ts"; export const Route = createFileRoute("/_layout")({ - component: () => ( - - - - - - - ), + loader: async ({ context: { queryClient } }) => + queryClient.ensureQueryData(getModelSourceQueryOptions), }); diff --git a/ui/src/routes/index.tsx b/ui/src/routes/index.tsx index 666ae3054..1228641ac 100644 --- a/ui/src/routes/index.tsx +++ b/ui/src/routes/index.tsx @@ -41,5 +41,5 @@ import { getDataSourcesQueryOptions } from "src/api/dataSourceApi.ts"; export const Route = createFileRoute("/")({ loader: async ({ context: { queryClient } }) => - await queryClient.ensureQueryData(getDataSourcesQueryOptions), + queryClient.ensureQueryData(getDataSourcesQueryOptions), }); From 5f34252ddd1c4e3df27ac8ca519bcf6c1627096a Mon Sep 17 00:00:00 2001 From: Elijah Williams Date: Tue, 12 Nov 2024 17:15:32 -0700 Subject: [PATCH 5/5] await query --- ui/src/routes/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/routes/index.tsx b/ui/src/routes/index.tsx index 1228641ac..666ae3054 100644 --- a/ui/src/routes/index.tsx +++ b/ui/src/routes/index.tsx @@ -41,5 +41,5 @@ import { getDataSourcesQueryOptions } from "src/api/dataSourceApi.ts"; export const Route = createFileRoute("/")({ loader: async ({ context: { queryClient } }) => - queryClient.ensureQueryData(getDataSourcesQueryOptions), + await queryClient.ensureQueryData(getDataSourcesQueryOptions), });