Strategies for chunking code?

[frankandrobot]

• Because of licensing issues, currently limited to running models locally on my M2. Namely, Llama-13b 4-bit via the llama.cpp bindings
• However, this model is currently limited to a context size of around 1100 (tokens?)
• Unfortunately, when I chunk code files to fit in this window, I will often get back very limited views.
• For instance, below, I asked for a modal, and I got back just the imports!
• Any strategies? Currently using "stuff" but perhaps I can use "reduce" on an entire matching file?

Sample Query with Matching Chunks

query: What is an example of modal ussage?

There are 4 matching chunks

> Entering new  chain...
Prompt after formatting:
Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.

import React from "react";
import { Modal } from "@internal/modal";
import type { ModalProps } from "@internal/modal";
import useComplianceModal from "./useComplianceModal";
import NetworkComplianceModalContent from "./NetworkComplianceModalContent";
import SubscriptionComplianceModalContent from "./SubscriptionComplianceModalContent";
import { NetworkBindDetails } from "../../types";
import { useBindModal } from "../BindModal";

...

Notebook

from langchain.llms import LlamaCpp
from langchain import PromptTemplate, LLMChain
from langchain.callbacks.manager import CallbackManager
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler

def load_llama(model_path):
    # Callbacks support token-wise streaming
    callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
    # Verbose is required to pass to the callback manager

    n_gpu_layers = 1 # Change this value based on your model and your GPU VRAM pool.
    n_batch = 1000 # Should be between 1 and n_ctx, consider the amount of VRAM in your GPU.

    llm = LlamaCpp(
        model_path=model_path,
        n_gpu_layers=n_gpu_layers, n_batch=n_batch,
        callback_manager=callback_manager, 
        verbose=True,
        n_ctx=1100,
    )
    return llm

import locale
from git import Repo
import os
from langchain.document_loaders import GitLoader
import re
import time
from langchain.document_loaders import WebBaseLoader;

locale.getpreferredencoding = lambda: "UTF-8"

def load_repo_branch(repo_path, repo_url):
    if os.path.exists(repo_path):
        repo = Repo(repo_path)
    else:
        repo = Repo.clone_from(repo_url, to_path=repo_path)

    branch = repo.head.reference
    return branch

def load_repo_documents(repo_path, branch, file_filter):
    start_time = time.time()

    print("--- Loading documents ---")
    
    loader = GitLoader(repo_path=repo_path,
                       branch=branch,
                       file_filter=lambda file_path: re.match(file_filter, file_path))
    data = loader.load()

    print (f'--- {time.time() - start_time} seconds ---')
    print (f'There are {len(data)} files')
    return data

from langchain.text_splitter import RecursiveCharacterTextSplitter
import time

def chunk_documents(data, chunk_size, chunk_overlap):
    start_time = time.time()

    print("--- Chunking documents ---")

    text_splitter = RecursiveCharacterTextSplitter(
        chunk_size = chunk_size,
        chunk_overlap = chunk_overlap)
    chunks = text_splitter.split_documents(data)

    print (f'--- {time.time() - start_time} seconds ---')
    print (f'There are {len(chunks)} chunks')
    return chunks

from langchain.embeddings import HuggingFaceEmbeddings

def huggingface_embeddings(embedding_model_path):
    embeddings = HuggingFaceEmbeddings()
    return embeddings

from langchain.vectorstores import Chroma

def chroma_store(chunks, embeddings, persist_directory, collection_name):
    start_time = time.time()
    print("--- Adding chunks to vectorstore ---")

    vectorstore = Chroma.from_documents(
        documents=chunks,
        embedding=embeddings,
        persist_directory=persist_directory,
        collection_name=collection_name)
    
    print (f'--- {time.time() - start_time} seconds ---')

    return vectorstore

def load_documents_v1(repo_path, persist_path, collection_name, embeddings):
    """It is *not* (yet) possible to replace files with updated versions."""
    if os.path.exists(persist_path):
        vectorstore = Chroma(
            embedding_function=embeddings,
            persist_directory=persist_path,
            collection_name=collection_name)
        return vectorstore
    
    branch = load_repo_branch(repo_path, "none")

    documents = load_repo_documents(
        repo_path=repo_path,
        branch=branch,
        file_filter=".*(ts|tsx|md)$")
    
    chunks = chunk_documents(
        data=documents, 
        chunk_size=900, 
        chunk_overlap=100)
    
    vectorstore = chroma_store(chunks, embeddings, persist_path, collection_name)
    
    vectorstore.persist()

    return vectorstore

text_embeddings_path = "/Users/uavalos/Documents/llm/manage01"
repo_path = "/Users/uavalos/Documents/manage"

embeddings = huggingface_embeddings("/Users/uavalos/Documents/gtr-t5-xxl")

vectorstore = load_documents_v1(
    repo_path=repo_path,
    persist_path=text_embeddings_path,
    collection_name="manage",
    embeddings=embeddings)
#vectorstore = chroma_store(chunks, embeddings, text_embeddings_path)
#vectorstore = faiss_store(chunks, embeddings)

model_path = "/Users/uavalos/Downloads/ggml-vicuna-13b-1.1/ggml-vic13b-q4_0.bin";

llm = load_llama(model_path)

from langchain.chains.question_answering import load_qa_chain
from langchain import PromptTemplate
import time

def query(llm, vectorstore, query_text):
    docs = vectorstore.similarity_search(query_text)
    print (f'There are {len(docs)} matching chunks')
    
    sources = map(lambda doc: doc.metadata['source'], docs)
    sources = list(set(sources))

    for source in sources:
        print(source)
    
     chain = load_qa_chain(llm, chain_type="stuff", verbose=True)

    start_time = time.time()
    result = chain.run(input_documents=docs, question=query)
    print (f'--- {time.time() - start_time} seconds ---')
    
    return result

query(llm, vectorstore, "What is an example of modal component usage?")