maggot fixes, improvements, os_command expression, and more

Author: Xpitfire

symai/backend/engine_embedding.py

@@ -17,6 +17,7 @@ def __init__(self):
         openai.api_key          = config['EMBEDDING_ENGINE_API_KEY']
         self.model              = config['EMBEDDING_ENGINE_MODEL']
         self.pricing            = self.api_pricing()
+        self.max_tokens         = self.api_max_tokens()
 
     def command(self, wrp_params):
         super().command(wrp_params)

symai/backend/engine_file.py

@@ -10,6 +10,47 @@ class FileEngine(Engine):
     def __init__(self):
         super().__init__()
 
+    def reset_eof_of_pdf_return_stream(self, pdf_stream_in: list):
+        actual_line = len(pdf_stream_in)  # Predefined value in case EOF not found
+        # find the line position of the EOF
+        for i, x in enumerate(pdf_stream_in[::-1]):
+            if b'%%EOF' in x:
+                actual_line = len(pdf_stream_in)-i
+                print(f'EOF found at line position {-i} = actual {actual_line}, with value {x}')
+                break
+
+        # return the list up to that point
+        return pdf_stream_in[:actual_line]
+
+    def fix_pdf(self, file_path: str):
+        # opens the file for reading
+        with open(file_path, 'rb') as p:
+            txt = (p.readlines())
+
+        # get the new list terminating correctly
+        txtx = self.reset_eof_of_pdf_return_stream(txt)
+
+        # write to new pdf
+        new_file_path = f'{file_path}_fixed.pdf'
+        with open(new_file_path, 'wb') as f:
+            f.writelines(txtx)
+
+        fixed_pdf = PyPDF2.PdfReader(new_file_path)
+        return fixed_pdf
+
+    def read_text(self, pdf_reader, range_):
+        txt = ''
+        n_pages = len(pdf_reader.pages)
+        if range_ is None:
+            for i in range(n_pages):
+                page = pdf_reader.pages[i]
+                txt += page.extract_text()
+        else:
+            for i in range(n_pages)[range_]:
+                page = pdf_reader.pages[i]
+                txt += page.extract_text()
+        return txt
+
     def forward(self, *args, **kwargs) -> List[str]:
         path          = kwargs['prompt']
         input_handler = kwargs['input_handler'] if 'input_handler' in kwargs else None
@@ -22,22 +63,30 @@ def forward(self, *args, **kwargs) -> List[str]:
             if isinstance(range_, tuple) or isinstance(range_, list):
                 range_ = slice(*range_)
 
-        if 'pdf' in path:
+        if '.pdf' in path:
             rsp = ''
-            with open(str(path), 'rb') as f:
-                # creating a pdf reader object
-                pdf_reader = PyPDF2.PdfReader(f)
-                n_pages = len(pdf_reader.pages)
-                if range_ is None:
-                    for i in range(n_pages):
-                        page = pdf_reader.pages[i]
-                        rsp += page.extract_text()
-                else:
-                    for i in range(n_pages)[range_]:
-                        page = pdf_reader.pages[i]
-                        rsp += page.extract_text()
+            try:
+                with open(str(path), 'rb') as f:
+                    # creating a pdf reader object
+                    pdf_reader = PyPDF2.PdfReader(f)
+                    rsp = self.read_text(pdf_reader, range_)
+            except Exception as e:
+                print(f'Error reading PDF: {e} | {path}')
+                if 'fix_pdf' not in kwargs or not kwargs['fix_pdf']:
+                    raise e
+                fixed_pdf = self.fix_pdf(str(path))
+                pdf_reader_fixed = PyPDF2.PdfReader(fixed_pdf)
+                rsp = self.read_text(pdf_reader_fixed, range_)
         else:
-            rsp = unpack.from_file(str(path))['content']
+            try:
+                file_ = unpack.from_file(str(path))
+                if 'content' in file_:
+                    rsp = file_['content']
+                else:
+                    rsp = str(file_)
+            except Exception as e:
+                print(f'Error reading file: {e} | {path}')
+                raise e
 
         output_handler = kwargs['output_handler'] if 'output_handler' in kwargs else None
         if output_handler:

symai/backend/engine_gptX_chat.py

@@ -32,7 +32,7 @@ def compute_remaining_tokens(self, prompts: dict) -> int:
         # iterate over prompts and compute number of tokens
         prompts_ = [role['content'] for role in prompts]
         prompt = ''.join(prompts_)
-        val = len(self.tokenizer.encode(prompt))
+        val = len(self.tokenizer.encode(prompt, disallowed_special=()))
         return int((self.max_tokens - val) * 0.98)
 
     def forward(self, prompts: List[str], *args, **kwargs) -> List[str]:
@@ -55,14 +55,14 @@ def forward(self, prompts: List[str], *args, **kwargs) -> List[str]:
 
         try:
             res = openai.ChatCompletion.create(model=model,
-                                                messages=prompts_,
-                                                max_tokens=max_tokens,
-                                                temperature=temperature,
-                                                frequency_penalty=frequency_penalty,
-                                                presence_penalty=presence_penalty,
-                                                top_p=top_p,
-                                                stop=stop,
-                                                n=1)
+                                               messages=prompts_,
+                                               max_tokens=max_tokens,
+                                               temperature=temperature,
+                                               frequency_penalty=frequency_penalty,
+                                               presence_penalty=presence_penalty,
+                                               top_p=top_p,
+                                               stop=stop,
+                                               n=1)
             output_handler = kwargs['output_handler'] if 'output_handler' in kwargs else None
             if output_handler:
                 output_handler(res)

symai/backend/engine_gptX_completion.py

@@ -31,7 +31,7 @@ def command(self, wrp_params):
     def compute_remaining_tokens(self, prompts: list) -> int:
         # iterate over prompts and compute number of tokens
         prompt = prompts[0]
-        val = len(self.tokenizer.encode(prompt))
+        val = len(self.tokenizer.encode(prompt, disallowed_special=()))
         return int((self.max_tokens - val) * 0.98)
 
     def forward(self, prompts: List[str], *args, **kwargs) -> List[str]:

symai/backend/mixin/openai.py

@@ -56,3 +56,6 @@ def api_max_tokens(self):
              self.model == 'babbage' or \
              self.model == 'ada':
             return 2_049
+
+        elif self.model == 'text-embedding-ada-002':
+            return 8_191

symai/extended/__init__.py

@@ -1,11 +1,11 @@
+from .arxiv_pdf_parser import *
+from .conversation import *
 from .crawler import *
 from .document import *
+from .file_merger import *
 from .graph import *
 from .html_style_template import *
 from .packages import *
-from .file_merger import *
 from .repo_cloner import *
 from .solver import *
 from .summarizer import *
-from .conversation import *
-from .arxiv_pdf_parser import *

symai/extended/arxiv_pdf_parser.py

@@ -15,7 +15,7 @@ def __init__(self, url_pattern: str = r'https://arxiv.org/(?:pdf|abs)/(\d+.\d+)(
         self.url_pattern = url_pattern
         self.merger = FileMerger()
 
-    def forward(self, data: Symbol) -> Symbol:
+    def forward(self, data: Symbol, **kwargs) -> Symbol:
         # Extract all urls from the data
         urls = re.findall(self.url_pattern, str(data))
 
@@ -39,7 +39,7 @@ def forward(self, data: Symbol) -> Symbol:
                     print('%r generated an exception: %s' % (url, exc))
 
         # Merge all pdfs into one file
-        merged_file = self.merger(output_path)
+        merged_file = self.merger(output_path, **kwargs)
 
         # Return the merged file as a Symbol
         return_file = self._to_symbol(merged_file)

symai/extended/document.py

@@ -4,12 +4,12 @@
 
 
 class DocumentRetriever(Expression):
-    def __init__(self, file_path: str, index_name: str = Indexer.DEFAULT):
+    def __init__(self, file_path: str, index_name: str = Indexer.DEFAULT, **kwargs):
         super().__init__()
         reader = FileReader()
         indexer = Indexer(index_name=index_name)
-        text = reader(file_path)
-        self.index = indexer(text)
+        text = reader(file_path, **kwargs)
+        self.index = indexer(text, **kwargs)
 
     def forward(self, query: Optional[Symbol]) -> Symbol:
         return self.index(query)

symai/extended/file_merger.py

@@ -16,7 +16,7 @@ def __init__(self, file_endings: List[str] = ['.py', '.md', '.txt', '.sh', '.pdf
         self.file_excludes = file_excludes
         self.reader = FileReader()
 
-    def forward(self, root_path: str) -> Symbol:
+    def forward(self, root_path: str, **kwargs) -> Symbol:
         """
         Method to find, read, merge and return contents of files in the form of a Symbol starting from the root_path.
 
@@ -37,7 +37,7 @@ def forward(self, root_path: str) -> Symbol:
                 # Look only for files with the specified endings
                 if file.endswith(tuple(self.file_endings)):
                     # Read in the file using the FileReader
-                    file_content = self.reader(file_path).value
+                    file_content = self.reader(file_path, **kwargs).value
 
                     # Append start and end markers for each file
                     file_content = f"# ----[FILE_START] {file_path}\n" + \

symai/extended/graph.py

@@ -26,11 +26,12 @@ class Graph(Expression):
     def static_context(self) -> str:
         return GRAPH_DESCRIPTION
 
-    def __init__(self, formatter: Callable = SentenceFormatter(), n_workers: int = 4):
+    def __init__(self, formatter: Callable = SentenceFormatter(), n_workers: int = 1, verbose: bool = False):
         super().__init__()
-        self.formatter = formatter
-        self.n_workers = n_workers
+        self.formatter       = formatter
+        self.n_workers       = n_workers
         self.sym_return_type = Graph
+        self.verbose         = verbose
 
     def process_symbol(self, s, *args, **kwargs):
         res = ''
@@ -49,6 +50,7 @@ def _func(_, text) -> str:
             pass
 
         if len(str(s)) > 0:
+            if self.verbose: print(s)
             r = _func(self, s)
             rec = str(r)
             lines = rec.split('\n')
@@ -57,18 +59,27 @@ def _func(_, text) -> str:
                 if len(l) > 0:
                     csv = l.split(',')
                     try:
-                        if len(csv) == 3 and csv[0].strip() != '' and csv[1].strip() != '' and csv[2].strip() > 0:
-                            test_ = int(csv[-1])
+                        if len(csv) == 3 and \
+                            csv[0].strip() != '' and \
+                                csv[1].strip() != '' and \
+                                    int(csv[2].strip()) > 0:
                             res += l + '\n'
-                    except:
+                    except Exception as e:
+                        if self.verbose: print(e)
                         pass
         return res
 
     def forward(self, sym: Symbol, **kwargs) -> Symbol:
         res = 'source,target,value\n'
         sym_list = self.formatter(sym).value
+        if self.n_workers == 1:
+            for s in sym_list:
+                res += self.process_symbol(s)
+            return res
         with Pool(self.n_workers) as p:
             results = p.map(self.process_symbol, sym_list)
         for r in results:
             res += r
         return res
+
+

symai/extended/os_command.py

@@ -0,0 +1,101 @@
+import platform
+import subprocess
+from typing import Callable, Dict, Iterator, List, Optional, Type
+
+from ..components import Function
+from ..post_processors import CodeExtractPostProcessor
+from ..symbol import Expression, Symbol
+
+Context = """[DESCRIPTION]:
+Adapt the user query to an OS patform command (commands must be executable in terminal, shell, bash or powershell)!
+Create only command adaptations based on programs that are specified in the programs list or are native platform commands!
+
+[PROGRAM_LIST]:
+
+{programs}
+For other command requests, reply sorry not supported.
+
+[PLATFORM]:
+
+The currently detected OS platform is:
+{platform}
+ONLY CREATE A COMMAND FOR THE CURRENT PLATFORM!
+
+[USER_QUERY]:
+
+{query}
+
+[METADATA]:
+
+Metadata is OPTIONAL and can help with the specificity of the command.
+{metadata}
+
+---------------------------
+
+[EXAMPLES]:
+
+If the current platform is Windows, and the user query is: "open file.txt", then the command should be:
+```powershell
+notepad file.txt
+```
+If the current platform is Linux, and the user query is: "open file.txt", then the command should be:
+```shell
+gedit file.txt
+```
+If the current platform is Mac, and the user query is: "open file.txt", then the command should be:
+```bash
+open file.txt
+```
+If the current platform is Windows, and the user query requires to open Spotify and play Taylor Swift, and Spotify is in the programs list, then the command could look like:
+```powershell
+Start-Process 'spotify:track:Anti-Hero%20by%20Taylor%20Swift'
+```
+If the current platform is Windows, and the user query requires to open Spotify and play a song, and Spotify is in the programs list, and metadata is added, then the command could look like:
+```powershell
+Start-Process 'spotify:track:https://open.spotify.com/track/0V3wPSX9ygBnCm8psDIegu?si=81646e6079d34526'
+```
+
+---------------------------
+
+Write an executable command that starts a process according to the user query, platform and the programs list. The commnad should be one line and should be direcly executable in terminal, shell, bash or powershell.
+"""
+
+
+class OSCommand(Expression):
+    def __init__(self, programs:        List[str],
+                       metadata:   Dict[str, str] = {},
+                       verbose:              bool = False,
+                       os_platform:           str = 'auto'):
+        super().__init__()
+        self.verbose:            bool = verbose
+        self.os_platform:         str = os_platform
+        self.programs:      List[str] = programs
+        self.metadata: Dict[str, str] = metadata
+
+        if self.os_platform == 'auto':
+            self.os_platform = platform.platform()
+        if len(programs) == 0:
+            raise Exception('No programs specified!')
+
+    def execute_os_command(self, *args, **kwargs):
+        command = args[0]
+        print(f'Executing {self.os_platform} command: {command}')
+        if 'linux' in self.os_platform.lower():
+            return [subprocess.run(["bash", "-c", str(command)])]
+        elif 'windows' in self.os_platform.lower():
+            return [subprocess.run(["powershell", "-Command", str(command)])]
+        elif 'mac' in self.os_platform.lower():
+            return [subprocess.run(["bash", "-c", str(command)])]
+        else:
+            raise Exception('Unsupported platform!')
+
+    def forward(self, sym: Symbol, **kwargs) -> Expression:
+        sym = self._to_symbol(sym)
+        kwargs['verbose'] = self.verbose
+
+        prompt = Context.format(programs=self.programs,
+                                platform=self.os_platform,
+                                query=sym,
+                                metadata=self.metadata)
+        command = sym.query(prompt, post_processors=[CodeExtractPostProcessor()], **kwargs)
+        return self.sym_return_type(self.output(command, expr=self.execute_os_command))

symai/extended/repo_cloner.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 from typing import Optional
 
-# from git import Repo
+from git import Repo
 
 from .. import Expression