Kotoba

import os, sys, json, re import pandas as pd import langchain as lc import camelot import pandasai import markdown from bs4 import BeautifulSoup

1. import pdftotree $ with tensorflow

import kotoba.chatbot_utils as c_t import importlib from pandasai.llm import BedrockClaude from pandasai.llm import LLM from pandasai.prompts import BasePrompt from langchain import PromptTemplate from langchain.chains import LLMChain

modL = ["gpt-4o@openai","gpt-4-turbo@openai","gpt-3.5-turbo@openai","mixtral-8x7b-instruct-v0.1@aws-bedrock","llama-2-70b-chat@aws-bedrock","codellama-34b-instruct@together-ai","gemma-7b-it@fireworks-ai","claude-3-haiku@anthropic","claude-3-opus@anthropic","claude-3-sonnet@anthropic","mistral-7b-instruct-v0.1@fireworks-ai","mistral-7b-instruct-v0.2@fireworks-ai"] os.environ['OPENAI_MODEL_NAME'] = modL[0] system_message = "You are a Data Analyst and pandas expert. Your goal is to help people generate high quality and robust code." model_params = {"do_sample": True,"top_p": 0.9,"top_k": 40,"temperature": 0.1,"max_new_tokens": 1024,"repetition_penalty": 1.03,"stop": [""]}

def html2df(fName,llm):

   with open(fName) as fByte:
       html_text = fByte.read()
   soup = BeautifulSoup(html_text, 'html.parser')
   tableL = soup.find_all('table')
   tableS = "".join([str(t) for t in tableL])
   tabDf = pd.read_html(tableS)
   for tab in tableL:
       t = str(tab)
       if re.search("flexibility gradually",t):
           tabD  = pd.read_html(t, header=[0,1])[0]
           break

   agent = pandasai.Agent(tabD, config={"llm": llm})
   df = pandasai.SmartDataframe(tabD, config={"llm": llm})
   return df

def md2df(text,llm):

   lines = text.split("\n")
   header = lines[0].strip("|").split("|")
   data = [] 
   for line in lines[2:]:
       if not line.strip():
           break
       
       cols = line.strip("|").split("|")
       row = dict(zip(header, cols))
       data.append(row)
   df = pd.DataFrame(data)
   sdf = pandasai.SmartDataframe(df, config={"llm": llm})
   return sdf

def get_local_llm():

   from pandasai.llm import HuggingFaceTextGen
   llm = HuggingFaceTextGen(inference_server_url="http://127.0.0.1:8080")
   return llm
   

def get_bedrock():

   bedrock_runtime_client = boto3.client('bedrock-runtime')
   llm = BedrockClaude(bedrock_runtime_client)
   return llm

def numeric_qa(question,dataframe,model=llm,qa_prompt=numeric_qa_prompt,to_html=False):

   """
   A function that passes a prompt, question and table to the LLM.
   There's an option of converting a data frame to HTML.
   """
   if to_html:
       dataframe = dataframe.to_html()
   prompt_qa = PromptTemplate(template=qa_prompt, input_variables=["text", "table"])
   llm_chain = LLMChain(prompt=prompt_qa, llm=model)
   llm_reply = llm_chain.predict(text = question, table = dataframe)
   return print(llm_reply)

if False:

   import seaborn as sns
   iris = sns.load_dataset('iris')
   iris.head()
   agent = pandasai.Agent(iris, config={"llm": llm})
   resp = agent.chat('Which is the most common specie?')
   sales_by_country = pd.DataFrame({
       "country": ["United States", "United Kingdom", "France", "Germany", "Italy", "Spain", "Canada", "Australia", "Japan", "China"],
       "sales": [5000, 3200, 2900, 4100, 2300, 2100, 2500, 2600, 4500, 7000]
   })
   agent = pandasai.Agent(sales_by_country, config={"llm": llm})
   resp = agent.chat('Which are the top 5 countries by sales?')