Kotoba

import re import nltk from nltk.corpus import stopwords from nltk.stem import PorterStemmer, WordNetLemmatizer

1. Initialize stemmer and lemmatizer

stemmer = PorterStemmer() lemmatizer = WordNetLemmatizer() STOPWORDS = set(stopwords.words('english'))

nltk.download('stopwords') nltk.download('wordnet')

def clean_text(text):

   # Original Text

# Example: "This is a Testing @username https://example.com

Paragraphs!

#happy :)"

   text = text.lower()  # Convert all characters in text to lowercase

# Example after this step: "i won't go there! this is a testing @username https://example.com

paragraphs!

#happy :)"

   text = re.sub(r'https?://\S+|www\.\S+', , text)  # Remove URLs

# Example after this step: "i won't go there! this is a testing @username

paragraphs!

#happy :)"

   text = re.sub(r'<.*?>', , text)  # Remove HTML tags
   # Example after this step: "i won't go there! this is a testing @username  paragraphs! #happy :)"

   text = re.sub(r'@\w+', , text)  # Remove mentions
   # Example after this step: "i won't go there! this is a testing   paragraphs! #happy :)"

   text = re.sub(r'#\w+', , text)  # Remove hashtags
   # Example after this step: "i won't go there! this is a testing   paragraphs!  :)"

   # Translate emoticons to their word equivalents
   emoticons = {':)': 'smile', ':-)': 'smile', ':(': 'sad', ':-(': 'sad'}
   words = text.split()
   words = [emoticons.get(word, word) for word in words]
   text = " ".join(words)
   # Example after this step: "i won't go there! this is a testing paragraphs! smile"

   text = re.sub(r'[^\w\s]', , text)  # Remove punctuations
   # Example after this step: "i won't go there this is a testing paragraphs smile"

   text = re.sub(r'\s+[a-zA-Z]\s+', ' ', text)  # Remove standalone single alphabetical characters
   # Example after this step: "won't go there this is testing paragraphs smile"

   text = re.sub(r'\s+', ' ', text, flags=re.I)  # Substitute multiple consecutive spaces with a single space
   # Example after this step: "won't go there this is testing paragraphs smile"

   # Remove stopwords
   text = ' '.join(word for word in text.split() if word not in STOPWORDS)
   # Example after this step: "won't go there testing paragraphs smile"

   # Stemming
   stemmer = PorterStemmer()
   text = ' '.join(stemmer.stem(word) for word in text.split())
   # Example after this step: "won't go there test paragraph smile"

   # Lemmatization. (flies --> fly, went --> go)
   lemmatizer = WordNetLemmatizer()
   text = ' '.join(lemmatizer.lemmatize(word) for word in text.split())

   return text

1. Assuming docs is a list of objects and each object has a page_content and metadata attribute.

for doc in docs:

   original_content = doc.page_content  # Save the original page_content.
   doc.page_content = clean_text(original_content)  # Update page_content with the cleaned text.

   # Assuming metadata is a dictionary, and updating it with the original page_content under the key 'prompt'.
   if doc.metadata is None:  # Check if metadata is None and initialize if necessary.
       doc.metadata = {}
   doc.metadata['prompt'] = original_content

print(docs[0])