Real folk

Kotoba: Difference between revisions

Page Discussion
No edit summary
No edit summary
Line 1: Line 1:
import re
from PIL import Image, ImageEnhance, ImageOps
import nltk
from textractor import Textractor
from nltk.corpus import stopwords
from textractor.visualizers.entitylist import EntityList
from nltk.stem import PorterStemmer, WordNetLemmatizer
from textractor.data.constants import TextractFeatures
from openpyxl import load_workbook
from openpyxl.utils.cell import range_boundaries


# Initialize stemmer and lemmatizer
def enhance_image(img):
stemmer = PorterStemmer()
    enhancer = ImageEnhance.Contrast(img)
lemmatizer = WordNetLemmatizer()
    enhanced_img = enhancer.enhance(1.5)
STOPWORDS = set(stopwords.words('english'))
    return enhanced_img


nltk.download('stopwords')
def ocr(input_file, output_dir, image=None):
nltk.download('wordnet')
    extractor = Textractor(region_name="us-east-1")
    output_filename = os.path.splitext(input_file)[0] + '.xlsx'
    output_filepath = os.path.join(output_dir, output_filename)
    if image is None:
        print('input:', input_file)
        image = Image.open(input_file)


    image = enhance_image(image)
    document = extractor.analyze_document(
        file_source=image,
        features=[TextractFeatures.TABLES],
        save_image=True
    )
    table = EntityList(document.tables[0])
    table[0].to_excel(filepath=output_filepath)
    wb = load_workbook(filename=output_filepath)


def clean_text(text):
    for st_name in wb.sheetnames:
    # Original Text
        st = wb[st_name]
    # Example: "This is a Testing @username https://example.com <p>Paragraphs!</p> #happy :)"
        mcr_coord_list = [mcr.coord for mcr in st.merged_cells.ranges]


    text = text.lower() # Convert all characters in text to lowercase
        for mcr in mcr_coord_list:
    # Example after this step: "i won't go there! this is a testing @username https://example.com <p>paragraphs!</p> #happy :)"
            min_col, min_row, max_col, max_row = range_boundaries(mcr)
            top_left_cell_value = st.cell(row=min_row, column=min_col).value
            st.unmerge_cells(mcr)
            for row in st.iter_rows(min_col=min_col, min_row=min_row, max_col=max_col, max_row=max_row):
                for cell in row:
                    cell.value = top_left_cell_value


     text = re.sub(r'https?://\S+|www\.\S+', '', text)  # Remove URLs
     wb.save(output_filepath)
    # Example after this step: "i won't go there! this is a testing @username  <p>paragraphs!</p> #happy :)"


    text = re.sub(r'<.*?>', '', text)  # Remove HTML tags
     return output_filepath
    # Example after this step: "i won't go there! this is a testing @username  paragraphs! #happy :)"
 
    text = re.sub(r'@\w+', '', text)  # Remove mentions
    # Example after this step: "i won't go there! this is a testing  paragraphs! #happy :)"
 
    text = re.sub(r'#\w+', '', text)  # Remove hashtags
    # Example after this step: "i won't go there! this is a testing  paragraphs!  :)"
 
    # Translate emoticons to their word equivalents
    emoticons = {':)': 'smile', ':-)': 'smile', ':(': 'sad', ':-(': 'sad'}
    words = text.split()
    words = [emoticons.get(word, word) for word in words]
    text = " ".join(words)
    # Example after this step: "i won't go there! this is a testing paragraphs! smile"
 
    text = re.sub(r'[^\w\s]', '', text)  # Remove punctuations
    # Example after this step: "i won't go there this is a testing paragraphs smile"
 
    text = re.sub(r'\s+[a-zA-Z]\s+', ' ', text)  # Remove standalone single alphabetical characters
    # Example after this step: "won't go there this is testing paragraphs smile"
 
    text = re.sub(r'\s+', ' ', text, flags=re.I)  # Substitute multiple consecutive spaces with a single space
    # Example after this step: "won't go there this is testing paragraphs smile"
 
    # Remove stopwords
    text = ' '.join(word for word in text.split() if word not in STOPWORDS)
    # Example after this step: "won't go there testing paragraphs smile"
 
    # Stemming
    stemmer = PorterStemmer()
    text = ' '.join(stemmer.stem(word) for word in text.split())
    # Example after this step: "won't go there test paragraph smile"
 
    # Lemmatization. (flies --> fly, went --> go)
    lemmatizer = WordNetLemmatizer()
    text = ' '.join(lemmatizer.lemmatize(word) for word in text.split())
 
     return text
 
# Assuming docs is a list of objects and each object has a page_content and metadata attribute.
for doc in docs:
    original_content = doc.page_content  # Save the original page_content.
    doc.page_content = clean_text(original_content)  # Update page_content with the cleaned text.
 
    # Assuming metadata is a dictionary, and updating it with the original page_content under the key 'prompt'.
    if doc.metadata is None:  # Check if metadata is None and initialize if necessary.
        doc.metadata = {}
    doc.metadata['prompt'] = original_content
print(docs[0])

Revision as of 12:06, 6 November 2024

from PIL import Image, ImageEnhance, ImageOps from textractor import Textractor from textractor.visualizers.entitylist import EntityList from textractor.data.constants import TextractFeatures from openpyxl import load_workbook from openpyxl.utils.cell import range_boundaries

def enhance_image(img): 

   enhancer = ImageEnhance.Contrast(img)
   enhanced_img = enhancer.enhance(1.5)
   return enhanced_img

def ocr(input_file, output_dir, image=None): 

   extractor = Textractor(region_name="us-east-1")
   output_filename = os.path.splitext(input_file)[0] + '.xlsx'
   output_filepath = os.path.join(output_dir, output_filename)
   if image is None:
       print('input:', input_file)
       image = Image.open(input_file)

   image = enhance_image(image)
   document = extractor.analyze_document(
       file_source=image,
       features=[TextractFeatures.TABLES],
       save_image=True
   )
   table = EntityList(document.tables[0])
   table[0].to_excel(filepath=output_filepath)
   wb = load_workbook(filename=output_filepath)

   for st_name in wb.sheetnames:
       st = wb[st_name]
       mcr_coord_list = [mcr.coord for mcr in st.merged_cells.ranges]

       for mcr in mcr_coord_list:
           min_col, min_row, max_col, max_row = range_boundaries(mcr)
           top_left_cell_value = st.cell(row=min_row, column=min_col).value
           st.unmerge_cells(mcr)
           for row in st.iter_rows(min_col=min_col, min_row=min_row, max_col=max_col, max_row=max_row):
               for cell in row:
                   cell.value = top_left_cell_value

   wb.save(output_filepath)

   return output_filepath
Retrieved from "https://intertino.it:443/wiki/index.php?title=Kotoba&oldid=153"