Web Automation using Python Script and Selenium in Microsoft Edge: Error in Entering and saving data

[Seenivasan Chinnachamy]

Hello Everyone,

I was trying to automate the data entry process in a website.

1. The server is too slow
2. The data entry process is hectic as there are tens of thousands of entries need to be made in the website
3. The elements are loaded dynamically
4. Only the intended section of the webpage is targeted.
5. Wait times are included to ensure element interaction
6. I tried to automate the process using python and selenium. I used web driver for Microsoft Edge. There is something wrong with the code that it fails to save the data every single time.

The following is the full code that I used.

from selenium import webdriver

from selenium.webdriver.chrome.service import Service

from selenium.webdriver.chrome.options import Options

from selenium.webdriver.common.by import By

from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.support import expected_conditions as EC

import pandas as pd

import time

# Load Excel data

excel_file = "D:\\Work\\867\\867_Demand_Test.xlsx"

# Load Habitation Code (Column B) and data from Columns H to P

df = pd.read_excel(excel_file, usecols="B,H:P", skiprows=0)

df.columns = ['Habitation Code'] + [f'Field_{i}' for i in range(1, 10)]

# Convert all values to strings with proper decimal formatting

def format_decimal(value):

    if pd.isna(value):

        return ""

    try:

        float_value = float(value)

        if float_value.is_integer():

            return str(int(float_value))

        else:

            return f"{float_value:.3f}".rstrip('0').rstrip('.')

    except ValueError:

        return str(value)

df = df.applymap(format_decimal)

# Configure Chrome options to disable notifications and dialogs

chrome_options = Options()

chrome_options.add_argument("--disable-notifications")

chrome_options.add_argument("--disable-popup-blocking")

# Path of Chrome WebDriver executable

chrome_driver_path = "C:\\WebDriver\\Chrome\\chromedriver.exe"

# Set up the Chrome Service and WebDriver

service = Service(executable_path=chrome_driver_path)

driver = webdriver.Chrome(service=service, options=chrome_options)

# Open the URL

driver.get('http://twadpms.in/twadpms')

# Maximize the browser window

driver.maximize_window()

# Initialize WebDriverWait

wait = WebDriverWait(driver, 20)

# Wait for the page to load

time.sleep(5)

# Locate and interact with login elements

try:

    wait.until(EC.visibility_of_element_located((By.XPATH, "//input[@id=':r2:']"))).send_keys('twad14253')

    wait.until(EC.visibility_of_element_located((By.ID, 'eyePassword'))).send_keys('Sridhar_51')

    wait.until(EC.element_to_be_clickable((By.XPATH, "//button[text()='Login']"))).click()

    print("Login successful.")

except Exception as e:

    print(f"Error during login: {e}")

    driver.quit()

# Wait for the "Expand More" SVG element and click it

try:

    wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "svg[data-testid='ExpandMoreIcon']"))).click()

    # Wait for the "Schema" SVG element and click it

    wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "svg[data-testid='SchemaIcon']"))).click()

    # Wait for the "Edit" SVG element and click it

    wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "svg[data-testid='EditIcon']"))).click()

    print("Navigated to Edit page.")

except Exception as e:

    print(f"Error during navigation: {e}")

    driver.quit()

# Locate and click the fifth "show more" button

try:

    show_more_buttons = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "button[aria-label='show more']")))

    show_more_buttons[4].click()

    time.sleep(15)

    print("Clicked the fifth 'show more' button.")

except Exception as e:

    print(f"Error finding 'show more' button: {e}")

    driver.quit()

# Function to input data into table rows

def fill_table_data():

    for i, row in df.iterrows():

        habitation_code = row['Habitation Code']

        data_columns = row[1:]

        updated = False  # Flag to track if any value was updated

        try:

            # Find the row element by habitation code

            row_xpath = f"//tr[td[2][contains(text(), '{habitation_code}')]]"

            row_element = wait.until(EC.presence_of_element_located((By.XPATH, row_xpath)))

            print(f"Found row for Habitation Code {habitation_code}")

            # Locate all input fields in the last 9 <td> elements of the row

            for j in range(7, 16):  # From the 8th to the 16th <td> (0-indexed)

                try:

                    # Construct the XPath for the input field within the j-th <td> element

                    input_xpath = f"{row_xpath}/td[{j+1}]/div/div/input"

                    input_field = wait.until(EC.visibility_of_element_located((By.XPATH, input_xpath)))

                    # Ensure the field is focused and scrolled into view

                    driver.execute_script("arguments[0].scrollIntoView(true);", input_field)

                    time.sleep(1)

                    # Get the new value as string

                    new_value = data_columns.iloc[j - 7]

                    # Clear and set the value using JavaScript

                    driver.execute_script("arguments[0].value = '';", input_field)

                    driver.execute_script("arguments[0].value = arguments[1];", input_field, new_value)

                    driver.execute_script("arguments[0].dispatchEvent(new Event('input', { bubbles: true }));", input_field)

                    driver.execute_script("arguments[0].dispatchEvent(new Event('change', { bubbles: true }));", input_field)

                    print(f"Attempted to populate input in column {j+1} with data: {new_value}")

                    # Wait a short while to ensure the value is processed

                    time.sleep(1)

                    # Read back the value from the input field to verify

                    entered_value = input_field.get_attribute('value')

                    print(f"Actual value in input field for column {j+1}: {entered_value}")

                    # Compare and handle discrepancies

                    if entered_value != new_value:

                        print(f"Discrepancy detected in column {j+1} for Habitation Code {habitation_code}. Expected: {new_value}, Found: {entered_value}")

                        time.sleep(1)

                        continue

                    print(f"Successfully populated input in column {j+1} with data: {new_value}")

                    updated = True

                    # Wait 7 seconds before entering the next field

                    time.sleep(7)

                except Exception as e:

                    print(f"Error locating or filling input field in column {j+1} for Habitation Code {habitation_code}: {e}")

            if updated:

                # Click the save button using CSS selector

                try:

                    # Ensure the save button is in view

                    save_button_selector = "button[type='submit'].MuiButton-containedSAVE"

                    save_button = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, save_button_selector)))

                    driver.execute_script("arguments[0].scrollIntoView(true);", save_button)

                    # Use a normal click to click the button

                    save_button.click()

                    print("Clicked the save button.")

                    # Wait for 90 seconds to ensure data is saved

                    time.sleep(90)

                except Exception as e:

                    print(f"Error clicking save button: {e}")

                print(f"Saved data for Habitation Code {habitation_code}")

        except Exception as e:

            print(f"Error processing row with Habitation Code {habitation_code}: {e}")

# Call the function to fill table data

fill_table_data()

# Keep the browser open until manually closed

print("Browser is open. Press Ctrl+C in the terminal to close it.")

# Keep the browser open indefinitely

try:

    while True:

        time.sleep(1)

except KeyboardInterrupt:

    print("Browser closed.")

# Close the browser

driver.quit()

The send keys method for populating input fields messes up with decimal inputs. Input field population using JavaScript did not actually enter any values in the input field.

The manual save button click generates a "Saved successfully" pop-up message in the webpage and it disappears almost instantly. But I don't know how to tell the script to wait for that message and move on to the next entry.

There are more than one save buttons in the web page and I tried using the XPath method and JavaScript to click the save button, but it does not seem to work. What is going wrong here? The web page is opened almost instantly. Login was successful. The intended section is navigated through. Values can be seen populated. But data saving is not happening. Python Terminal shows everything populated successfully.

Please do offer a solution to this issue.