I have a python script which uses selenium for web-scraping on an ubuntu server. I am creating fresh driver object in a while loop in every iteration and quitting it after the use.
The script runs fine for sometime. then it stops creating new driver, with an exception ‘Too many open files’.

Here is the script:

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
from bs4 import BeautifulSoup
import requests
import os
import logging
from PyPDF2 import PdfReader
from webdriver_manager.chrome import ChromeDriverManager
from openai import OpenAI
import undetected_chromedriver as uc
import time
import shutil
import glob
import logging
import csv
import traceback



logging.basicConfig(format='%(asctime)s - %(name)s - %(process)d - %(threadName)s - %(levelname)s - %(message)s',
                        level=logging.WARNING,
                        handlers=[logging.FileHandler("/root/nse-announcements/logs-warnings.log"),
                                  logging.StreamHandler()])

url = 'https://www.nseindia.com/companies-listing/corporate-filings-announcements'
bot_token = '7069953058:AAGsJ-hihPjME'
bot_chatID = os.getenv('TELEGRAM_BOT_CHAT_ID')
openai_api_key = os.getenv('OPENAI_API_KEY')
openAI_view = ''
first_page_text = ''
path_to_pdf = ''
pdf_link = ''
subject = ''

pdf_link_file_path = '/root/nse-announcements/announcements_pdf_links.txt'
temp_pdf = '/root/nse-announcements/temp.pdf'
user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36'
download_directory = "/root/nse-announcements/pdfs-of-announcements"
csv_file_path = '/root/nse-announcements/nse_announcements.csv'
csv_headers = ['Date', 'Symbol', 'Company Name', 'Subject', 'Details', 'OpenAI View', 'Broadcast Date/Time', 'PDF Link']

with requests.Session() as session:
    response = session.get(f'https://api.telegram.org/bot{bot_token}/sendMessage', 
                params={'chat_id': bot_chatID, 'text': "Hello Unix User! I'm now active and ready to fetch NSE announcements for you."})
    


def clear_tmp_directory():
    tmp_path = '/tmp'

    tmp_dirs = glob.glob(f"{tmp_path}/tmp*" )
    for directory in tmp_dirs:
        try:
            shutil.rmtree(directory)
        except Exception as e:
            pass

def clean_scoped_directories():
    tmp_path = '/tmp'

    scoped_dirs = glob.glob(f"{tmp_path}/scoped_dir*")
    for directory in scoped_dirs:
        try:
            shutil.rmtree(directory)
        except Exception as e:
            pass
            # logging.debug(f"Failed to remove {directory}: {str(e)}")


    chrome_items = glob.glob(f"{tmp_path}/.com.google.Chrome.*")
    for item in chrome_items:
        try:
            if os.path.isdir(item):
                shutil.rmtree(item)
            else:
                os.remove(item)
        except Exception as e:
            # Check if the error is because the item is not a directory
            if "Errno 20" in str(e):
                try:
                    os.remove(item)  # Attempt to delete it as a file
                except Exception as e:
                    pass
                    # logging.debug(f"Failed to remove {item} as a file: {e}")
            # else:
                # logging.debug(f"Failed to remove {item}: {e}")

def clean_text(text):
    return text.replace('&', '').replace('<', '').replace('>', '').replace('"', '').replace("'", "")

def send_telegram_message(data):
    message = f"<b>Symbol:</b> {data['Symbol']}n" 
              f"<b>{data['pref_alert']}</b>n" 
              f"<b>Company Name:</b> {data['Company Name']}n" 
              f"<b>Broadcast Date/Time:</b> {data['Broadcast Date/Time']}n" 
              f"<b>Subject:</b> {data['Subject']}n" 
              f"<b>Details:</b> {data['Details']}n" 
              f"<b>OpenAI:</b> {data['OpenAI']}n" 
              f"<b>PDF Link:</b> {data['PDF Link']}"

    url = f'https://api.telegram.org/bot{bot_token}/sendMessage'
    params = {
        'chat_id': bot_chatID,
        'parse_mode': 'HTML',
        'text': message
    }
    max_attempts = 30  # Maximum number of attempts to send the message
    attempt = 0
    connect_timeout, read_timeout = 10, 30  # Timeouts
    delay_seconds = 1  # Initial delay for retries

    while attempt < max_attempts:
        try:
            with requests.Session() as session:
                response = session.get(url, params=params, timeout=(connect_timeout, read_timeout))
                if response.status_code == 200:
                    return True
                elif response.status_code in {301, 302, 307, 308}:
                    # Handle redirection, if needed
                    url = response.headers['Location']
                elif 400 <= response.status_code < 500:
                    logging.warning(f'Client error: {response.status_code} - {response.text}')
                    break  # Stop retrying for client errors
                elif 500 <= response.status_code < 600:
                    logging.warning(f'Server error: {response.status_code} - {response.text}')
                    # Consider retrying or handling server-side errors
                else:
                    logging.warning(f'Unhandled status code: {response.status_code}')
        except Exception as e:
            logging.warning(f'Exception in send_telegram_message: {str(e)}')
            time.sleep(delay_seconds)
            delay_seconds *= 2  # Exponential backoff
        finally:
            attempt += 1  # Ensure attempt is incremented

    return False

def wait_for_download(path_to_pdf):
    timeout = 5  # Maximum time to wait for the download to appear (in seconds)
    start_time = time.time()
    while True:
        elapsed_time = time.time() - start_time
        if os.path.exists(path_to_pdf) and os.path.getsize(path_to_pdf) > 0:
            return True
        if elapsed_time > timeout:
            return False
        time.sleep(1)

def process_pdf(pdf_link):
    global path_to_pdf, openAI_view
    driver = None
    try:
        chrome_options = uc.ChromeOptions()
        chrome_options.add_argument('--headless')
        chrome_options.add_argument("--no-sandbox")
        chrome_options.add_argument("--disable-dev-shm-usage")
        chrome_options.add_argument('log-level=3')
        chrome_options.add_argument("--disable-gpu")
        chrome_options.add_argument("--disable-webgl")
        chrome_options.add_experimental_option('prefs', {
            "download.default_directory": download_directory,
            "download.prompt_for_download": False,
            "download.directory_upgrade": True,
            "plugins.always_open_pdf_externally": True
        })
        driver = uc.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options)
        time.sleep(2)
        driver.execute_cdp_cmd('Network.setUserAgentOverride', {'userAgent': user_agent})

        pdf_file_name = pdf_link.split('/')[-1]
        path_to_pdf = os.path.join(download_directory, pdf_file_name)

        for attempt in range(10):  # Retry attempts
            driver.get(pdf_link)
            if wait_for_download(path_to_pdf):
                with open(path_to_pdf, 'rb') as file:
                    reader = PdfReader(file)
                    first_page = reader.pages[0]
                    first_page_text = first_page.extract_text()
                    if first_page_text:
                        openAI_view = 'first_page_text'
                        return first_page_text
                    else:
                        logging.warning(f'PDF {pdf_link} first page contains image only content or no text.')
                        openAI_view = 'image PDF'
                os.remove(path_to_pdf)  # Move deletion outside of the try-except
                return False
            time.sleep(5)  # Wait 5 seconds before the next retry

        
        logging.warning(f"Failed to fetch PDF {pdf_link} after multiple attempts.")
        openAI_view = 'failed to fetch PDF'
        return False

    except Exception as e:
        logging.warning(f'Exception in process_pdf: {str(e)}')
        openAI_view = 'error in processing PDF'
        return False

    finally:
        if driver:
            driver.quit()


def summarize_text(text):
    client = OpenAI(api_key=openai_api_key)
    
    if client.api_key is None:
        return "API key not found in environment variables"
    
    system_msg = 'You know what is most relevant and important for the investors of a company.'
    user_msg = f'Summarize important information (ignore addresses) in company announcement text: {text}'
    # system_msg = 'Identify and summarize key details from company announcements related specifically to fund raising through equity shares, warrants, or other securities. Focus on details such as the type of offering, amount being raised, type of investors, and any relevant approvals or voting outcomes. Ignore routine operational details and addresses.'
    # user_msg = f'Analyze this announcement text and summarize any sections related to fund raising through equity shares or warrants: {text}'


    attempts = 0
    max_attempts = 3  # Set the number of attempts to 3

    while attempts < max_attempts:
        attempts += 1
        # Create a chat completion using the client
        chat_completion = client.chat.completions.create(
            messages=[
                {"role": "system", "content": system_msg},
                {"role": "user", "content": user_msg}
            ],
            model="gpt-3.5-turbo",
            temperature=0.3,
            max_tokens=150,
        )

        reason = chat_completion.choices[0].finish_reason

        # Handling for the 'stop' finish reason
        if reason == 'stop':
            return chat_completion.choices[0].message.content

        # Handling for the 'length' finish reason (output truncated due to max tokens)
        elif reason == 'length':
            return f"Output truncated: {chat_completion.choices[0].message.content}"

        # Handling for the 'function_call' finish reason (model initiated a function call)
        elif reason == 'function_call':
            return "Function call initiated by the model."

        # Handling for the 'content_filter' finish reason (content omitted by filters)
        elif reason == 'content_filter':
            return "Content filtered due to safety protocols."

        # Handling for the 'null' finish reason (response still in progress or incomplete)
        elif reason == 'null':
            return "Response still in progress or incomplete."
    
        time.sleep(2)
    return "Failed to summarize text after maximum attempts."

def write_to_csv(data):
    with open(csv_file_path, 'a', newline='') as csvfile:  # 'a' mode to append to the file
        writer = csv.DictWriter(csvfile, fieldnames=csv_headers)
        writer.writerow(data)


existing_links = set()
# Load existing links from file if it exists
try:
    with open(pdf_link_file_path, 'r') as file:
        existing_links = set(line.strip() for line in file)
except FileNotFoundError:
    pass

try:
    with open(csv_file_path, 'x', newline='') as csvfile:  # 'x' mode to create and fail if exists
        writer = csv.DictWriter(csvfile, fieldnames=csv_headers)
        writer.writeheader()
except FileExistsError:
    pass

avoidable_keywords = ['74(5)', 'GST authority', 'delisting']

avoidable_subjects_details = ['Analysts/Institutional', 'FDA Inspection']


while True:
    driver = None
    try:
        clean_scoped_directories()
        clear_tmp_directory()
        chrome_options = uc.ChromeOptions()
        chrome_options.add_argument('--headless')
        chrome_options.add_argument("--no-sandbox")
        chrome_options.add_argument("--disable-dev-shm-usage")
        chrome_options.add_argument('log-level=3')
        chrome_options.add_argument("--disable-gpu")
        chrome_options.add_argument("--disable-webgl")
        chrome_options.add_experimental_option('prefs', {
            "download.default_directory": download_directory,
            "download.prompt_for_download": False,
            "download.directory_upgrade": True,
            "plugins.always_open_pdf_externally": True
            })
        driver = uc.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options) 
        time.sleep(2)
        driver.execute_cdp_cmd('Network.setUserAgentOverride', {'userAgent': user_agent})
        wait = WebDriverWait(driver, 20)
        driver.get(url)
        time.sleep(5)
        wait.until(EC.presence_of_element_located(
                (By.XPATH, '//table[@id="CFanncEquityTable"]/tbody/tr[1]/td[6]')))
        html_source = driver.page_source
        soup = BeautifulSoup(html_source, 'html.parser')
        table = soup.find('table', {'id': 'CFanncEquityTable'})
        driver.quit()
        if not table or table.find_all('a', href=True) == []:
            continue

        for row in table.find_all('tr'): 
            openAI_view = ''
            first_page_text = ''
            path_to_pdf = ''
            pdf_link = ''
            subject = ''
            pref_alert = ''
            cells = row.find_all('td')
            if len(cells) > 4:
                pdf_link = cells[4].find('a', href=True)['href'] if cells[4].find('a', href=True) and cells[4].find('a', href=True)['href'].endswith('.pdf') else None
                subject = clean_text(cells[2].text.strip())
                details = clean_text(cells[3].text.strip())
                company_name = clean_text(cells[1].text.strip())
                broadcast_date_time = cells[5].text.strip().split("Exchange")[0].strip()
                symbol = cells[0].text.strip()
                original_symbol = symbol
                symbol = clean_text(symbol)
                if symbol != original_symbol:
                    symbol = symbol + " (Edited)"

                if pdf_link and pdf_link not in existing_links:
                    found_subject = None
                    for keyword in avoidable_subjects_details:
                        if keyword.lower() in subject.lower() or keyword.lower() in details.lower():
                            found_subject = keyword
                            break

                    if found_subject is not None:
                        logging.warning(f'************** SKIPPING PDF SINCE IT CONTAINS THE AVOIDABLE SUBJECT/DETAIL: {found_subject} ***************')
                        existing_links.add(pdf_link)
                        with open(pdf_link_file_path, 'a') as file:
                            file.write(pdf_link + 'n')
                        data = {
                            'Date': time.strftime("%Y-%m-%d"), 
                            'Symbol': symbol,
                            'Company Name': company_name,
                            'Subject': subject,
                            'Details': details,
                            'OpenAI View': 'SKIP',
                            'Broadcast Date/Time': broadcast_date_time,
                            'PDF Link': pdf_link
                        }
                        write_to_csv(data)
                        continue

                    first_page_text = process_pdf(pdf_link)
                    if first_page_text:
                        os.remove(path_to_pdf)
                        found_keyword = None
                        preferential_issue_keywords = ['preferential issue', 'equity shares', 'convertible warrants', 'raising funds']
                        fund_raising_found = 'preferential' in first_page_text.lower() and any(keyword in first_page_text.lower() for keyword in preferential_issue_keywords)
                        if fund_raising_found:
                            pref_alert = '*** PREFERENTIAL ISSUE ALERT ***'
                        else:
                            for keyword in avoidable_keywords:
                                if keyword.lower() in first_page_text.lower():
                                    found_keyword = keyword
                                    break

                            if found_keyword is not None:
                                logging.warning(f'************** SKIPPING PDF SINCE IT CONTAINS THE AVOIDABLE KEYWORD: {found_keyword} ***************')
                                existing_links.add(pdf_link)
                                with open(pdf_link_file_path, 'a') as file:
                                    file.write(pdf_link + 'n')
                                data = {
                                    'Date': time.strftime("%Y-%m-%d"), 
                                    'Symbol': symbol,
                                    'Company Name': company_name,
                                    'Subject': subject,
                                    'Details': details,
                                    'OpenAI View': 'SKIP',
                                    'Broadcast Date/Time': broadcast_date_time,
                                    'PDF Link': pdf_link
                                }
                                write_to_csv(data)
                                continue

                        openAI_view = summarize_text(first_page_text)

                    data = {
                        'Symbol': symbol,
                        'Company Name': company_name,
                        'Subject': subject,
                        'Details': details,
                        'OpenAI': openAI_view,
                        'Broadcast Date/Time': broadcast_date_time,
                        'PDF Link': pdf_link
                    }
                    should_link_be_saved = send_telegram_message(data)
                    if should_link_be_saved:
                        existing_links.add(pdf_link)
                        with open(pdf_link_file_path, 'a') as file:
                            file.write(pdf_link + 'n')

                data = {
                        'Date': time.strftime("%Y-%m-%d"), 
                        'Symbol': symbol,
                        'Company Name': company_name,
                        'Subject': subject,
                        'Details': details,
                        'OpenAI View': openAI_view,
                        'Broadcast Date/Time': broadcast_date_time,
                        'PDF Link': pdf_link
                    }
                write_to_csv(data)

    except Exception as e:
        tb = traceback.format_exc()
        logging.warning(f'Exception in TRY BLOCK {str(e)}nTraceback: {tb}')
        if 'too many open files' in str(e).lower():
                break
    finally:
        if driver:
            driver.quit()

I ran lsof and this is the output:

root@nse-announcements:~# lsof -c python
COMMAND    PID USER   FD   TYPE             DEVICE SIZE/OFF   NODE NAME
python  120968 root  cwd    DIR              253,1     4096 787286 /root/nse-announcements
python  120968 root  rtd    DIR              253,1     4096      2 /
python  120968 root  txt    REG              253,1  6752256   1967 /usr/bin/python3.11
python  120968 root  mem    REG              253,1  5026584 790814 /root/nse-announcements/venv/lib/python3.11/site-packages/pydantic_core/_pydantic_core.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1   141872   2769 /usr/lib/x86_64-linux-gnu/libgcc_s.so.1
python  120968 root  mem    REG              253,1   311112   6376 /usr/lib/python3.11/lib-dynload/_decimal.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1   264392 790605 /root/nse-announcements/venv/lib/python3.11/site-packages/charset_normalizer/md__mypyc.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1    74496   6361 /usr/lib/python3.11/lib-dynload/_asyncio.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1   143992   6371 /usr/lib/python3.11/lib-dynload/_ctypes.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1  5235544   3132 /usr/lib/x86_64-linux-gnu/libcrypto.so.3
python  120968 root  mem    REG              253,1    43560   2790 /usr/lib/x86_64-linux-gnu/libffi.so.8.1.2
python  120968 root  mem    REG              253,1    24336   6381 /usr/lib/python3.11/lib-dynload/_multiprocessing.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1    54488   6380 /usr/lib/python3.11/lib-dynload/_multibytecodec.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1   202904   3071 /usr/lib/x86_64-linux-gnu/liblzma.so.5.4.1
python  120968 root  mem    REG              253,1    14712  74280 /usr/lib/x86_64-linux-gnu/librt.so.1
python  120968 root  mem    REG              253,1   671960   3133 /usr/lib/x86_64-linux-gnu/libssl.so.3
python  120968 root  mem    REG              253,1  3052896   4166 /usr/lib/locale/locale-archive
python  120968 root  mem    REG              253,1    74848   3879 /usr/lib/x86_64-linux-gnu/libbz2.so.1.0.4
python  120968 root  mem    REG              253,1  2105184  71658 /usr/lib/x86_64-linux-gnu/libc.so.6
python  120968 root  mem    REG              253,1    14480  73974 /usr/lib/x86_64-linux-gnu/libpthread.so.0
python  120968 root  mem    REG              253,1    45080   6379 /usr/lib/python3.11/lib-dynload/_lzma.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1    32088   6362 /usr/lib/python3.11/lib-dynload/_bz2.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1   216944   6360 /usr/lib/python3.11/lib-dynload/_ssl.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1    63696   6359 /usr/lib/python3.11/lib-dynload/_hashlib.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1    14504   6369 /usr/lib/python3.11/lib-dynload/_contextvars.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1    35032   1885 /usr/lib/x86_64-linux-gnu/libuuid.so.1.3.0
python  120968 root  DEL    REG               0,26            1032 /dev/shm/sem.LNVy9P
python  120968 root  mem    REG              253,1    16064 790602 /root/nse-announcements/venv/lib/python3.11/site-packages/charset_normalizer/md.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1    14688   6392 /usr/lib/python3.11/lib-dynload/_uuid.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1    49128   6377 /usr/lib/python3.11/lib-dynload/_json.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1    23664   6383 /usr/lib/python3.11/lib-dynload/_queue.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1    14472   6391 /usr/lib/python3.11/lib-dynload/_typing.cpython-311-x86_64-linux-gnu.so
python  120968 root  mem    REG              253,1   357584   1908 /usr/lib/locale/C.utf8/LC_CTYPE
python  120968 root  mem    REG              253,1   174336   5421 /usr/lib/x86_64-linux-gnu/libexpat.so.1.8.10
python  120968 root  mem    REG              253,1   121200   3887 /usr/lib/x86_64-linux-gnu/libz.so.1.2.13
python  120968 root  mem    REG              253,1   957008  71661 /usr/lib/x86_64-linux-gnu/libm.so.6
python  120968 root  mem    REG              253,1    27028  75433 /usr/lib/x86_64-linux-gnu/gconv/gconv-modules.cache
python  120968 root  mem    REG              253,1   232568  71655 /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
python  120968 root    0r   CHR                1,3      0t0      5 /dev/null
python  120968 root    1u  unix 0xffff9ab807f43b80      0t0 724111 type=STREAM (CONNECTED)
python  120968 root    2u  unix 0xffff9ab807f43b80      0t0 724111 type=STREAM (CONNECTED)
python  120968 root    3w   REG              253,1      882 793138 /root/nse-announcements/logs-warnings.log
python  120968 root    4u  IPv4             724137      0t0    TCP nse-announcements:35066->149.154.167.220:https (ESTABLISHED)
python  120968 root    5u  IPv6             724255      0t0    TCP localhost:46688->localhost:60247 (CLOSE_WAIT)
python  120968 root    6u  IPv6             725494      0t0    TCP localhost:52052->localhost:38957 (CLOSE_WAIT)
python  120968 root   13u  IPv6             730702      0t0    TCP localhost:33460->localhost:60785 (ESTABLISHED)
python  120968 root   14w  FIFO               0,14      0t0 728361 pipe
python  120968 root   19r  FIFO               0,14      0t0 730626 pipe
root@nse-announcements:~#

python is leaving TCP connections in CLOSE_WAIT state. I don’t know where these connections are being created in the script. totally mysterious.
what can be done here so that scripts runs indefinitely, as it is supposed to.