Multithreading stuck at last future
def download_files_from_folder(base_url, folder_name): folder_url = f”{base_url}{folder_name}/” response = requests.get(folder_url) soup = BeautifulSoup(response.content, “html.parser”) links = soup.find_all(“a”) peninsular_rain_rates = [] east_rain_rates = [] completed_urls = [] with ThreadPoolExecutor(max_workers=15) as executor: futures = {executor.submit(process_gz_file, folder_url + link.get(“href”)): link.get(“href”) for link in links if link.get(“href”).endswith(“.gz”)} for future in as_completed(futures): peninsular, east, completed = future.result() peninsular_rain_rates.extend(peninsular) east_rain_rates.extend(east) completed_urls.extend(completed) futures.pop(future) […]
Multithreading stucks at last future
def download_files_from_folder(base_url, folder_name): folder_url = f”{base_url}{folder_name}/” response = requests.get(folder_url) soup = BeautifulSoup(response.content, “html.parser”) links = soup.find_all(“a”) peninsular_rain_rates = [] east_rain_rates = [] completed_urls = [] with ThreadPoolExecutor(max_workers=15) as executor: futures = {executor.submit(process_gz_file, folder_url + link.get(“href”)): link.get(“href”) for link in links if link.get(“href”).endswith(“.gz”)} for future in as_completed(futures): peninsular, east, completed = future.result() peninsular_rain_rates.extend(peninsular) east_rain_rates.extend(east) completed_urls.extend(completed) futures.pop(future) […]