Download file to AWS Lambda /tmp file directory using Chromedriver - amazon-web-services

Question

I am trying to automate the download of a file to the /tmp directory in AWS Lambda using Chromedriver from this website https://registry.verra.org/app/search/VCS/All%20Projects. The steps are to 1) click on the 'Search' button, 2) wait for the results to load, 3) click on the 'Excel' logo to download the file.
I've referenced and tried the code provided in these 2 questions to change the download directory of Chromedriver but the file is not downloaded in the /tmp path.
AWS Lambda download a file using Chromedriver
prefs = {
"profile.default_content_settings.popups": 0,
"download.default_directory": r"/tmp",
"directory_upgrade": True
}
options.add_experimental_option("prefs", prefs)
Unable to change the default download location of chrome in AWS lambda to /tmp using Selenium
options = webdriver.ChromeOptions()
prefs = {"browser.downloads.dir": "//tmp//", "download.default_directory": "//tmp//", "directory_upgrade": True}
options.add_experimental_option("prefs", prefs)
For reference, here is my code which works fine in local but not in AWS Lambda.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import os
import requests
import requests.auth
import json
import csv
def lambda_handler(event, context):
# change directory to /tmp folder
os.chdir('/tmp')
# get dataset from website
options = Options()
options.binary_location = '/opt/headless-chromium'
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--single-process')
options.add_argument('--disable-dev-shm-usage')
## SAVE TO TMP DIRECTORY
# set download settings
prefs = {
"profile.default_content_settings.popups": 0,
"download.default_directory": r"/tmp",
"directory_upgrade": True
}
options.add_experimental_option("prefs", prefs)
## open Chrome webdriver
driver = webdriver.Chrome('/opt/chromedriver',options=options)
driver.maximize_window()
driver.get('https://registry.verra.org/app/search/VCS/All%20Projects')
# wait for 60 seconds for website content to load
print("Waiting for website to load...")
element1 = WebDriverWait(driver, 60).until(EC.presence_of_element_located((By.XPATH, '/html/body/apx-root/div/div/apx-search-page/div/apx-search-container/div/div[2]/div/div[1]/apx-search-selection-criteria/div/form/div[2]/div/button[1]')))
print("Website loaded!")
# click on search button to load results
search_btn = driver.find_element(By.XPATH, '/html/body/apx-root/div/div/apx-search-page/div/apx-search-container/div/div[2]/div/div[1]/apx-search-selection-criteria/div/form/div[2]/div/button[1]')
search_btn.click()
# wait for results to load for 100 seconds - determine by checking the page numbers
element2 = WebDriverWait(driver, 100).until(EC.presence_of_element_located((By.XPATH, '/html/body/apx-root/div/div/apx-search-page/div/apx-search-container/div/div[2]/div/div[2]/apx-project-search-results/div/div/kendo-grid/kendo-pager/kendo-pager-numeric-buttons/ul/li[1]/a')))
print("Results loaded!")
# wait for download button to load for 100 seconds - determine by detecting presence of download button
element = WebDriverWait(driver, 100).until(EC.presence_of_element_located((By.XPATH, '/html/body/apx-root/div/div/apx-search-page/div/apx-search-container/div/div[2]/div/div[2]/apx-project-search-results/div/apx-search-results-header/div/button[1]')))
download_btn = driver.find_element(By.XPATH, '/html/body/apx-root/div/div/apx-search-page/div/apx-search-container/div/div[2]/div/div[2]/apx-project-search-results/div/apx-search-results-header/div/button[1]')
# click on download button
# if element is not clickable
filepath = driver.execute_script("arguments[0].click();", element)
# wait for 60 seconds for file to download
time.sleep(60)
# check if file is downloaded to /tmp directory
# Method 2
list = os.listdir('/tmp')
print("list", list)
response = {
"statusCode": 200,
"body": "Selenium Headless Chrome Initialized"
}
return response
Could this be a versioning issue? Because my code only works if the runtime settings is for Python 3.7.
Selenium version: selenium/python/lib/python3.7/site-packages selenium==3.8.0\ (runtime Python 3.7)
Chromedriver version: https://chromedriver.storage.googleapis.com/2.37/chromedriver_linux64.zip\ (only runtime Python 3.9 works, changing to Python 3.7 does not work)
Headless Chrome: https://github.com/adieuadieu/serverless-chrome/releases/download/v1.0.0-41/stable-headless-chromium-amazonlinux-2017-03.zip