selenium click skip_button("introjs-skipbutton")

标签：handle skipbutton chrome skip button self soup options page

class INTERFACING():

    def __init__(self):
        self.driver_initialized = False
        self.driver = ''
        self.MAX_TRIALS = 2
        # self.chrome_version = get_google_chrome_version()

    def make_soup(self):
        return BeautifulSoup(self.driver.page_source, 'lxml')  # etree.HTML()

    def current_url(self):
        return self.driver.current_url

    def get_driver(self):

        # uc.TARGET_VERSION = get_google_chrome_version()
        chrome_options = uc.ChromeOptions()

        # chrome_options.add_argument("--headless")
        chrome_options.add_argument("--window-size=1920.,1080")
        chrome_options.add_argument("--disable-extensions")
        chrome_options.add_argument("--disable-dev-shm-usage")
        chrome_options.add_argument("--disable-popup-blocking")
        chrome_options.add_argument("--profile-directory=Default")
        chrome_options.add_argument("--ignore-certificate-errors")
        chrome_options.add_argument("--disable-plugins-discovery")
        chrome_options.add_argument("--incognito")
        chrome_options.add_argument("--no-first-run")
        chrome_options.add_argument("--no-service-autorun")
        chrome_options.add_argument("--no-default-browser-check")
        chrome_options.add_argument("--password-store=basic")
        chrome_options.add_argument("--no-sandbox")
        chrome_options.add_argument('--disable-application-cache')
        chrome_options.add_argument('--disable-gpu')
        chrome_options.add_argument("--disable-setuid-sandbox")
        chrome_options.add_argument(
            "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36"
        )
        self.driver = uc.Chrome(options=chrome_options, version_main="113")
        # self.browser = uc.Chrome(options=chrome_options, version_main=113)
        time.sleep(10)
        self.driver_initialized = True

    def close_driver(self):
        self.driver.quit()

    def get_selenium_response(self, url):

        # try:
        if not self.driver_initialized:
            self.get_driver()
        else:
            pass
        self.driver.get(url)
        time.sleep(3)
        soup = self.make_soup()
        return soup

    def get_page_source(self):
        return self.driver.page_source

    def clicking(self, xpath):
        elem = self.driver.find_element(By.XPATH, xpath)
        elem.click()
        time.sleep(random.randint(2, 3))

    def entering_values(self, xpath, value):
        elem = self.driver.find_element(By.XPATH, xpath)
        elem.clear()
        elem.send_keys(value)
        time.sleep(random.randint(2, 4))

    def send_keys(self, xpath):
        elem = self.driver.find_element(By.XPATH, xpath).send_keys(Keys.RETURN)

    def going_back(self):
        self.driver.execute_script("window.history.go(-1)")
        time.sleep(1)

    def refresh_page(self):
        self.driver.refresh()

    def close_handle(self):
        self.driver.close()

    def get_current_handle(self):
        return self.driver.current_window_handle

    def get_all_handles(self):
        return self.driver.window_handles

    def swtich_to_window(self, handle):
        self.driver.switch_to.window(handle)

    def switch_handle(self, second_handle=''):

        all_handles = self.get_all_handles()
        for handle in all_handles:
            self.main_page_handle = self.get_current_handle()
            if handle == self.main_page_handle:
                continue

            if second_handle and handle == second_handle:
                continue

            self.swtich_to_window(handle)

            return handle

    def close_handles(self, page_handle, second_handle):

        all_handles = self.get_all_handles()

        for handle in all_handles:
            if handle == page_handle:
                try:
                    self.close_handle()
                except:
                    pass

        self.swtich_to_window(second_handle)

    def skip_button(self, class_item):
        count = 0
        while 1:

            soup = self.make_soup()

            try:
                self.clicking(f'//a[contains(@class,"{class_item}")]')
                break
            except Exception as error:
                print('skip button not yet visible')

            if count > 3:
                try:
                    all_results = soup.find('table', class_='el-table__body').tbody.find_all('tr', class_=re.compile(
                        'el-table__row'))
                    return True
                except:
                    if soup.find('span', class_='el-table__empty-text') is not None:
                        return True
            try:
                self.clicking('//span[text()="Medical Devices"]')
                break
            except:
                pass

            time.sleep(2)

            count += 1

            if count == 20:
                break




    def search_data(self, current_query,page_num):
        if page_num == 1:
            self.entering_values('//*[@id="home"]/main/div[1]/div[7]/div/div[2]/input',current_query)
            self.clicking('//*[@id="home"]/main/div[1]/div[7]/div/div[2]/div/button')
            second_handle = self.switch_handle()
            self.skip_button('introjs-nextbutton')
            self.skip_button("introjs-skipbutton")
            soup = self.make_soup()

            if soup.find('span', class_='el-table__empty-text') is not None:
                pass
            else:
                print('Selecting 20 per page...')
                count = 0
                while True:
                    soup = self.make_soup()
                    try:
                        page_selector = soup.find('input', class_='el-input__inner')
                        if page_selector.attrs.get("placeholder"):
                            break
                    except Exception as error:
                        print('Record not yet loaded: ', count)
                    time.sleep(3)
                    count += 1
                    if not count % 3:
                        print('page refreshed....')
                        self.refresh_page()
                    if count >= 51:
                        break
                self.clicking('//*[@id="home"]/div[3]/div[3]/div/div/span[2]/div/div[1]/input')
                self.clicking(
                    '//ul[@class="el-scrollbar__view el-select-dropdown__list"]//span[text()="20条/页"]')
        if page_num != 1:
            while 1:
                try:
                    self.entering_values('//input[@type="number"]',page_num)
                    break
                except:
                    print('error in entering page num')

                time.sleep(3)
            self.send_keys('//input[@type="number"]')
            time.sleep(3)
        while 1:
            soup = self.make_soup()

            try:
                all_results = soup.find('table', class_='el-table__body').tbody.find_all('tr', class_=re.compile(
                    'el-table__row'))
            except:
                if soup.find('span', class_='el-table__empty-text') is not None:
                    print('No Results...')
                    all_results = []

            total_results = int(soup.find('span', class_='el-pagination__total').text.strip().split()[1])
            ending_page = total_results // 20 + 1

            while 1:

                # sometimes it takes long to load all the records on the page, so here making sure we loaded all 20 records or
                # if not then making usre it's the last page
                soup = self.make_soup()
                all_results = soup.find('table', class_='el-table__body').tbody.find_all('tr', class_=re.compile(
                    'el-table__row'))
                if len(all_results) == 20:
                    break
                if len(all_results) < 20 and ending_page == page_num:
                    break
                print(all_results, " : ", total_results, " : ", ending_page)
                time.sleep(3)

            # each click on the site opens a new window, so here we are switching windows and then closing windows once data read.
            for _result in range(len(all_results)):
                result = all_results[_result].find_all('td')
                if not result:
                    continue

                result_title = result[1].text.strip()

                print(page_num, " : ", ending_page, " : ", _result, " / ", len(all_results), " : ",
                      result_title, " : ", total_results, " : ", ending_page)

            print(f"page_num: {page_num} Done!")
            page_num += 1
            if page_num > ending_page:
                break
            next_button = soup.find('button', class_='btn-next').attrs
            if 'disabled' in next_button:
                break
            self.clicking('//button[@class="btn-next"]')
            time.sleep(3)
        # self.close_handles(second_handle, self.main_page_handle)




if __name__ == '__main__':
    REY_NUM = 5
    next_year = datetime.now().year + 1
    url = r'https://www.nmpa.gov.cn/datasearch/search-result.html'
    # with Display(visible=0, size=(1920, 1080)) as display:
    for _ in range(REY_NUM):
        try:
            handle = INTERFACING()
            soup = handle.get_selenium_response(url)
            handle.skip_button("introjs-skipbutton")

            soup = handle.make_soup()

            if soup.find('div', class_='header-main') is None:
                print("访问失败！")

            main_page_handle = handle.get_current_handle()

            count = 0
            while 1:
                soup = handle.make_soup()

                try:
                    handle.clicking('//span[text()="Medical Devices"]')
                    break
                except Exception as error:
                    print('Medical button not yet visible')

                try:
                    all_results = soup.find('table', class_='el-table__body').tbody.find_all('tr',class_=re.compile('el-table__row'))
                    break
                except:
                    pass

                response = handle.skip_button('introjs-skipbutton')

                if response:
                    break

                time.sleep(1)

                count += 1

                if count >= 5:
                    break
            handle.clicking("//*[@class='pc-max el-row']/div/a[@title='一次性使用医疗器械产品']")
            for device_type in ["械备", "注进", "注准"]:
                for year in range(2020, 2022):
                    current_query = f'{device_type}{year}'
                    handle.search_data(current_query,1)
                    # print(f'{device_type}{_year}')



        except Exception as e:
            print(f'爬取NMPADisposableProductsRequester数据失败: 详情{e}')

        handle.close_driver()
        time.sleep(60)
    else:
        raise Exception(
            f"已经重试{REY_NUM}次, 爬取NMPADisposableProductsRequester数据失败, 详情{e}")

　　跳过弹窗  handle.skip_button("introjs-skipbutton")

 之后

 

 

class INTERFACING():
    def __init__(self):        self.driver_initialized = False        self.driver = ''        self.MAX_TRIALS = 2        # self.chrome_version = get_google_chrome_version()
    def make_soup(self):        return BeautifulSoup(self.driver.page_source, 'lxml')  # etree.HTML()
    def current_url(self):        return self.driver.current_url
    def get_driver(self):
        # uc.TARGET_VERSION = get_google_chrome_version()        chrome_options = uc.ChromeOptions()
        # chrome_options.add_argument("--headless")        chrome_options.add_argument("--window-size=1920.,1080")        chrome_options.add_argument("--disable-extensions")        chrome_options.add_argument("--disable-dev-shm-usage")        chrome_options.add_argument("--disable-popup-blocking")        chrome_options.add_argument("--profile-directory=Default")        chrome_options.add_argument("--ignore-certificate-errors")        chrome_options.add_argument("--disable-plugins-discovery")        chrome_options.add_argument("--incognito")        chrome_options.add_argument("--no-first-run")        chrome_options.add_argument("--no-service-autorun")        chrome_options.add_argument("--no-default-browser-check")        chrome_options.add_argument("--password-store=basic")        chrome_options.add_argument("--no-sandbox")        chrome_options.add_argument('--disable-application-cache')        chrome_options.add_argument('--disable-gpu')        chrome_options.add_argument("--disable-setuid-sandbox")        chrome_options.add_argument(            "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36"        )        self.driver = uc.Chrome(options=chrome_options, version_main="113")        # self.browser = uc.Chrome(options=chrome_options, version_main=113)        time.sleep(10)        self.driver_initialized = True
    def close_driver(self):        self.driver.quit()
    def get_selenium_response(self, url):
        # try:        if not self.driver_initialized:            self.get_driver()        else:            pass        self.driver.get(url)        time.sleep(3)        soup = self.make_soup()        return soup
    def get_page_source(self):        return self.driver.page_source
    def clicking(self, xpath):        elem = self.driver.find_element(By.XPATH, xpath)        elem.click()        time.sleep(random.randint(2, 3))
    def entering_values(self, xpath, value):        elem = self.driver.find_element(By.XPATH, xpath)        elem.clear()        elem.send_keys(value)        time.sleep(random.randint(2, 4))
    def send_keys(self, xpath):        elem = self.driver.find_element(By.XPATH, xpath).send_keys(Keys.RETURN)
    def going_back(self):        self.driver.execute_script("window.history.go(-1)")        time.sleep(1)
    def refresh_page(self):        self.driver.refresh()
    def close_handle(self):        self.driver.close()
    def get_current_handle(self):        return self.driver.current_window_handle
    def get_all_handles(self):        return self.driver.window_handles
    def swtich_to_window(self, handle):        self.driver.switch_to.window(handle)
    def switch_handle(self, second_handle=''):
        all_handles = self.get_all_handles()        for handle in all_handles:            self.main_page_handle = self.get_current_handle()            if handle == self.main_page_handle:                continue
            if second_handle and handle == second_handle:                continue
            self.swtich_to_window(handle)
            return handle
    def close_handles(self, page_handle, second_handle):
        all_handles = self.get_all_handles()
        for handle in all_handles:            if handle == page_handle:                try:                    self.close_handle()                except:                    pass
        self.swtich_to_window(second_handle)
    def skip_button(self, class_item):        count = 0        while 1:
            soup = self.make_soup()
            try:                self.clicking(f'//a[contains(@class,"{class_item}")]')                break            except Exception as error:                print('skip button not yet visible')
            if count > 3:                try:                    all_results = soup.find('table', class_='el-table__body').tbody.find_all('tr', class_=re.compile(                        'el-table__row'))                    return True                except:                    if soup.find('span', class_='el-table__empty-text') is not None:                        return True            try:                self.clicking('//span[text()="Medical Devices"]')                break            except:                pass
            time.sleep(2)
            count += 1
            if count == 20:                break



    def search_data(self, current_query,page_num):        if page_num == 1:            self.entering_values('//*[@id="home"]/main/div[1]/div[7]/div/div[2]/input',current_query)            self.clicking('//*[@id="home"]/main/div[1]/div[7]/div/div[2]/div/button')            second_handle = self.switch_handle()            self.skip_button('introjs-nextbutton')            self.skip_button("introjs-skipbutton")            soup = self.make_soup()
            if soup.find('span', class_='el-table__empty-text') is not None:                pass            else:                print('Selecting 20 per page...')                count = 0                while True:                    soup = self.make_soup()                    try:                        page_selector = soup.find('input', class_='el-input__inner')                        if page_selector.attrs.get("placeholder"):                            break                    except Exception as error:                        print('Record not yet loaded: ', count)                    time.sleep(3)                    count += 1                    if not count % 3:                        print('page refreshed....')                        self.refresh_page()                    if count >= 51:                        break                self.clicking('//*[@id="home"]/div[3]/div[3]/div/div/span[2]/div/div[1]/input')                self.clicking(                    '//ul[@class="el-scrollbar__view el-select-dropdown__list"]//span[text()="20条/页"]')        if page_num != 1:            while 1:                try:                    self.entering_values('//input[@type="number"]',page_num)                    break                except:                    print('error in entering page num')
                time.sleep(3)            self.send_keys('//input[@type="number"]')            time.sleep(3)        while 1:            soup = self.make_soup()
            try:                all_results = soup.find('table', class_='el-table__body').tbody.find_all('tr', class_=re.compile(                    'el-table__row'))            except:                if soup.find('span', class_='el-table__empty-text') is not None:                    print('No Results...')                    all_results = []
            total_results = int(soup.find('span', class_='el-pagination__total').text.strip().split()[1])            ending_page = total_results // 20 + 1
            while 1:
                # sometimes it takes long to load all the records on the page, so here making sure we loaded all 20 records or                # if not then making usre it's the last page                soup = self.make_soup()                all_results = soup.find('table', class_='el-table__body').tbody.find_all('tr', class_=re.compile(                    'el-table__row'))                if len(all_results) == 20:                    break                if len(all_results) < 20 and ending_page == page_num:                    break                print(all_results, " : ", total_results, " : ", ending_page)                time.sleep(3)
            # each click on the site opens a new window, so here we are switching windows and then closing windows once data read.            for _result in range(len(all_results)):                result = all_results[_result].find_all('td')                if not result:                    continue
                result_title = result[1].text.strip()
                print(page_num, " : ", ending_page, " : ", _result, " / ", len(all_results), " : ",                      result_title, " : ", total_results, " : ", ending_page)
            print(f"page_num: {page_num} Done!")            page_num += 1            if page_num > ending_page:                break            next_button = soup.find('button', class_='btn-next').attrs            if 'disabled' in next_button:                break            self.clicking('//button[@class="btn-next"]')            time.sleep(3)        # self.close_handles(second_handle, self.main_page_handle)



if __name__ == '__main__':    REY_NUM = 5    next_year = datetime.now().year + 1    url = r'https://www.nmpa.gov.cn/datasearch/search-result.html'    # with Display(visible=0, size=(1920, 1080)) as display:    for _ in range(REY_NUM):        try:            handle = INTERFACING()            soup = handle.get_selenium_response(url)            handle.skip_button("introjs-skipbutton")
            soup = handle.make_soup()
            if soup.find('div', class_='header-main') is None:                print("访问失败！")
            main_page_handle = handle.get_current_handle()
            count = 0            while 1:                soup = handle.make_soup()
                try:                    handle.clicking('//span[text()="Medical Devices"]')                    break                except Exception as error:                    print('Medical button not yet visible')
                try:                    all_results = soup.find('table', class_='el-table__body').tbody.find_all('tr',class_=re.compile('el-table__row'))                    break                except:                    pass
                response = handle.skip_button('introjs-skipbutton')
                if response:                    break
                time.sleep(1)
                count += 1
                if count >= 5:                    break            handle.clicking("//*[@class='pc-max el-row']/div/a[@title='一次性使用医疗器械产品']")            for device_type in ["械备", "注进", "注准"]:                for year in range(2020, 2022):                    current_query = f'{device_type}{year}'                    handle.search_data(current_query,1)                    # print(f'{device_type}{_year}')


        except Exception as e:            print(f'爬取NMPADisposableProductsRequester数据失败: 详情{e}')
        handle.close_driver()        time.sleep(60)    else:        raise Exception(            f"已经重试{REY_NUM}次, 爬取NMPADisposableProductsRequester数据失败, 详情{e}")

标签：handle,skipbutton,chrome,skip,button,self,soup,options,page
From： https://www.cnblogs.com/avivi/p/17458068.html