提取申请人和产品名称

import random
import re
import os
import time
import pandas as pd
from pyvirtualdisplay import Display
import undetected_chromedriver as uc
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup


class INTERFACING():

    def __init__(self):
        self.driver_initialized = False
        self.driver = ''
        self.MAX_TRIALS = 2
        self.chrome_version =get_google_chrome_version()

    def make_soup(self):
        return BeautifulSoup(self.driver.page_source, 'lxml')  # etree.HTML()

    def current_url(self):
        return self.driver.current_url

    def get_driver(self):

        uc.TARGET_VERSION = get_google_chrome_version()
        chrome_options = uc.ChromeOptions()
        # chrome_options.add_argument("--headless")
        chrome_options.add_argument("--window-size=1920.,1080")
        chrome_options.add_argument("--disable-extensions")
        chrome_options.add_argument("--disable-dev-shm-usage")
        chrome_options.add_argument("--disable-popup-blocking")
        chrome_options.add_argument("--profile-directory=Default")
        chrome_options.add_argument("--ignore-certificate-errors")
        chrome_options.add_argument("--disable-plugins-discovery")
        chrome_options.add_argument("--incognito")
        chrome_options.add_argument("--no-first-run")
        chrome_options.add_argument("--no-service-autorun")
        chrome_options.add_argument("--no-default-browser-check")
        chrome_options.add_argument("--password-store=basic")
        chrome_options.add_argument("--no-sandbox")
        chrome_options.add_argument('--disable-application-cache')
        chrome_options.add_argument('--disable-gpu')
        chrome_options.add_argument("--disable-setuid-sandbox")
        chrome_options.add_argument(
            "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36"
        )
        self.driver = uc.Chrome(options=chrome_options, version_main=get_google_chrome_version())
        time.sleep(10)
        self.driver_initialized = True

    def close_driver(self):
        self.driver.quit()

    def get_selenium_response(self, url):

        # try:
        if not self.driver_initialized:
            self.get_driver()
        else:
            pass
        self.driver.get(url)
        time.sleep(3)
        soup = self.make_soup()
        return soup

    def get_page_source(self):
        return self.driver.page_source

    def clicking(self, xpath):
        elem = self.driver.find_element(By.XPATH, xpath)
        elem.click()
        time.sleep(random.randint(2, 3))

    def entering_values(self, xpath, value):
        elem = self.driver.find_element(By.XPATH, xpath)
        elem.clear()
        elem.send_keys(value)
        time.sleep(random.randint(2, 4))

    def send_keys(self, xpath):
        elem = self.driver.find_element(By.XPATH, xpath).send_keys(Keys.RETURN)

    def going_back(self):
        self.driver.execute_script("window.history.go(-1)")
        time.sleep(1)

    def refresh_page(self):
        self.driver.refresh()

    def close_handle(self):
        self.driver.close()

    def get_current_handle(self):
        return self.driver.current_window_handle

    def get_all_handles(self):
        return self.driver.window_handles

    def swtich_to_window(self, handle):
        self.driver.switch_to.window(handle)


def get_google_chrome_version():
    try:
        search_pattern = "(\d+?)\."
        chrome_version = os.popen("google-chrome -version").read()
        chrome_version = re.search(search_pattern, chrome_version).group(1)
        return chrome_version
    except Exception as e:
        raise Exception(f"获取google chrome的版本错误，详情：{e}")


def extract_product_applicant(soup):
    product_pattern = re.compile(r'产品名称：\s*([^<>]+)')
    applicant_pattern = re.compile(r'申\s*请\s*人：\s*([^<>]+)')

    product_matches = soup(text=product_pattern)
    applicant_matches = soup(text=applicant_pattern)

    products = [product_pattern.findall(match)[0] for match in product_matches]
    applicants = [applicant_pattern.findall(match)[0] for match in applicant_matches]
    if not products:
        product_pattern = re.compile("产品名称<\/strong>：\s*([^<>]+)")
        applicant_pattern = re.compile("申\s*请\s*人<\/strong>：\s*([^<>]+)")

        product_matches = soup(text=product_pattern)
        applicant_matches = soup(text=applicant_pattern)

        products = [product_pattern.findall(match)[0] for match in product_matches]
        applicants = [applicant_pattern.findall(match)[0] for match in applicant_matches]

    return list(zip(products, applicants))


base_url = "https://www.cmde.org.cn"
result_df = pd.DataFrame(columns=["index", "time", "product_name", "product_company", "ID", "reason"])
num = -1
with Display(visible=False, size=(1920, 1080)) as display:
    for page in range(1, 10):
        if page == 1:
            soup = INTERFACING().get_selenium_response("https://www.cmde.org.cn/xwdt/shpgzgg/cxyxgsh/index.html")
        else:
            soup = INTERFACING().get_selenium_response(
                f"https://www.cmde.org.cn/xwdt/shpgzgg/cxyxgsh/index_{page - 1}.html")

        groups_selector = soup.select('.list > ul >li')
        for index, group in enumerate(groups_selector[17:]):
            group_href = group.find('a')['href']
            update_time = group.find('span').text[1:-1]
            group_href = group_href if "http" in group_href else base_url + "/" + group_href.replace("../", "")
            detail_soup = INTERFACING().get_selenium_response(group_href)
            if not detail_soup:
                print(f"第{page}页,第{index}条没有抓取到")
                continue

            data = extract_product_applicant(detail_soup)
            if not data:
                # 找到表格
                table = detail_soup.find('table')
                if not table:
                    print(f"!!!!!!!!!!!!!!!!!!!!!!第{page}页,第{index}条没有抓取到")
                    continue
                # 提取表格数据
                rows = table.find_all('tr')
                data = []
                for row in rows[1:]:
                    cells = row.find_all('td')
                    values = [cell.text.strip() for cell in cells]
                    data.append(values)

                # 提取表头
                header_cells = rows[0].find_all('td')
                headers = [cell.text.strip() for cell in header_cells]

                # 将数据转换为DataFrame
                df = pd.DataFrame(data, columns=headers)

                # 输出结果
                print(df)
                for i in range(len(df)):
                    num += 1
                    result_df.loc[num, "index"] = num
                    result_df.loc[num, "time"] = update_time
                    result_df.loc[num, "product_name"] = df.loc[i, '产品名称']
                    result_df.loc[num, "product_company"] = df.loc[i, '申请人']
                    result_df.loc[num, "ID"] = df.loc[i, '受理号']
                    result_df.loc[num, "reason"] = df.loc[i, '同意理由']
                result_df.to_excel("result.xlsx")
                print(f"第{page}页,第{index}条抓取完成")
                continue

            for i, (product_name, applicant) in enumerate(data):
                num += 1
                result_df.loc[num, "index"] = num
                result_df.loc[num, "time"] = update_time
                result_df.loc[num, "product_name"] = product_name
                result_df.loc[num, "product_company"] = applicant
            result_df.to_excel(f"result_1_4.xlsx")

            print(f"第{page}页,第{index}条抓取完成")