从国家统计局网站爬取省级到村级别的行政区划代码（2022最新）

import cn.hutool.http.HttpUtil;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.Charset;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;


/**
 * 从国家统计局网站爬取省级到村级别的行政区划代码
 */
public class Area {
    public static final String baseUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2022/";
    //设置utf-8  2020版需要用GBK
    public static final String CHARSET = "utf-8";

    public static StringBuffer result = new StringBuffer();


    /**
     * 读省的信息
     *
     * @param args
     * @throws Exception
     */
    public static void main(String[] args) throws Exception {
        String url = baseUrl + "index.html";
        //如果需要设置代理
        //initProxy("10.10.13.200", "80");
//        String str = getContent(url).toUpperCase();
        String str = HttpUtil.get(url).toUpperCase();
        String[] arrs = str.split("<A");

        for (String s : arrs) {
            if (s.indexOf("HREF") != -1 && s.indexOf(".HTML") != -1) {
                //2020版用的是 ' ，不是 \"，BR后面没有空格，市县同理
                String a = s.substring(7, s.indexOf("\">"));
                String areaid = a.replace(".HTML", "") + "0000";
                String name = s.substring(s.indexOf("\">") + 2, s.indexOf("<BR />"));
                saveCity(areaid, name, "0", 1);
                System.out.println("爬取:" + areaid + "---" + name);
                boolean success = false;
                while (!success) {
                    try {
                        readShi(a, areaid);
                        success = true;
                    } catch (Exception e) {
                        e.printStackTrace();
                        success = false;
                        Thread.sleep(60000);
                    }
                }

            }
        }
    }

    /**
     * 读市的数据
     *
     * @param -list
     * @throws Exception
     */
    public static void readShi(String url, String paretid) throws Exception {
        String content = getContent(baseUrl + url).toUpperCase();
        String[] citys = content.split("CITYTR");
        //\"><TD><A HREF=\"11/1101.HTML\">110100000000</A></TD><TD><A HREF=\"11/1101.HTML\">市辖区</A></TD></td><TR CLASS=\"
        for (int c = 1, len = citys.length; c < len; c++) {
            String[] strs = citys[c].split("<A HREF=\"");
            String cityUrl = null;
            String areaid = "";
            String areaname = "";
            for (int si = 1; si < 3; si++) {
                if (si == 1) {//取链接和编码
                    cityUrl = strs[si].substring(0, strs[si].indexOf("\">"));
                    areaid = strs[si].substring(strs[si].indexOf("\">") + 2, strs[si].indexOf("</A>"));
                } else {
                    areaname = strs[si].substring(strs[si].indexOf("\">") + 2, strs[si].indexOf("</A>"));
                    System.out.println("爬取:" + strs[si].substring(strs[si].indexOf("\">") + 2, strs[si].indexOf("</A>")));
                }
            }
            saveCity(areaid, areaname, paretid, 2);
            boolean success = false;
            while (!success) {
                try {
                    readXian(cityUrl.substring(0, cityUrl.indexOf("/") + 1), cityUrl, areaid);
                    success = true;
                } catch (IOException e) {
                    e.printStackTrace();
                    success = false;
                    Thread.sleep(60000);
                }
            }
        }
    }

    /**
     * 读县的数据
     *
     * @param url
     * @throws Exception
     */
    public static void readXian(String prix, String url, String paretid) throws Exception {
        String content = getContent(baseUrl + url).toUpperCase();
        String[] citys = content.split("COUNTYTR");
        for (int i = 1; i < citys.length; i++) {
            String cityUrl = null;
            String areaid = "";
            String areaname = "";
            //发现石家庄有一个县居然没超链接，特殊处理
            if (citys[i].indexOf("<A HREF=\"") == -1) {
                areaid = citys[i].substring(6, 18);
                areaname = citys[i].substring(citys[i].indexOf("</TD><TD>") + 9, citys[i].lastIndexOf("</TD>"));
            } else {
                String[] strs = citys[i].split("<A HREF=\"");
                for (int si = 1; si < 3; si++) {
                    if (si == 1) {//取链接和编码
                        cityUrl = strs[si].substring(0, strs[si].indexOf("\">"));
                        areaid = strs[si].substring(strs[si].indexOf("\">") + 2, strs[si].indexOf("</A>"));
                    } else {
                        areaname = strs[si].substring(strs[si].indexOf("\">") + 2, strs[si].indexOf("</A>"));
                    }
                }
            }
            saveCity(areaid, areaname, paretid, 3);
            boolean success = false;
            while (!success) {
                try {
                    if (null != cityUrl) {
                        readZhen(prix, cityUrl, areaid);
                    }
                    success = true;
                } catch (IOException e) {
                    e.printStackTrace();
                    success = false;
                    Thread.sleep(60000);
                }
            }
        }
    }

    /**
     * 读镇的数据
     *
     * @param url
     * @throws Exception
     */
    public static void readZhen(String prix, String url, String paretid) throws Exception {
        String content = getContent(baseUrl + prix + url).toUpperCase();
        String myPrix = (prix + url).substring(0, (prix + url).lastIndexOf("/") + 1);
        String[] citys = content.split("TOWNTR");
        for (int i = 1; i < citys.length; i++) {
            String[] strs = citys[i].split("<A HREF=\"");
            String cityUrl = null;
            String areaid = "";
            String areaname = "";
            for (int si = 1; si < 3; si++) {
                if (si == 1) {//取链接和编码
                    cityUrl = strs[si].substring(0, strs[si].indexOf("\">"));
                    areaid = strs[si].substring(strs[si].indexOf("\">") + 2, strs[si].indexOf("</A>"));

                } else {
                    areaname = strs[si].substring(strs[si].indexOf("\">") + 2, strs[si].indexOf("</A>"));
                }
            }
            saveCity(areaid, areaname, paretid, 4);
        }
    }

    /**
     * 读村/街道的数据
     *
     * @param url
     * @throws Exception
     */
    public static void readCun(String prix, String url, BufferedWriter bw) throws Exception {
        String content = getContent(baseUrl + prix + url).toUpperCase();
        String[] citys = content.split("VILLAGETR");
        for (int i = 1; i < citys.length; i++) {
            String[] strs = citys[i].split("<TD>");

            bw.write("<tr><td>");
            bw.write(strs[1].substring(0, strs[1].indexOf("</TD>")));
            bw.write("</td>");

            bw.write("<td></td><td></td><td></td><td></td>");
            bw.write("<td>");
            bw.write(strs[2].substring(0, strs[2].indexOf("</TD>")));
            bw.write("</td><td>");
            bw.write(strs[3].substring(0, strs[3].indexOf("</TD>")));
            bw.write("</td></tr>");
        }
    }


    //设置代理
    public static void initProxy(String host, String port) {
        System.setProperty("http.proxyType", "4");
        System.setProperty("http.proxyPort", port);
        System.setProperty("http.proxyHost", host);
        System.setProperty("http.proxySet", "true");
    }


    //获取网页的内容
    public static String getContent(String strUrl) throws Exception {
        try {
            URL url = new URL(strUrl);
            BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream(), Charset.forName(CHARSET)));
            String s = "";
            StringBuffer sb = new StringBuffer("");
            while ((s = br.readLine()) != null) {
                sb.append(s);
            }
            br.close();
            return sb.toString();
        } catch (Exception e) {
            System.out.println("can\"t open url:" + strUrl);
            throw e;
        }
    }


    public static void saveCity(String areaid, String areaname, String paretid, int level) {
        try {
            String URL = "jdbc:mysql://127.0.0.1:3306/test?useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC&useSSL=false";
            String USER = "root";
            String PASSWORD = "*******";
            //1.加载驱动程序
            Class.forName("com.mysql.jdbc.Driver");
            //2.获得数据库链接
            Connection conn = DriverManager.getConnection(URL, USER, PASSWORD);
            //3.通过数据库的连接操作数据库，实现增删改查（使用Statement类）
            String s = "insert into area(code,`name`,p_code,`level`) values(?,?,?,?)";
            PreparedStatement pst = conn.prepareStatement(s);

            pst.setString(1, level > 3 ? areaid.substring(0, 9) : areaid.substring(0, 6));
            pst.setString(2, areaname);
            pst.setString(3, paretid.replace("000000", ""));
            pst.setInt(4, level);

            pst.execute();
            //关闭资源
            pst.close();
            conn.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}