如何在Java语言中使用爬虫ip

企业客户做大数据抓取都会用到爬虫IP，质量好的爬虫IP可以让爬虫工作事半功倍，如何高效的爬取目标数据就显得尤为重要。影响这样的结果不仅仅是因为爬虫IP问题，还有可能是技术在写代码时候的优化问题。下文是有关使用java语言的代码示例可以一起看看。

Java HttpURLConnection

package com.qgproxy;

  import java.io.ByteArrayOutputStream;

  import java.io.InputStream;

  import java.net.Authenticator;

  import java.net.HttpURLConnection;

  import java.net.InetSocketAddress;

  import java.net.PasswordAuthentication;

  import java.net.Proxy;

  import java.net.URL;

  class QGProxyAuthenticatorg extends Authenticator {

    private String user, password;

    public QGProxyAuthenticator(String user, String password) {

      this.user = user;

      this.password = password;

    }

    protected PasswordAuthentication getPasswordAuthentication() {

      return new PasswordAuthentication(user, password.toCharArray());

    }

  }

  class QGProxy {

    public static void main(String args[]) {

      String targetUrl = "http://jshk.com.cn";

      String proxyIp = "219.151.125.106";

      int proxyPort = 31615;

      String authKey = "895314XY";

      String password = "24D6YB309ZCB";

      try {

        URL url = new URL(targetUrl);

        Authenticator.setDefault(new QGProxyAuthenticator(authKey, password));

        InetSocketAddress socketAddress = new InetSocketAddress(proxyIp, proxyPort);

        Proxy proxy = new Proxy(Proxy.Type.HTTP, socketAddress);

        HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);

        byte[] response = readStream(connection.getInputStream());

        System.out.println(new String(response));

      } catch (Exception e) {

        System.out.println(e.getLocalizedMessage());

      }

    }

    public static byte[] readStream(InputStream inStream) throws Exception {

      ByteArrayOutputStream outSteam = new ByteArrayOutputStream();

      byte[] buffer = new byte[1024];

      int len = -1;

      while ((len = inStream.read(buffer)) != -1) {

        outSteam.write(buffer, 0, len);

      }

      outSteam.close();

      inStream.close();

      return outSteam.toByteArray();

    }

  }

Java okhttp

package com.qgproxy;

import okhttp3.*;

import java.io.IOException;

import java.net.InetSocketAddress;

import java.net.Proxy;

import java.util.concurrent.TimeUnit;

public class QGProxy {

    final static String proxyIp = "219.151.125.106";

    final static Integer proxyPort = 31615;

    final static String authKey = "895314XY";

    final static String password = "24D6YB309ZCB";

    static OkHttpClient client;

    static {

    Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, proxyPort));

        Authenticator proxyAuthenticator = (route, response) -> {

            String credential = Credentials.basic(authKey, password);

            return response.request().newBuilder().header("Proxy-Authorization", credential).build();

        };

        client = new OkHttpClient().newBuilder()

      .connectTimeout(10, TimeUnit.SECONDS)

      .readTimeout(10, TimeUnit.SECONDS)

      .proxy(proxy)

      .proxyAuthenticator(proxyAuthenticator)

      .connectionPool(new ConnectionPool(4, 2, TimeUnit.SECONDS))

      .build();

  }

    public static void main(String[] args) throws IOException {

        Request request = new Request.Builder().url("http://jshk.com.cn").build();

        Response response = client.newCall(request).execute();

        System.out.println(response.body().string());

    }

}