webmagic有一个selenium模块,其中实现了一个SeleniumDownloader。但是感觉灵活性不大。所以我就自己参考实现了一个。
  首先是WebDriverPool用来管理WebDriver池:
   import java.util.ArrayList;
  import java.util.concurrent.BlockingDeque;
  import java.util.concurrent.LinkedBlockingDeque;
  import java.util.concurrent.TimeUnit;
  import java.util.concurrent.atomic.AtomicInteger;
  import org.openqa.selenium.WebDriver;
  import org.openqa.selenium.phantomjs.PhantomJSDriver;
  import org.openqa.selenium.phantomjs.PhantomJSDriverService;
  import org.openqa.selenium.remote.DesiredCapabilities;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  import net.xby1993.common.util.FileUtil;
  /**
  * @author taojw
  */
  public class WebDriverPool {
  private Logger logger = LoggerFactory.getLogger(getClass());
  private int CAPACITY = 5;
  private AtomicInteger refCount = new AtomicInteger(0);
  private static final String DRIVER_PHANTOMJS = "phantomjs";
  /**
  * store webDrivers available
  */
  private BlockingDeque<WebDriver> innerQueue = new LinkedBlockingDeque<WebDriver>(
  CAPACITY);
  private static String PHANTOMJS_PATH;
  private static DesiredCapabilities caps = DesiredCapabilities.phantomjs();
  static {
  PHANTOMJS_PATH = FileUtil.getCommonProp("phantomjs.path");
  caps.setJavascriptEnabled(true);
  caps.setCapability(
  PhantomJSDriverService.PHANTOMJS_EXECUTABLE_PATH_PROPERTY,
  PHANTOMJS_PATH);
  caps.setCapability("takesScreenshot", false);
  caps.setCapability(
  PhantomJSDriverService.PHANTOMJS_PAGE_CUSTOMHEADERS_PREFIX
  + "User-Agent",
  "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36");
  ArrayList<String> cliArgsCap = new ArrayList<String>();
  //http://phantomjs.org/api/command-line.html
  cliArgsCap.add("--web-security=false");
  cliArgsCap.add("--ssl-protocol=any");
  cliArgsCap.add("--ignore-ssl-errors=true");
  cliArgsCap.add("--load-images=false"); //不加载图片
  caps.setCapability(PhantomJSDriverService.PHANTOMJS_CLI_ARGS,
  cliArgsCap);
  caps.setCapability(
  PhantomJSDriverService.PHANTOMJS_GHOSTDRIVER_CLI_ARGS,
  new String[] {"--logLevel=INFO"});
  }
  public WebDriverPool() {
  }
  public WebDriverPool(int poolsize) {
  this.CAPACITY = poolsize;
  innerQueue = new LinkedBlockingDeque<WebDriver>(poolsize);
  }
  public WebDriver get() throws InterruptedException {
  WebDriver poll = innerQueue.poll();
  if (poll != null) {
  return poll;
  }
  if (refCount.get() < CAPACITY) {
  synchronized (innerQueue) {
  if (refCount.get() < CAPACITY) {
  WebDriver mDriver = new PhantomJSDriver(caps);
  // 尝试性解决:https://github.com/ariya/phantomjs/issues/11526问题
  mDriver.manage().timeouts()
  .pageLoadTimeout(60, TimeUnit.SECONDS);
  // mDriver.manage().window().setSize(new Dimension(1366,
  // 768));
  innerQueue.add(mDriver);
  refCount.incrementAndGet();
  }
  }
  }
  return innerQueue.take();
  }
  public void returnToPool(WebDriver webDriver) {
  // webDriver.quit();
  // webDriver=null;
  innerQueue.add(webDriver);
  }
  public void close(WebDriver webDriver) {
  refCount.decrementAndGet();
  webDriver.quit();
  webDriver = null;
  }
  public void shutdown() {
  try {
  for (WebDriver driver : innerQueue) {
  close(driver);
  }
  innerQueue.clear();
  } catch (Exception e) {
  //            e.printStackTrace();
  logger.warn("webdriverpool关闭失败",e);
  }
  }
  }
  之后便是SeleniumDownloader
   import org.openqa.selenium.By;
  import org.openqa.selenium.Cookie;
  import org.openqa.selenium.WebDriver;
  import org.openqa.selenium.WebElement;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  import us.codecraft.webmagic.Page;
  import us.codecraft.webmagic.Request;
  import us.codecraft.webmagic.Site;
  import us.codecraft.webmagic.Task;
  import us.codecraft.webmagic.downloader.Downloader;
  import us.codecraft.webmagic.selector.Html;
  import us.codecraft.webmagic.selector.PlainText;
  import us.codecraft.webmagic.utils.UrlUtils;
  import java.util.Map;
  /**
  * @author taojw
  *
  */
  public class SeleniumDownloader  implements Downloader{
  private static final Logger log=LoggerFactory.getLogger(SeleniumDownloader.class);
  private int sleepTime=3000;//3s
  private SeleniumAction action=null;
  private WebDriverPool webDriverPool=new WebDriverPool();
  public SeleniumDownloader(){
  }
  public SeleniumDownloader(int sleepTime,WebDriverPool pool){
  this(sleepTime,pool,null);
  }
  public SeleniumDownloader(int sleepTime,WebDriverPool pool,SeleniumAction action){
  this.sleepTime=sleepTime;
  this.action=action;
  if(pool!=null){
  webDriverPool=pool;
  }
  }
  public SeleniumDownloader setSleepTime(int sleepTime) {
  this.sleepTime = sleepTime;
  return this;
  }
  public void setOperator(SeleniumAction action){
  this.action=action;
  }
  @Override
  public Page download(Request request, Task task) {
  WebDriver webDriver;
  try {
  webDriver = webDriverPool.get();
  } catch (InterruptedException e) {
  log.warn("interrupted", e);
  return null;
  }
  log.info("downloading page " + request.getUrl());
  Page page = new Page();
  try {
  webDriver.get(request.getUrl());
  Thread.sleep(sleepTime);
  } catch (InterruptedException e) {
  e.printStackTrace();
  } catch (Exception e) {
  webDriverPool.close(webDriver);
  page.setSkip(true);
  return page;
  }
  //        WindowUtil.changeWindow(webDriver);
  WebDriver.Options manage = webDriver.manage();
  Site site = task.getSite();
  if (site.getCookies() != null) {
  for (Map.Entry<String, String> cookieEntry : site.getCookies()
  .entrySet()) {
  Cookie cookie = new Cookie(cookieEntry.getKey(),
  cookieEntry.getValue());
  manage.addCookie(cookie);
  }
  }
  manage.window().maximize();
  if(action!=null){
  action.execute(webDriver);
  }
  SeleniumAction reqAction=(SeleniumAction) request.getExtra("action");
  if(reqAction!=null){
  reqAction.execute(webDriver);
  }
  WebElement webElement = webDriver.findElement(By.xpath("/html"));
  String content = webElement.getAttribute("outerHTML");
  page.setRawText(content);
  page.setHtml(new Html(UrlUtils.fixAllRelativeHrefs(content,
  webDriver.getCurrentUrl())));
  page.setUrl(new PlainText(webDriver.getCurrentUrl()));
  page.setRequest(request);
  webDriverPool.returnToPool(webDriver);
  return page;
  }
  @Override
  public void setThread(int thread) {
  }
  }
  这里的扩展性主要体现在,我加入了SeleniumAction接口,可以在SeleniumDownloader初始化的时候配置一个全局的SeleniumAction,以及为每个Request配置对应的SeleniumAction。 SeleniumAction接口如下:
   public interface SeleniumAction {
  void execute(WebDriver driver);
  }
  它会获得一个WebDriver实例,你可以在里面进行任意的Selenium操作。
      上文内容不用于商业目的,如涉及知识产权问题,请权利人联系博为峰小编(021-64471599-8052),我们将立即处理。
 
 
 
 
 
 
      了解更多课程内容及课程安排,可咨询QQ 2852509883 或致电客服 400-821-0951(工作日9:00-17:30)
【看这里】技术交流、拓展人脉、领取福利欢迎加入博为峰网校大课堂>>>