CMS项目实现异步图片下载

业务描述: Cms 项目是我们内部的一个管理系统主要是管理内部的文章和网站的内容,  现在有这么一个需求是:我在其它网站上复制过来的文章要保存在我们的CMS系统中,但保存的要把远程的图片地扯一起保存,这样保存会出现一个问题,一但对方服务器,断开就图片显示不了,我们团队中马上想到一个解决的办法,在保存的时候分折HTML 代码,取得远程的图片地扯,然后在把图片下载下来,最后把本地的图片地扯填到原来的HTML代码中, 这种思路不可取,因为在保存文章的时候需等到图片下载完才能保存,这样会很忙,会让运营方,无法忍受这种方式,


,需得取得网上的图片SRC 然后把图片下载下来.成为我们自己的服务器上的图片, 但如果同步实现的话,会很忙,这样操作会给运营方代来了很多不便的地方,为了解决这个问题,

  只有通过异步方式可取了,先分折HTML代码,取得SRC地扯,假设本地图片地扯存在,先填冲到HTML当中,,把假设的数据和远程的地扯保存在数据库中, 然后用线程池的方式,去读数据库中地扯下载, 这个只能在JVM起动了,就可以调用下载功能.


让运营保存的时候就不会感觉的慢了.


具体的实现如下:


   create table CMS_IMAGES
(
  id     LONG PRIMARY KEY, --id
  wsrc   varchar2(256),--远程图片地扯
  lsrc   varchar2(256),--本地图片地扯
  state NUMBER,  --此图片的状态  1未处理 2下载中 3下载失败
  last_modified  LONG --上次状态修改时间
  modified_count number--状态修改次数-缺省为0
)
  先解释一下,   3下载失败了,等两个小时,重新通线程调用一次,因为有的网站,不能让你老是在他的网站上面下载图片的,只有过些时间再请求一次,就可以下载,实在不能下载的图片,比率很小,


/**
* 下载图片的后台线程池
* @author
* @version 1.0
*/


public class ThreadPoolManager {
protected static Logger log = LoggerFactory
   .getLogger(ThreadPoolManager.class);

private static ThreadPoolManager tpm = new ThreadPoolManager();

// 线程池维护线程的最少数量
private final static int CORE_POOL_SIZE = 4;

// 线程池维护线程的最大数量
private final static int MAX_POOL_SIZE = 10;

// 线程池维护线程所允许的空闲时间
private final static int KEEP_ALIVE_TIME = 0;

// 线程池所使用的缓冲队列大小
private final static int WORK_QUEUE_SIZE = 10;

// 图片缓冲队列
private Queue msgQueue = new LinkedList();

// 访问消息缓存的调度线程
final Runnable accessBufferThread = new Runnable() {
  public void run() {
   // 查看是否有待定请求，如果有，则创建一个新的Thread，并添加到线程池中
   if (hasMoreAcquire()) {
    ImageDO image = (ImageDO) msgQueue.poll();
    Runnable task = new UploadThread(image);
    threadPool.execute(task);
   }

   if (hasImagesInDB()) {
    try {
     BeanFactoryService beanFactory = (BeanFactoryService) BizServiceManager
       .getInstance().getService(
         BeanFactoryService.SERVICE_NAME);
     ImageManager imageManager = (ImageManager) beanFactory
       .getBean("imageManager");
     List images = imageManager
       .getImages(ImageDO.STATE_READY);
     for (int i = 0; i < images.size(); i++) {
      ImageDO img = images.get(i);
      img.setState(ImageDO.STATE_RUNNING);
      img.setModifiedCount(img.getModifiedCount() + 1);
      img.setLastModified(CalendarUtil.getCurrentDate()
        .getTime());
      imageManager.updateImage(img);

      Runnable task = new UploadThread(img);
      threadPool.execute(task);
     }

    } catch (Exception e) {
     log.error("获取下载图片队列失败", e);
    }

   }

  }
};

// 用于被拒绝任务的处理程序
final RejectedExecutionHandler handler = new RejectedExecutionHandler() {
  public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
   log.error(((UploadThread) r).getImg() + "   放入队列中重新等待执行");
   msgQueue.offer(((UploadThread) r).getImg());
  }
};

// 管理数据库访问的线程池
final ThreadPoolExecutor threadPool = new ThreadPoolExecutor(
   CORE_POOL_SIZE, MAX_POOL_SIZE, KEEP_ALIVE_TIME, TimeUnit.SECONDS,
   new ArrayBlockingQueue(WORK_QUEUE_SIZE), this.handler);

// 调度线程池
final ScheduledExecutorService scheduler = Executors
   .newScheduledThreadPool(1);

final ScheduledFuture taskHandler = scheduler.scheduleAtFixedRate(
   accessBufferThread, 0, 5, TimeUnit.SECONDS);

public static ThreadPoolManager newInstance() {
  return tpm;
}

private ThreadPoolManager() {
}

private boolean hasMoreAcquire() {
  return !msgQueue.isEmpty();
}

private boolean hasImagesInDB() {
  boolean result = false;
  try {
   BeanFactoryService beanFactory = (BeanFactoryService) BizServiceManager
     .getInstance().getService(BeanFactoryService.SERVICE_NAME);
   if (beanFactory == null)
    return false;
   ImageManager imageManager = (ImageManager) beanFactory
     .getBean("imageManager");
   if (imageManager == null)
    return false;
   int imagesCount = imageManager.getImagesCount(ImageDO.STATE_READY);
   result = (imagesCount != 0);
  } catch (Exception e) {
   log.error("获取下载图片队列失败", e);
  }
  return result;
}

public void processImg(ImageDO img) {
  Runnable task = new UploadThread(img);
  threadPool.execute(task);
}


public long getCompleteTasksCount() {
  return this.threadPool.getCompletedTaskCount();
}

public long getActiveTasksCount() {
  return this.threadPool.getActiveCount();
}

public long getAllTasksCount() {
  return this.threadPool.getTaskCount();
}

public long getQuequeCount() {
  return this.msgQueue.size();
}

public boolean getRunningStatus() {
  return !this.threadPool.isShutdown();
}

public long getScheduledQuequeSize() {
  return ((ScheduledThreadPoolExecutor)this.scheduler).getQueue().size();
}

}



public class UploadThread implements Runnable {
private ImageDO img;

protected Logger log = LoggerFactory.getLogger(UploadThread.class);

public ImageDO getImg() {
  return img;
}

public void setImg(ImageDO img) {
  this.img = img;
}

public UploadThread() {
  super();
}

public UploadThread(ImageDO img) {
  this.img = img;
}

public void run() {
  BeanFactoryService beanFactory = (BeanFactoryService) BizServiceManager
    .getInstance().getService(BeanFactoryService.SERVICE_NAME);
  ImageManager imageManager = (ImageManager) beanFactory
    .getBean("imageManager");
  if (UploadUtil.upload(img)) {
   try {
    log.error("下載圖片成功:"+ img);
    imageManager.deleteImageByid(img.getId());
   } catch (ManagerException e) {
    log.error("删除图片记录失败:"+ img.getId(), e);
   }
  } else {
   img.setState(ImageDO.STATE_FAIL);
   try {
    imageManager.updateImage(img);
   } catch (ManagerException e) {
    log.error("设置图片下载状态失败:"+ img.getId(), e);
   }
  }
}

}
public class UploadUtil {
private static Logger logger = Logger.getLogger(UploadUtil.class);
private static FileStorage attachmentFileStorage;

private static String imgServer;

/**
  * @param imgServer
  *            the imgServer to set
  */
public void setImgServer(String imgServer) {
  UploadUtil.imgServer = imgServer;
}

public static String getUrlPrefix() {
  return imgServer;
}

public static boolean upload(ImageDO image) {
  String url = image.getWsrc();
   HttpMethod httpMethod = getHttpMethod(url);
   ByteArrayDataSource ds = new ByteArrayDataSource(httpMethod
     .getResponseBodyAsStream(), "image/jpeg");
   ds.setName("upload.jpg");
   boolean result = attachmentFileStorage.save(ds, image.getLsrc());
   httpMethod.releaseConnection();
   return result;
  } catch (Exception e) {
   logger.error("upload image fail:" + url, e);
  }
  return false;
}

private static HttpMethod getHttpMethod(String url) throws IOException {
  HttpMethod httpMethod = new GetMethod(url);
  httpMethod.setRequestHeader("Cache-Control", "no-cache");
  httpMethod.setRequestHeader("Accept-Language", "zh-cn");
  HttpClient clientTemp = new HttpClient(); // HttpClient创建
  HttpClientParams clientParams = clientTemp.getParams();
  clientParams.setParameter("http.socket.timeout", 5000); // 5秒socket等待数据
  clientParams.setParameter("http.connection.timeout", 5000); // 5秒http
  // connection建立超时
  clientParams.setParameter("http.connection-manager.timeout", 5000L); // 5秒从http
  // yahoo相册的图片需要设置Referer头
  if (url.toLowerCase().indexOf("yahoo") != -1) {
   httpMethod.addRequestHeader("Referer", "http://forum.taobao.com/");
  }
  // connection
  // manager获取可用的Http
  // connection超时
  clientParams.setParameter("http.method.retry-handler",
    new DefaultHttpMethodRetryHandler()); // 如果Http出错，三次重试
  clientTemp.executeMethod(httpMethod);
  return httpMethod;
}

public void setAttachmentFileStorage(FileStorage attachmentFileStorage) {
  UploadUtil.attachmentFileStorage = attachmentFileStorage;
}

public static void main(String[] args) throws Exception {
  HttpMethod httpMethod = getHttpMethod("http://cn.photos.yahoo.com/users/47df51c0z8a48b60a/255c/__sr_/dbce.jpg?wa86dTAyD7Rq05sklm3fMQ--&amp;F18");
  InputStream inputStream = httpMethod.getResponseBodyAsStream();
  FileOutputStream os = new FileOutputStream("c:/test.jpg");

  IOUtils.copy(inputStream, os);
  os.close();

  httpMethod.releaseConnection();
}
}


public class HtmlParseUtil {
protected static Logger log = LoggerFactory.getLogger(HtmlParseUtil.class);

public static String processImg(String imageURL) {
  if (imageURL.toLowerCase().indexOf("taobao.com") == -1) {
   BeanFactoryService beanFactory = (BeanFactoryService) BizServiceManager
     .getInstance().getService(BeanFactoryService.SERVICE_NAME);
   ImageManager imageManager = (ImageManager) beanFactory
     .getBean("imageManager");
   FileStorage fileStorage = (FileStorage) beanFactory
     .getBean("attachmentFileStorage");

   // 产生本地扯的图片文件名
   String localFileName = FileNameGenerater
     .getLocalFileName("upload.jpg");

   ImageDO image = new ImageDO();
   image.setWsrc(imageURL);
   image.setState(ImageDO.STATE_READY);
   image.setLsrc(localFileName);
   image.setLastModified(CalendarUtil.getCurrentDate().getTime());

   try {
    imageManager.insertImage(image);
   
    log.error("新增图片下载:" + image);
    ThreadPoolManager tpm = ThreadPoolManager.newInstance();
    tpm.processImg(image);

    return UploadUtil.getUrlPrefix() + fileStorage.getNamespace()
      + "/" + localFileName;
   } catch (ManagerException e) {
    log.error("新增图片下载失败:" + imageURL, e);
   }
  }
  return imageURL;
}

}

看看大家有没有更好的方式,设计出JVM启动就每30秒,就会去用线程去读数据库呢.有没有更好的设计思路