PageRenderTime 37ms CodeModel.GetById 17ms RepoModel.GetById 0ms app.codeStats 0ms

/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/JsonFilePageModelPipeline.java

https://gitlab.com/taichu/webmagic
Java | 56 lines | 38 code | 7 blank | 11 comment | 2 complexity | ceccdef63cdefc8dff4c53ee8f7d7d0d MD5 | raw file
  1. package us.codecraft.webmagic.pipeline;
  2. import com.alibaba.fastjson.JSON;
  3. import org.apache.commons.codec.digest.DigestUtils;
  4. import org.apache.commons.lang3.builder.ToStringBuilder;
  5. import org.slf4j.Logger;
  6. import org.slf4j.LoggerFactory;
  7. import us.codecraft.webmagic.Task;
  8. import us.codecraft.webmagic.model.HasKey;
  9. import us.codecraft.webmagic.utils.FilePersistentBase;
  10. import java.io.FileWriter;
  11. import java.io.IOException;
  12. import java.io.PrintWriter;
  13. /**
  14. * Store results objects (page models) to files in JSON format.<br>
  15. * Use model.getKey() as file name if the model implements HasKey.<br>
  16. * Otherwise use SHA1 as file name.
  17. *
  18. * @author code4crafter@gmail.com <br>
  19. * @since 0.2.0
  20. */
  21. public class JsonFilePageModelPipeline extends FilePersistentBase implements PageModelPipeline {
  22. private Logger logger = LoggerFactory.getLogger(getClass());
  23. /**
  24. * new JsonFilePageModelPipeline with default path "/data/webmagic/"
  25. */
  26. public JsonFilePageModelPipeline() {
  27. setPath("/data/webmagic/");
  28. }
  29. public JsonFilePageModelPipeline(String path) {
  30. setPath(path);
  31. }
  32. @Override
  33. public void process(Object o, Task task) {
  34. String path = this.path + PATH_SEPERATOR + task.getUUID() + PATH_SEPERATOR;
  35. try {
  36. String filename;
  37. if (o instanceof HasKey) {
  38. filename = path + ((HasKey) o).key() + ".json";
  39. } else {
  40. filename = path + DigestUtils.md5Hex(ToStringBuilder.reflectionToString(o)) + ".json";
  41. }
  42. PrintWriter printWriter = new PrintWriter(new FileWriter(getFile(filename)));
  43. printWriter.write(JSON.toJSONString(o));
  44. printWriter.close();
  45. } catch (IOException e) {
  46. logger.warn("write file error", e);
  47. }
  48. }
  49. }