diff --git a/README.en.md b/README.en.md index c572c23c..48ab5a82 100644 --- a/README.en.md +++ b/README.en.md @@ -57,15 +57,13 @@ Considering space issues, the pictures of other types of documents will not be s - Redisson - Jodconverter > Dependencies -- Redis +- Redis(Optional, Unnecessary by default) - OpenOffice or LibreOffice 1. First step:`git pull https://github.com/kekingcn/file-online-preview.git` 2. Second step:configure redis address and OpenOffice directory,such as ``` -#=============================================#Spring Redisson Configuration#===================================# -spring.redisson.address = 192.168.1.204:6379 ##The folder for files which are uploaded to the server(Because of running as jar) file.dir = C:\\Users\\yudian\\Desktop\\dev\\ ## openoffice configuration diff --git a/README.md b/README.md index 639f6e99..d0f4df40 100644 --- a/README.md +++ b/README.md @@ -50,15 +50,13 @@ QQ群号:613025121 - redisson - jodconverter > 依赖外部环境 -- redis +- redis (可选,默认不用) - OpenOffice或者LibreOffice 1. 第一步:pull项目https://github.com/kekingcn/file-online-preview.git -2. 第二步:配置redis地址和OpenOffice目录,如 +2. 第二步:配置OpenOffice目录,如 ``` -#=============================================#spring Redisson配置#===================================# -spring.redisson.address = 192.168.1.204:6379 ##资源映射路径(因为jar方式运行的原因) file.dir = C:\\Users\\yudian\\Desktop\\dev\\ ## openoffice相关配置 diff --git a/jodconverter-web/pom.xml b/jodconverter-web/pom.xml index 25e66eb0..56ea74d4 100644 --- a/jodconverter-web/pom.xml +++ b/jodconverter-web/pom.xml @@ -149,6 +149,11 @@ guava 19.0 + + com.googlecode.concurrentlinkedhashmap + concurrentlinkedhashmap-lru + 1.4.2 + diff --git a/jodconverter-web/src/main/java/cn/keking/config/RedissonConfig.java b/jodconverter-web/src/main/java/cn/keking/config/RedissonConfig.java index 78aa7861..2a3d7d27 100644 --- a/jodconverter-web/src/main/java/cn/keking/config/RedissonConfig.java +++ b/jodconverter-web/src/main/java/cn/keking/config/RedissonConfig.java @@ -1,10 +1,9 @@ package cn.keking.config; import io.netty.channel.nio.NioEventLoopGroup; -import org.redisson.Redisson; -import org.redisson.api.RedissonClient; import org.redisson.client.codec.Codec; import org.redisson.config.Config; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -14,6 +13,7 @@ import org.springframework.util.ClassUtils; * Created by kl on 2017/09/26. * redisson 客户端配置 */ +@ConditionalOnExpression("'${cache.type:default}'.equals('redis')") @ConfigurationProperties(prefix = "spring.redisson") @Configuration public class RedissonConfig { @@ -42,8 +42,8 @@ public class RedissonConfig { private String codec="org.redisson.codec.JsonJacksonCodec"; - @Bean(destroyMethod = "shutdown") - RedissonClient redisson() throws Exception { + @Bean + Config config() throws Exception { Config config = new Config(); config.useSingleServer().setAddress(address) .setConnectionMinimumIdleSize(connectionMinimumIdleSize) @@ -69,7 +69,7 @@ public class RedissonConfig { config.setThreads(thread); config.setEventLoopGroup(new NioEventLoopGroup()); config.setUseLinuxNativeEpoll(false); - return Redisson.create(config); + return config; } public int getThread() { diff --git a/jodconverter-web/src/main/java/cn/keking/service/FileConverQueueTask.java b/jodconverter-web/src/main/java/cn/keking/service/FileConverQueueTask.java index 59a05e4c..a687c41e 100644 --- a/jodconverter-web/src/main/java/cn/keking/service/FileConverQueueTask.java +++ b/jodconverter-web/src/main/java/cn/keking/service/FileConverQueueTask.java @@ -2,9 +2,8 @@ package cn.keking.service; import cn.keking.model.FileAttribute; import cn.keking.model.FileType; +import cn.keking.service.cache.CacheService; import cn.keking.utils.FileUtils; -import org.redisson.api.RBlockingQueue; -import org.redisson.api.RedissonClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -28,7 +27,7 @@ public class FileConverQueueTask { FilePreviewFactory previewFactory; @Autowired - RedissonClient redissonClient; + CacheService cacheService; @Autowired FileUtils fileUtils; @@ -36,7 +35,7 @@ public class FileConverQueueTask { @PostConstruct public void startTask(){ ExecutorService executorService = Executors.newFixedThreadPool(3); - executorService.submit(new ConverTask(previewFactory,redissonClient,fileUtils)); + executorService.submit(new ConverTask(previewFactory,cacheService,fileUtils)); logger.info("队列处理文件转换任务启动完成 "); } @@ -44,13 +43,13 @@ public class FileConverQueueTask { FilePreviewFactory previewFactory; - RedissonClient redissonClient; + CacheService cacheService; FileUtils fileUtils; - public ConverTask(FilePreviewFactory previewFactory, RedissonClient redissonClient,FileUtils fileUtils) { + public ConverTask(FilePreviewFactory previewFactory, CacheService cacheService,FileUtils fileUtils) { this.previewFactory = previewFactory; - this.redissonClient = redissonClient; + this.cacheService = cacheService; this.fileUtils=fileUtils; } @@ -58,8 +57,7 @@ public class FileConverQueueTask { public void run() { while (true) { try { - final RBlockingQueue queue = redissonClient.getBlockingQueue(FileConverQueueTask.queueTaskName); - String url = queue.take(); + String url = cacheService.takeQueueTask(); if(url!=null){ FileAttribute fileAttribute=fileUtils.getFileAttribute(url); logger.info("正在处理转换任务,文件名称【{}】",fileAttribute.getName()); diff --git a/jodconverter-web/src/main/java/cn/keking/service/cache/CacheService.java b/jodconverter-web/src/main/java/cn/keking/service/cache/CacheService.java new file mode 100644 index 00000000..fed1514b --- /dev/null +++ b/jodconverter-web/src/main/java/cn/keking/service/cache/CacheService.java @@ -0,0 +1,29 @@ +package cn.keking.service.cache; + +import java.util.List; +import java.util.Map; + +/** + * @auther: chenjh + * @time: 2019/4/2 16:45 + * @description + */ +public interface CacheService { + final String REDIS_FILE_PREVIEW_PDF_KEY = "converted-preview-pdf-file"; + final String REDIS_FILE_PREVIEW_IMGS_KEY = "converted-preview-imgs-file";//压缩包内图片文件集合 + + final Integer DEFAULT_PDF_CAPACITY = 500000; + final Integer DEFAULT_IMG_CAPACITY = 500000; + + void initPDFCachePool(Integer capacity); + void initIMGCachePool(Integer capacity); + void putPDFCache(String key, String value); + void putImgCache(String key, List value); + Map getPDFCache(); + String getPDFCache(String key); + Map> getImgCache(); + List getImgCache(String key); + + void addQueueTask(String url); + String takeQueueTask() throws InterruptedException; +} diff --git a/jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceJDKImpl.java b/jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceJDKImpl.java new file mode 100644 index 00000000..01320859 --- /dev/null +++ b/jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceJDKImpl.java @@ -0,0 +1,102 @@ +package cn.keking.service.cache.impl; + +import cn.keking.service.cache.CacheService; +import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; +import com.googlecode.concurrentlinkedhashmap.Weighers; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +/** + * @auther: chenjh + * @time: 2019/4/2 17:21 + * @description + */ +@Service +@ConditionalOnExpression("'${cache.type:default}'.equals('default')") +public class CacheServiceJDKImpl implements CacheService { + + private Map pdfCache; + + private Map> imgCache; + + private static final int QUEUE_SIZE = 500000; + + private BlockingQueue blockingQueue = new ArrayBlockingQueue(QUEUE_SIZE); + + @Override + public void initPDFCachePool(Integer capacity) { + pdfCache = new ConcurrentLinkedHashMap.Builder() + .maximumWeightedCapacity(capacity).weigher(Weighers.singleton()) + .build(); + } + + @Override + public void initIMGCachePool(Integer capacity) { + imgCache = new ConcurrentLinkedHashMap.Builder>() + .maximumWeightedCapacity(capacity).weigher(Weighers.singleton()) + .build(); + } + + @Override + public void putPDFCache(String key, String value) { + if (pdfCache == null) { + initPDFCachePool(CacheService.DEFAULT_PDF_CAPACITY); + } + pdfCache.put(key, value); + } + + @Override + public void putImgCache(String key, List value) { + if (imgCache == null) { + initIMGCachePool(CacheService.DEFAULT_IMG_CAPACITY); + } + imgCache.put(key, value); + } + + @Override + public Map getPDFCache() { + if (pdfCache == null) { + initPDFCachePool(CacheService.DEFAULT_PDF_CAPACITY); + } + return pdfCache; + } + + @Override + public String getPDFCache(String key) { + if (pdfCache == null) { + initPDFCachePool(CacheService.DEFAULT_PDF_CAPACITY); + } + return pdfCache.get(key); + } + + @Override + public Map> getImgCache() { + if (imgCache == null) { + initPDFCachePool(CacheService.DEFAULT_IMG_CAPACITY); + } + return imgCache; + } + + @Override + public List getImgCache(String key) { + if (imgCache == null) { + initPDFCachePool(CacheService.DEFAULT_IMG_CAPACITY); + } + return imgCache.get(key); + } + + @Override + public void addQueueTask(String url) { + blockingQueue.add(url); + } + + @Override + public String takeQueueTask() throws InterruptedException { + return String.valueOf(blockingQueue.take()); + } +} diff --git a/jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceRedisImpl.java b/jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceRedisImpl.java new file mode 100644 index 00000000..43335925 --- /dev/null +++ b/jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceRedisImpl.java @@ -0,0 +1,91 @@ +package cn.keking.service.cache.impl; + +import cn.keking.service.FileConverQueueTask; +import cn.keking.service.cache.CacheService; +import org.redisson.Redisson; +import org.redisson.api.RBlockingQueue; +import org.redisson.api.RMapCache; +import org.redisson.api.RedissonClient; +import org.redisson.config.Config; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; + +/** + * @auther: chenjh + * @time: 2019/4/2 18:02 + * @description + */ +@ConditionalOnExpression("'${cache.type:default}'.equals('redis')") +@Service +public class CacheServiceRedisImpl implements CacheService { + + private Config config; + + @Autowired + public CacheServiceRedisImpl(Config config) { + this.config = config; + this.redissonClient = Redisson.create(config); + } + + private RedissonClient redissonClient; + + @Override + public void initPDFCachePool(Integer capacity) { + + } + + @Override + public void initIMGCachePool(Integer capacity) { + + } + + @Override + public void putPDFCache(String key, String value) { + RMapCache convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_PDF_KEY); + convertedList.fastPut(key, value); + } + + @Override + public void putImgCache(String key, List value) { + RMapCache> convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_IMGS_KEY); + convertedList.fastPut(key, value); + } + + @Override + public Map getPDFCache() { + return redissonClient.getMapCache(REDIS_FILE_PREVIEW_PDF_KEY); + } + + @Override + public String getPDFCache(String key) { + RMapCache convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_PDF_KEY); + return convertedList.get(key); + } + + @Override + public Map> getImgCache() { + return redissonClient.getMapCache(REDIS_FILE_PREVIEW_IMGS_KEY); + } + + @Override + public List getImgCache(String key) { + RMapCache> convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_IMGS_KEY); + return convertedList.get(key); + } + + @Override + public void addQueueTask(String url) { + RBlockingQueue queue = redissonClient.getBlockingQueue(FileConverQueueTask.queueTaskName); + queue.addAsync(url); + } + + @Override + public String takeQueueTask() throws InterruptedException { + RBlockingQueue queue = redissonClient.getBlockingQueue(FileConverQueueTask.queueTaskName); + return queue.take(); + } +} diff --git a/jodconverter-web/src/main/java/cn/keking/utils/FileUtils.java b/jodconverter-web/src/main/java/cn/keking/utils/FileUtils.java index c76b6adb..a5158d03 100644 --- a/jodconverter-web/src/main/java/cn/keking/utils/FileUtils.java +++ b/jodconverter-web/src/main/java/cn/keking/utils/FileUtils.java @@ -2,9 +2,8 @@ package cn.keking.utils; import cn.keking.model.FileAttribute; import cn.keking.model.FileType; +import cn.keking.service.cache.CacheService; import com.google.common.collect.Lists; -import org.redisson.api.RMapCache; -import org.redisson.api.RedissonClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -27,11 +26,9 @@ import java.util.Map; public class FileUtils { Logger log= LoggerFactory.getLogger(getClass()); - - final String REDIS_FILE_PREVIEW_PDF_KEY = "converted-preview-pdf-file"; - final String REDIS_FILE_PREVIEW_IMGS_KEY = "converted-preview-imgs-file";//压缩包内图片文件集合 @Autowired - RedissonClient redissonClient; + CacheService cacheService; + @Value("${file.dir}") String fileDir; @@ -48,8 +45,7 @@ public class FileUtils { * @return */ public Map listConvertedFiles() { - RMapCache convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_PDF_KEY); - return convertedList; + return cacheService.getPDFCache(); } /** @@ -57,8 +53,7 @@ public class FileUtils { * @return */ public String getConvertedFile(String key) { - RMapCache convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_PDF_KEY); - return convertedList.get(key); + return cacheService.getPDFCache(key); } /** @@ -170,8 +165,7 @@ public class FileUtils { } public void addConvertedFile(String fileName, String value){ - RMapCache convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_PDF_KEY); - convertedList.fastPut(fileName, value); + cacheService.putPDFCache(fileName, value); } /** @@ -180,8 +174,7 @@ public class FileUtils { * @return */ public List getRedisImgUrls(String fileKey){ - RMapCache convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_IMGS_KEY); - return convertedList.get(fileKey); + return cacheService.getImgCache(fileKey); } /** @@ -190,8 +183,7 @@ public class FileUtils { * @param imgs */ public void setRedisImgUrls(String fileKey,List imgs){ - RMapCache convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_IMGS_KEY); - convertedList.fastPut(fileKey,imgs); + cacheService.putImgCache(fileKey, imgs); } /** * 判断文件编码格式 diff --git a/jodconverter-web/src/main/java/cn/keking/web/controller/OnlinePreviewController.java b/jodconverter-web/src/main/java/cn/keking/web/controller/OnlinePreviewController.java index 5d8dbb95..3adc8dfd 100644 --- a/jodconverter-web/src/main/java/cn/keking/web/controller/OnlinePreviewController.java +++ b/jodconverter-web/src/main/java/cn/keking/web/controller/OnlinePreviewController.java @@ -1,12 +1,10 @@ package cn.keking.web.controller; -import cn.keking.service.FileConverQueueTask; import cn.keking.service.FilePreview; import cn.keking.service.FilePreviewFactory; +import cn.keking.service.cache.CacheService; import org.apache.commons.io.IOUtils; -import org.redisson.api.RBlockingQueue; -import org.redisson.api.RedissonClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; @@ -26,7 +24,6 @@ import java.net.URLConnection; import java.net.URLDecoder; import java.util.Arrays; import java.util.List; -import java.util.Map; /** * @author yudian-it @@ -38,7 +35,7 @@ public class OnlinePreviewController { FilePreviewFactory previewFactory; @Autowired - RedissonClient redissonClient; + CacheService cacheService; /** * @param url @@ -126,8 +123,7 @@ public class OnlinePreviewController { @GetMapping("/addTask") @ResponseBody public String addQueueTask(String url) { - final RBlockingQueue queue = redissonClient.getBlockingQueue(FileConverQueueTask.queueTaskName); - queue.addAsync(url); + cacheService.addQueueTask(url); return "success"; } diff --git a/jodconverter-web/src/main/resources/application-dev.properties b/jodconverter-web/src/main/resources/application-dev.properties index af963da7..963db0f0 100644 --- a/jodconverter-web/src/main/resources/application-dev.properties +++ b/jodconverter-web/src/main/resources/application-dev.properties @@ -1,5 +1,3 @@ -#redis连接 -spring.redisson.address = 192.168.1.204:6379 ##资源映射路径 file.dir = C:\\Users\\yudian\\Desktop\\dev\\ spring.resources.static-locations = classpath:/META-INF/resources/,classpath:/resources/,classpath:/static/,classpath:/public/,file:${file.dir} @@ -12,4 +10,7 @@ spring.http.multipart.max-file-size=100MB ##文本类型 simText = txt,html,xml,java,properties,mp3,mp4,sql #多媒体类型 -media=mp3,mp4,flv,rmvb \ No newline at end of file +media=mp3,mp4,flv,rmvb +#缓存及队列实现类型,默认为JDK实现,可选redis(需要加spring.redisson.address等配置) +#cache.type = redis +#spring.redisson.address = 192.168.1.204:6379 \ No newline at end of file diff --git a/jodconverter-web/src/main/resources/application-prod.properties b/jodconverter-web/src/main/resources/application-prod.properties index 44b05736..1afd5ca1 100644 --- a/jodconverter-web/src/main/resources/application-prod.properties +++ b/jodconverter-web/src/main/resources/application-prod.properties @@ -1,19 +1,11 @@ -#=============================================#spring Redisson����#===================================# -spring.redisson.address = 10.19.140.7:6379 -spring.redisson.database = 0 -##��Դӳ��·��(��Ϊjar��ʽ���е�ԭ��) file.dir = /data/file-preview/convertedFile/ spring.resources.static-locations = classpath:/META-INF/resources/,classpath:/resources/,classpath:/static/,classpath:/public/,file:${file.dir} -## openoffice������� office.home = /opt/openoffice4 - -## �������� server.tomcat.uri-encoding = utf-8 converted.file.charset = utf-8 -## �ļ��ϴ����ֵ spring.http.multipart.max-file-size = 100MB - -## ֧�ֵ����ı���ʽ���ļ����� simText = txt,html,xml,java,properties,sql,js,md,json,conf,ini,vue,php,py,bat,gitignore,log,htm,css,cnf - -media=mp3,mp4,flv \ No newline at end of file +media=mp3,mp4,flv +cache.type = redis +spring.redisson.address = 10.19.140.7:6379 +spring.redisson.database = 0 \ No newline at end of file diff --git a/jodconverter-web/src/main/resources/application-uat.properties b/jodconverter-web/src/main/resources/application-uat.properties index 117ca36d..057f7953 100644 --- a/jodconverter-web/src/main/resources/application-uat.properties +++ b/jodconverter-web/src/main/resources/application-uat.properties @@ -1,15 +1,12 @@ -#=============================================#spring Redisson����#===================================# -spring.redisson.address = 192.168.1.204:6379 -spring.redisson.database = 3 -##��Դӳ��·��(��Ϊjar��ʽ���е�ԭ��) file.dir = /data/filepreview/ spring.resources.static-locations = classpath:/META-INF/resources/,classpath:/resources/,classpath:/static/,classpath:/public/,file:${file.dir} -## openoffice������� openOfficePath = 123 office.home = /opt/openoffice4 server.tomcat.uri-encoding = utf-8 converted.file.charset = utf-8 spring.http.multipart.max-file-size = 100MB -## ֧�ֵ����ı���ʽ���ļ����� simText = txt,html,xml,java,properties,sql,js,md,json,conf,ini,vue,php,py,bat,gitignore,log,htm,css,cnf -media=mp3,mp4,flv \ No newline at end of file +media=mp3,mp4,flv +cache.type = redis +spring.redisson.address = 192.168.1.204:6379 +spring.redisson.database = 3 \ No newline at end of file