Skip to content

F
fp-acq-wz
提交 07447b81 authored 作者: Matrix's avatar Matrix
浏览文件
操作

删减了不必要的东西，ACQ初版

上级 d3f8c6c3
流水线 #202 已失败 于阶段
        全部展开 隐藏空白字符变更
        内嵌 并排
        正在显示 包含 413 行增加 和 907 行删除
        acq-jwt/pom.xml deleted 100644 → 0
        浏览文件 @ d3f8c6c3
        <?xml version="1.0" encoding="UTF-8"?>
        <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <modelVersion>4.0.0</modelVersion>
        <parent>
        <artifactId>fp-acq-wz</artifactId>
        <groupId>com.zjty.fp</groupId>
        <version>0.0.1-SNAPSHOT</version>
        </parent>
        <artifactId>acq-jwt</artifactId>
        <name>acq-jwt</name>
        <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        </properties>
        <dependencies>
        <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-security</artifactId>
        </dependency>
        <dependency>
        <groupId>io.jsonwebtoken</groupId>
        <artifactId>jjwt</artifactId>
        <version>0.7.0</version>
        </dependency>
        </dependencies>
        </project>
        acq-misc/pom.xml
        浏览文件 @ 07447b81
        ......@@ -65,6 +65,13 @@
        <version>3.2.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.yy/android-util -->
        <dependency>
        <groupId>com.yy</groupId>
        <artifactId>android-util</artifactId>
        <version>1.0.2</version>
        </dependency>
        <!--swagger2 enable dependency-->
        <dependency>
        <groupId>org.springframework.boot</groupId>
        ......
        acq-misc/src/main/java/com/zjty/fp/acq/misc/entity/PsspCount.java 0 → 100644
        浏览文件 @ 07447b81
        package com.zjty.fp.acq.misc.entity;
        /**
        * PsspCount.
        *
        * @author Matrix <xhyrzldf@gmail.com>
        * @since 2020/7/29 at 12:20 下午
        */
        public class PsspCount {
        /**
        * 代表下一个文件应当用的后缀号
        */
        public static int count = -1;
        public static final String COUNT_PATH = "files/pssp/count";
        public static final String COUNT_ADDRESS = "files/pssp/count/count.txt";
        }
        acq-misc/src/main/java/com/zjty/fp/acq/misc/utils/DeflaterUtils.java 0 → 100644
        浏览文件 @ 07447b81
        package com.zjty.fp.acq.misc.utils;
        import com.loopj.android.http.Base64;
        import javax.annotation.Nullable;
        import java.io.ByteArrayOutputStream;
        import java.util.zip.DataFormatException;
        import java.util.zip.Deflater;
        import java.util.zip.Inflater;
        /**
        * DeflaterUtils 压缩字符串
        *
        * @author Matrix <xhyrzldf@gmail.com>
        * @since 2020/7/29 at 9:43 上午
        */
        public class DeflaterUtils {
        /**
        * 压缩
        */
        public static String zipString(String unzipString) {
        /**
        *
        * 0 ~ 9 压缩等级 低到高
        * public static final int BEST_COMPRESSION = 9; 最佳压缩的压缩级别。
        * public static final int BEST_SPEED = 1; 压缩级别最快的压缩。
        * public static final int DEFAULT_COMPRESSION = -1; 默认压缩级别。
        * public static final int DEFAULT_STRATEGY = 0; 默认压缩策略。
        * public static final int DEFLATED = 8; 压缩算法的压缩方法(目前唯一支持的压缩方法)。
        * public static final int FILTERED = 1; 压缩策略最适用于大部分数值较小且数据分布随机分布的数据。
        * public static final int FULL_FLUSH = 3; 压缩刷新模式，用于清除所有待处理的输出并重置拆卸器。
        * public static final int HUFFMAN_ONLY = 2; 仅用于霍夫曼编码的压缩策略。
        * public static final int NO_COMPRESSION = 0; 不压缩的压缩级别。
        * public static final int NO_FLUSH = 0; 用于实现最佳压缩结果的压缩刷新模式。
        * public static final int SYNC_FLUSH = 2; 用于清除所有未决输出的压缩刷新模式; 可能会降低某些压缩算法的压缩率。
        */
        //使用指定的压缩级别创建一个新的压缩器。
        Deflater deflater = new Deflater(Deflater.BEST_COMPRESSION);
        //设置压缩输入数据。
        deflater.setInput(unzipString.getBytes());
        //当被调用时，表示压缩应该以输入缓冲区的当前内容结束。
        deflater.finish();
        final byte[] bytes = new byte[256];
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream(256);
        while (!deflater.finished()) {
        //压缩输入数据并用压缩数据填充指定的缓冲区。
        int length = deflater.deflate(bytes);
        outputStream.write(bytes, 0, length);
        }
        //关闭压缩器并丢弃任何未处理的输入。
        deflater.end();
        return Base64.encodeToString(outputStream.toByteArray(), Base64.NO_PADDING);
        }
        /**
        * 解压缩
        */
        @Nullable
        public static String unzipString(String zipString) {
        byte[] decode = Base64.decode(zipString, Base64.NO_PADDING);
        //创建一个新的解压缩器 https://www.yiibai.com/javazip/javazip_inflater.html
        Inflater inflater = new Inflater();
        //设置解压缩的输入数据。
        inflater.setInput(decode);
        final byte[] bytes = new byte[256];
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream(256);
        try {
        //finished() 如果已到达压缩数据流的末尾，则返回true。
        while (!inflater.finished()) {
        //将字节解压缩到指定的缓冲区中。
        int length = inflater.inflate(bytes);
        outputStream.write(bytes, 0, length);
        }
        } catch (DataFormatException e) {
        e.printStackTrace();
        return null;
        } finally {
        //关闭解压缩器并丢弃任何未处理的输入。
        inflater.end();
        }
        return outputStream.toString();
        }
        }
        acq-misc/src/main/java/com/zjty/fp/acq/misc/utils/FileCreator.java
        浏览文件 @ 07447b81
        package com.zjty.fp.acq.misc.utils;
        import com.google.common.collect.Lists;
        import com.zjty.fp.acq.misc.entity.PsspCount;
        import lombok.extern.slf4j.Slf4j;
        import org.springframework.context.annotation.Scope;
        import org.springframework.stereotype.Component;
        ......@@ -13,6 +15,8 @@ import java.text.SimpleDateFormat;
        import java.util.Date;
        import java.util.List;
        import static com.zjty.fp.acq.misc.entity.PsspCount.*;
        @Slf4j
        @Scope("prototype")
        @Component
        ......@@ -32,9 +36,47 @@ public class FileCreator {
        return new File(pathname + filename);
        }
        public File createFileAndZip(String subName, String catalog, String data) {
        String zipData = DeflaterUtils.zipString(data);
        String pathname = "files/" + subName + "/" + catalog + "/";
        String filename = getPsspFileName(subName, catalog);
        Path path = Paths.get(pathname + filename);
        boolean make = createFilePath(new File(pathname));
        if (make) {
        try {
        Files.write(path, zipData.getBytes());
        } catch (IOException e) {
        log.info("生成文件时出现异常：" + e);
        }
        }
        //生成完毕后需要更新一下count的值
        count += 1;
        //写入到count文件中去
        Path countPath = Paths.get(COUNT_ADDRESS);
        //先删除原有的count文件，再写入现有的
        try {
        Files.deleteIfExists(countPath);
        } catch (IOException e) {
        log.warn("删除原count文件失败!原因:{}", e.toString());
        }
        boolean countMake = createFilePath(new File(COUNT_PATH));
        if (countMake) {
        try {
        Files.write(Paths.get(COUNT_ADDRESS), String.valueOf(count).getBytes());
        log.info("[pssp]磁盘count值更新成功:count = {}", count);
        } catch (IOException e) {
        log.info("[pssp]生成文件时出现异常：" + e);
        }
        }
        return new File(pathname + filename);
        }
        public File createFileNoTime(String subName, String catalog, String data) {
        String pathname = "files/" + subName + "/" + catalog + "/";
        String filename = subName + "-" + catalog + ".txt";
        String filename = subName + "-" + catalog + ".txt";
        Path path = Paths.get(pathname + filename);
        boolean make = createFilePath(new File(pathname));
        if (make) {
        ......@@ -47,16 +89,16 @@ public class FileCreator {
        return new File(pathname + filename);
        }
        public File createFileInUTF(String subName, String catalog,Long index, List<String> list){
        public File createFileInUTF(String subName, String catalog, Long index, List<String> list) {
        String pathname = "files/" + subName + "/" + catalog + "/";
        String filename = subName + "-" + catalog + "-" +index+".txt";
        String filename = subName + "-" + catalog + "-" + index + ".txt";
        Path path = Paths.get(pathname + filename);
        boolean make = createFilePath(new File(pathname));
        if (make) {
        try {
        Files.write(path,list);
        Files.write(path, list);
        } catch (IOException e) {
        log.info("生成文件时出现异常：{}" + e);
        log.info("生成文件时出现异常: " + e);
        }
        }
        return new File(pathname + filename);
        ......@@ -69,24 +111,25 @@ public class FileCreator {
        boolean make = createFilePath(new File(pathname));
        if (make) {
        try {
        Files.write(path,list);
        Files.write(path, list);
        } catch (IOException e) {
        log.info("生成文件时出现异常：" + e.getMessage());
        }
        }
        return new File(pathname + filename);
        }
        public File createFileStms(String subName, String catalog, List<String> list){
        public File createFileStms(String subName, String catalog, List<String> list) {
        String pathname = "files/" + subName + "/" + catalog + "/";
        String filename = getFileNameByDateStms(subName, catalog);
        String fileNameTmp=getFileNameByDateStmsTmp(subName, catalog);
        String fileNameTmp = getFileNameByDateStmsTmp(subName, catalog);
        Path path = Paths.get(pathname + filename);
        boolean make = createFilePath(new File(pathname));
        if (make) {
        try {
        File file= File.createTempFile(filename,".tmp",new File(pathname));
        Files.write(path,list);
        new File(pathname + filename).renameTo(new File(pathname+fileNameTmp));
        File file = File.createTempFile(filename, ".tmp", new File(pathname));
        Files.write(path, list);
        new File(pathname + filename).renameTo(new File(pathname + fileNameTmp));
        file.delete();
        } catch (IOException e) {
        log.info("生成文件时出现异常：" + e.getMessage());
        ......@@ -94,17 +137,18 @@ public class FileCreator {
        }
        return new File(pathname + filename);
        }
        public File createFileStms(String subName, String catalog, List<String> list,int count){
        public File createFileStms(String subName, String catalog, List<String> list, int count) {
        String pathname = "files/" + subName + "/" + catalog + "/";
        String filename = getFileNameByDateStms(subName, catalog,count);
        String fileNameTmp=getFileNameByDateStmsTmp(subName, catalog,count);
        String filename = getFileNameByDateStms(subName, catalog, count);
        String fileNameTmp = getFileNameByDateStmsTmp(subName, catalog, count);
        Path path = Paths.get(pathname + filename);
        boolean make = createFilePath(new File(pathname));
        if (make) {
        try {
        File file= File.createTempFile(filename,".tmp",new File(pathname));
        Files.write(path,list);
        new File(pathname + filename).renameTo(new File(pathname+fileNameTmp));
        File file = File.createTempFile(filename, ".tmp", new File(pathname));
        Files.write(path, list);
        new File(pathname + filename).renameTo(new File(pathname + fileNameTmp));
        file.delete();
        } catch (IOException e) {
        log.info("生成文件时出现异常：" + e.getMessage());
        ......@@ -112,6 +156,7 @@ public class FileCreator {
        }
        return new File(pathname + filename);
        }
        /**
        * 根据日期生成文件名
        * 以天为单位
        ......@@ -122,20 +167,50 @@ public class FileCreator {
        return subName + "-" + catalog + "-" + sdf.format(new Date()) + ".txt";
        }
        private String getFileNameByDateStms(String subName, String catalog,int count) {
        return subName + "-" + catalog + count+".tmp";
        private String getFileNameByDateStms(String subName, String catalog, int count) {
        return subName + "-" + catalog + count + ".tmp";
        }
        private String getFileNameByDateStmsTmp(String subName, String catalog,int count) {
        return subName + "-" + catalog + count +".txt";
        private String getFileNameByDateStmsTmp(String subName, String catalog, int count) {
        return subName + "-" + catalog + count + ".txt";
        }
        private String getFileNameByDateStms(String subName, String catalog) {
        return subName + "-" + catalog +".tmp";
        return subName + "-" + catalog + ".tmp";
        }
        private String getFileNameByDateStmsTmp(String subName, String catalog) {
        return subName + "-" + catalog +".txt";
        return subName + "-" + catalog + ".txt";
        }
        private String getPsspFileName(String subName, String catalog) {
        //count 先从内存里取读 内存里读不到从磁盘读
        int count = PsspCount.count;
        //如果count为初始值-1,则代表还没有从文件中读取之前的记录
        if (count == -1) {
        //读取文件获得文件编号后缀值
        Path path = Paths.get(COUNT_ADDRESS);
        List<String> data = Lists.newArrayList();
        try {
        data = Files.readAllLines(path);
        } catch (IOException e) {
        log.info("读取文件时出现异常：" + e);
        }
        count = data.get(0) == null ? 0 : Integer.parseInt(data.get(0));
        }
        if (count <= 0) count = 0;
        String psspFileName = subName + "-" + catalog + count + ".txt";
        log.info("成功生成文件名:{}", psspFileName);
        return psspFileName;
        }
        /**
        ......
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/base/beans/PsspInitialRunner.java
        浏览文件 @ 07447b81
        package com.zjty.fp.acq.pssp.base.beans;
        import com.zjty.fp.acq.pssp.base.constant.DictConst;
        import com.zjty.fp.acq.pssp.subject.entity.location.Region;
        import com.zjty.fp.acq.pssp.subject.entity.location.Website;
        import com.google.common.collect.Lists;
        import com.zjty.fp.acq.pssp.subject.service.RegionService;
        import com.zjty.fp.acq.pssp.subject.service.WebsiteService;
        import lombok.extern.slf4j.Slf4j;
        ......@@ -12,7 +10,14 @@ import org.springframework.context.annotation.Bean;
        import org.springframework.context.annotation.Configuration;
        import org.springframework.core.annotation.Order;
        import java.util.Map;
        import java.io.File;
        import java.io.IOException;
        import java.nio.file.Files;
        import java.nio.file.Path;
        import java.nio.file.Paths;
        import java.util.List;
        import static com.zjty.fp.acq.misc.entity.PsspCount.*;
        /**
        * PsspInitialRunner 初始化Runner程序
        ......@@ -35,23 +40,45 @@ public class PsspInitialRunner {
        @Bean
        public CommandLineRunner initializeDictMap() {
        return args -> {
        log.info("[pssp] 正在尝试初始化 pssp 网站Map字段表");
        log.info("[pssp] 正在尝试初始化 count 文件");
        // List<Website> -> key : id , value : Website
        Map<Long, Website> websiteMap = websiteService.findDictMap();
        DictConst.WEBSITE_MAP = websiteMap;
        boolean countExists = Files.exists(Paths.get(COUNT_ADDRESS));
        log.info("[pssp] 初始化 pssp 网站字典Map完毕,字典表数据大小为{}", websiteMap.size());
        if (countExists) {
        log.info("[pssp]磁盘已存在count文件，跳过初始化流程，将磁盘的count读取入内存中");
        //读取文件获得文件编号后缀值
        Path path = Paths.get(COUNT_ADDRESS);
        List<String> data = Lists.newArrayList();
        log.info("[pssp] 正在尝试初始化 pssp 地区Map字段表");
        try {
        data = Files.readAllLines(path);
        } catch (IOException e) {
        log.info("读取文件时出现异常：" + e);
        }
        count = data.get(0) == null ? 0 : Integer.parseInt(data.get(0));
        } else {
        log.info("[pssp] 磁盘不存在count文件，初始化count文件");
        boolean countMake = createFilePath(new File(COUNT_PATH));
        // List<Website> -> key : code , value : Region
        Map<String, Region> regionMap = regionService.findDictMap();
        DictConst.REGION_MAP = regionMap;
        count = 0;
        if (countMake) {
        try {
        Files.write(Paths.get(COUNT_ADDRESS), String.valueOf(count).getBytes());
        log.info("[pssp]磁盘count值更新成功:count = {}", count);
        } catch (IOException e) {
        log.info("[pssp]生成文件时出现异常：" + e);
        }
        }
        log.info("[pssp]初始化count文件完毕!");
        }
        log.info("[pssp] 初始化 pssp 地区字典Map完毕,字典表数据大小为{}", regionMap.size());
        };
        }
        private boolean createFilePath(File file) {
        return file.exists() || file.mkdirs();
        }
        }
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/controller/PsspApiController.java deleted 100644 → 0
        浏览文件 @ d3f8c6c3
        package com.zjty.fp.acq.pssp.subject.controller;
        import com.google.common.collect.ImmutableMap;
        import com.zjty.fp.acq.misc.utils.DateTimeUtil;
        import com.zjty.fp.acq.pssp.subject.entity.location.Alert;
        import com.zjty.fp.acq.pssp.subject.entity.vo.PsspPageVo;
        import com.zjty.fp.acq.pssp.subject.service.AlertService;
        import com.zjty.fp.acq.pssp.subject.service.WebsiteService;
        import org.springframework.beans.factory.annotation.Autowired;
        import org.springframework.data.domain.Page;
        import org.springframework.data.domain.PageRequest;
        import org.springframework.data.domain.Sort;
        import org.springframework.http.ResponseEntity;
        import org.springframework.web.bind.annotation.GetMapping;
        import org.springframework.web.bind.annotation.RequestMapping;
        import org.springframework.web.bind.annotation.RequestParam;
        import org.springframework.web.bind.annotation.RestController;
        import java.time.LocalDate;
        import java.time.format.DateTimeFormatter;
        import static org.springframework.http.ResponseEntity.ok;
        /**
        * 第三方api接口
        *
        * @author : Matrix [xhyrzldf@foxmail.com]
        * @since : 18-8-22
        */
        @RequestMapping("/api/pssp")
        @RestController
        public class PsspApiController {
        @Autowired
        private AlertService alertService;
        @Autowired
        private WebsiteService websiteService;
        /**
        * 获得所有网站数据
        */
        @GetMapping("/websites")
        public ResponseEntity findAll() {
        return ResponseEntity.ok(websiteService.findAllData());
        }
        @GetMapping("/alerts")
        public ResponseEntity<PsspPageVo> findAllData(
        @RequestParam(value = "startTime") String startTime,
        @RequestParam(value = "endTime") String endTime,
        @RequestParam(value = "page", defaultValue = "1") int page,
        @RequestParam(value = "size", defaultValue = "10") int size,
        @RequestParam(value = "sortDirection", defaultValue = "ASC") String sortDirection
        ) {
        page--;
        //默认为ASC
        Sort.Direction direction = Sort.Direction.ASC;
        if ("DESC".equals(sortDirection)) {
        direction = Sort.Direction.DESC;
        }
        LocalDate localStartTime = LocalDate.parse(startTime, DateTimeFormatter.ofPattern("yyyy-MM-dd"));
        LocalDate localEndTime = LocalDate.parse(endTime, DateTimeFormatter.ofPattern("yyyy-MM-dd"));
        Page<Alert> alertPage = alertService.findAllDataBetweenTime(localStartTime, localEndTime, new PageRequest(page, size, direction, "tmFetch"));
        return ResponseEntity.ok(PsspPageVo.trans2PageVo(alertPage, page));
        }
        @GetMapping("/alerts/after")
        public ResponseEntity<PsspPageVo> findDataAfterId(
        @RequestParam(value = "primaryId") Long id,
        @RequestParam(value = "page", defaultValue = "1") int page,
        @RequestParam(value = "size", defaultValue = "10") int size) {
        page--;
        Page<Alert> alertPage = alertService.findAllAfterId(id, new PageRequest(page, size));
        return ResponseEntity.ok(PsspPageVo.trans2PageVo(alertPage, page));
        }
        @GetMapping("/alerts/recommend")
        public ResponseEntity findRecommendTime() {
        String recommendTime = DateTimeUtil.formatDateTimetoString(alertService.findEarliestTime(), "yyyy-MM-dd HH:mm:ss");
        return ok(ImmutableMap.of("recommendTime", recommendTime));
        }
        }
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/controller/PsspTestController.java deleted 100644 → 0
        浏览文件 @ d3f8c6c3
        package com.zjty.fp.acq.pssp.subject.controller;
        import com.zjty.fp.acq.pssp.subject.service.AlertService;
        import com.zjty.fp.acq.pssp.subject.service.RegionService;
        import com.zjty.fp.acq.pssp.subject.service.WebsiteService;
        import com.zjty.fp.acq.pssp.task.CollectDataTask;
        import org.springframework.beans.factory.annotation.Autowired;
        import org.springframework.http.ResponseEntity;
        import org.springframework.web.bind.annotation.GetMapping;
        import org.springframework.web.bind.annotation.RequestMapping;
        import org.springframework.web.bind.annotation.RequestParam;
        import org.springframework.web.bind.annotation.RestController;
        /**
        * @author matrix
        */
        @RestController
        @RequestMapping("/pssp/test")
        public class PsspTestController {
        @Autowired
        AlertService alertService;
        @Autowired
        WebsiteService websiteService;
        @Autowired
        RegionService regionService;
        @Autowired
        CollectDataTask psspTask;
        /**
        * 抓取所有报警数据(当前月)
        */
        @GetMapping("/alerts/all")
        public ResponseEntity<String> fetchAllAlerts() {
        alertService.fetchAllData();
        return ResponseEntity.ok("alerts fetch job is running success");
        }
        /**
        * 抓取更新的数据
        */
        @GetMapping("/alerts/updated")
        public ResponseEntity<String> fetchUpdatedAlerts() {
        alertService.fetchUpdatedData();
        return ResponseEntity.ok("update alerts data success");
        }
        /**
        * 抓取所有网页数据
        */
        @GetMapping("/websites/all")
        public ResponseEntity<String> fetchAllWebsites() {
        //抓取指定月份的数据
        websiteService.fetchAllData();
        return ResponseEntity.ok("fetch websites success");
        }
        /**
        * 抓取更新的数据
        */
        @GetMapping("/websites/updated")
        public ResponseEntity<String> fetchUpdatedWebsites() {
        websiteService.fetchUpdatedData();
        return ResponseEntity.ok("update websites data success ");
        }
        /**
        * 抓取所有区域数据
        */
        @GetMapping("/regions/all")
        public ResponseEntity<String> fetchAllRegions() {
        regionService.fetchAllData();
        return ResponseEntity.ok("fetch regions success");
        }
        /**
        * 抓取更新的数据
        */
        @GetMapping("/regions/updated")
        public ResponseEntity<String> fetchUpdatedRegions() {
        regionService.fetchUpdatedData();
        return ResponseEntity.ok("update regions data success ");
        }
        }
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/entity/location/Alert.java
        浏览文件 @ 07447b81
        ......@@ -142,39 +142,4 @@ public class Alert {
        @Column(name = "e_status")
        private Integer eStatus;
        public PsspEvent toEvent() {
        //从constMap中查询出web与region的名称
        String idWebName = DictConst.WEBSITE_MAP
        .getOrDefault(this.idWeb.longValue(), Website.defaultInstance())
        .getName();
        String regionName = DictConst.REGION_MAP
        .getOrDefault(String.valueOf(this.codeOrg), Region.defaultInstance())
        .getName();
        return new PsspEvent("pssp",
        tmFetch,
        String.valueOf(primaryId),
        id,
        codeOrg,
        regionName,
        idWebCategory,
        idWeb,
        idWebName,
        matchKeyword,
        title,
        abs,
        webUrl,
        webSnapshot,
        docUrl,
        docSnapshot,
        handlerUser,
        eInZip,
        eIdent,
        eAlert,
        eType,
        eStatus);
        }
        }
        \ No newline at end of file
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/entity/remote/RemoteWebsite.java
        浏览文件 @ 07447b81
        ......@@ -58,8 +58,6 @@ public class RemoteWebsite {
        private Integer statusError;
        private Integer idUser;
        public Website toDo() {
        ModelMapper mapper = new ModelMapper();
        mapper.getConfiguration().setAmbiguityIgnored(true);
        ......
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/repository/location/AlertRepository.java deleted 100644 → 0
        浏览文件 @ d3f8c6c3
        package com.zjty.fp.acq.pssp.subject.repository.location;
        import com.zjty.fp.acq.pssp.subject.entity.location.Alert;
        import org.springframework.data.domain.Page;
        import org.springframework.data.domain.Pageable;
        import org.springframework.data.jpa.repository.JpaRepository;
        import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
        import org.springframework.data.jpa.repository.Query;
        import org.springframework.stereotype.Repository;
        import java.util.Date;
        import java.util.List;
        /**
        * @author matrix
        */
        @Repository
        public interface AlertRepository extends JpaRepository<Alert, Long>, JpaSpecificationExecutor<Alert> {
        /**
        * 查询最新时间的一条对象
        *
        * @return 最新时间的一条报警
        */
        Alert findFirstByOrderByTmFetchDesc();
        /**
        * 查询指定月份最新的一条报警数据,如果时间相同,取最大id
        *
        * @param specifiedMonth 例如201802,201911格式为`%Y%m`
        * @return 指定月份最新的一条报警数据
        */
        @SuppressWarnings("SqlResolve")
        @Query(value = "select * from pssp_alert p where date_format(p.tm_fetch,'%Y%m')= ?1 order by p.tm_fetch desc,p.id desc limit 1", nativeQuery = true)
        Alert findSpecifiedMonthLastedData(String specifiedMonth);
        /**
        * 查询最老时间的一条对象
        *
        * @return 最远时间距离的一条报警
        */
        Alert findFirstByOrderByTmFetch();
        /**
        * 查询指定id之后的数据
        * @param id
        * @return
        */
        Page<Alert> findByPrimaryIdAfter(Long id,Pageable pageable);
        /**
        * 查询指定时间的告警数据
        *
        * @param tmFetch 要查询的时间
        * @return 报警数据集合
        */
        List<Alert> findByTmFetch(Date tmFetch);
        /**
        * 查询今日的报警信息
        *
        * @return 今日报警数据集合
        */
        @Query(value = "SELECT * FROM pssp_alert p WHERE TO_DAYS(p.tm_fetch) = TO_DAYS(NOW())", nativeQuery = true)
        List<Alert> findTodayData();
        /**
        * 查询指定时间段内的分页告警数据
        *
        * @param startTime 开始时间
        * @param endTime 结束时间
        * @param pageable 分页参数
        * @return 报警集合
        */
        @Query("select o from Alert o where o.tmFetch between ?1 and ?2")
        Page<Alert> findBetweenTime(Date startTime, Date endTime, Pageable pageable);
        /**
        * 查询指定时间段内的告警数据
        *
        * @param startTime 开始时间
        * @param endTime 结束时间
        * @return 报警集合
        */
        @Query("select o from Alert o where o.tmFetch between ?1 and ?2")
        List<Alert> findBetweenTime(Date startTime, Date endTime);
        /**
        * 查询指定时间后的告警数据
        *
        * @param time 指定的时间
        * @return 报警集合
        */
        @Query("select o from Alert o where o.tmFetch >= ?1")
        List<Alert> findAfterTime(Date time);
        }
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/repository/location/RegionRepository.java deleted 100644 → 0
        浏览文件 @ d3f8c6c3
        package com.zjty.fp.acq.pssp.subject.repository.location;
        import com.zjty.fp.acq.pssp.subject.entity.location.Region;
        import org.springframework.data.jpa.repository.JpaRepository;
        import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
        import org.springframework.stereotype.Repository;
        /**
        * RegionRepository.
        *
        * @author Matrix <xhyrzldf@gmail.com>
        * @since 2019-04-08 at 16:35
        */
        @Repository
        public interface RegionRepository extends JpaRepository<Region,String>, JpaSpecificationExecutor<Region> {
        }
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/repository/location/WebsiteRepository.java deleted 100644 → 0
        浏览文件 @ d3f8c6c3
        package com.zjty.fp.acq.pssp.subject.repository.location;
        import com.zjty.fp.acq.pssp.subject.entity.location.Website;
        import org.springframework.data.jpa.repository.JpaRepository;
        import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
        import org.springframework.stereotype.Repository;
        /**
        * fusion-platform.
        *
        * @author : Matrix [xhyrzldf@gmail.com]
        * 18-9-6 .
        */
        @Repository
        public interface WebsiteRepository extends JpaRepository<Website, Long>, JpaSpecificationExecutor<Website> {
        /**
        * 获取最新ID的一条网站数据
        *
        * @return 最新ID的网站数据
        */
        Website findFirstByOrderByIdDesc();
        }
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/service/AlertService.java
        浏览文件 @ 07447b81
        ......@@ -15,26 +15,4 @@ import java.util.List;
        */
        public interface AlertService extends PsspTimeService<Alert> {
        /**
        * 根据主键id集合查询alert对象
        *
        * @param primaryIds 主键id集合
        * @return 报警集合
        */
        List<Alert> findAll(List<Long> primaryIds);
        /**
        * 查询指定分布式主键id之后的报警数据集合
        * @param primaryId 分布式主键ID
        * @return 报警集合
        */
        Page<Alert> findAllAfterId(Long primaryId, Pageable pageable);
        /**
        * 查询出报警数据的最早时间点.
        *
        * @return 最早的一条报警数据的时间
        */
        Date findEarliestTime();
        }
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/service/PsspService.java
        浏览文件 @ 07447b81
        ......@@ -23,13 +23,4 @@ public interface PsspService<T> {
        */
        void fetchUpdatedData();
        /**
        * 获取所有数据
        *
        * @return 所有的数据
        */
        List<T> findAllData();
        }
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/service/PsspTimeService.java
        浏览文件 @ 07447b81
        ......@@ -15,52 +15,4 @@ import java.util.List;
        * @since 2019-04-08 at 16:19
        */
        public interface PsspTimeService<T> extends PsspService<T>{
        /**
        *
        * 根据时间范围分页查询结果集
        * @param startTime 开始时间
        * @param endTime 结束时间
        * @param pageable 分页参数
        * @return 结果集
        */
        Page<T> findAllDataBetweenTime(LocalDate startTime, LocalDate endTime, Pageable pageable);
        /**
        * 获取指定年月的数据
        *
        * @param year 年
        * @param month 月
        * @return 指定年月的数据集合
        */
        List<T> findAllDataByMonth(int year, int month);
        /**
        * 获取指定日的数据
        *
        * @param year 年
        * @param month 月
        * @param day 日
        * @return 指定年月日的数据集合
        * @throws Exception 异常
        */
        List<T> findAllDataByDate(int year, int month, int day) throws Exception;
        /**
        * 获取最近N天的数据
        *
        * @param days 指定最近几天的时间
        * @return 最近指定天数的数据
        */
        List<T> findRecentDaysData(int days);
        /**
        * 获取指定时间内最近N天的数据
        *
        * @param specifiedTime 指定时间
        * @param days 指定最近几天的时间
        * @return 最近指定天数的数据
        */
        List<T> findRecentDaysData(LocalDateTime specifiedTime, int days);
        }
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/service/RegionService.java
        浏览文件 @ 07447b81
        ......@@ -2,8 +2,6 @@ package com.zjty.fp.acq.pssp.subject.service;
        import com.zjty.fp.acq.pssp.subject.entity.location.Region;
        import java.util.Map;
        /**
        * RegionService.
        *
        ......@@ -11,11 +9,4 @@ import java.util.Map;
        * @since 2019-04-08 at 16:22
        */
        public interface RegionService extends PsspService<Region> {
        /**
        * 地区数据以字典的形式返回
        *
        * @return key : code ,value : {@link Region}
        */
        Map<String, Region> findDictMap();
        }
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/service/WebsiteService.java
        浏览文件 @ 07447b81
        ......@@ -13,18 +13,4 @@ import java.util.Map;
        */
        public interface WebsiteService extends PsspService<Website> {
        /**
        * 获取所有监控网站的数量
        *
        * @return 所有被监控的网站的数量
        */
        long getCount();
        /**
        * 网站数据以字典的形式返回
        *
        * @return key : id ,value : {@link Website}
        */
        Map<Long, Website> findDictMap();
        }
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/service/impl/RegionServiceImpl.java
        浏览文件 @ 07447b81
        package com.zjty.fp.acq.pssp.subject.service.impl;
        import com.zjty.fp.acq.pssp.base.constant.DictConst;
        import com.zjty.fp.acq.misc.utils.FileCreator;
        import com.zjty.fp.acq.misc.utils.JacksonUtil;
        import com.zjty.fp.acq.pssp.subject.entity.location.Region;
        import com.zjty.fp.acq.pssp.subject.entity.remote.RemoteRegion;
        import com.zjty.fp.acq.pssp.subject.repository.location.RegionRepository;
        import com.zjty.fp.acq.pssp.subject.repository.remote.RemoteRegionRepository;
        import com.zjty.fp.acq.pssp.subject.service.RegionService;
        import lombok.extern.slf4j.Slf4j;
        ......@@ -12,14 +12,9 @@ import org.springframework.stereotype.Service;
        import javax.transaction.Transactional;
        import java.util.List;
        import java.util.Map;
        import java.util.concurrent.CompletableFuture;
        import java.util.function.Function;
        import java.util.stream.Collectors;
        import static java.util.Objects.nonNull;
        import static java.util.stream.Collectors.toList;
        import static java.util.stream.Collectors.toMap;
        /**
        * RegionServiceImpl. 区域接口实现类
        ......@@ -33,7 +28,7 @@ import static java.util.stream.Collectors.toMap;
        public class RegionServiceImpl implements RegionService {
        @Autowired
        private RegionRepository regionRepository;
        FileCreator fileCreator;
        @Autowired
        private RemoteRegionRepository remoteRegionRepository;
        ......@@ -53,66 +48,42 @@ public class RegionServiceImpl implements RegionService {
        // 异步
        CompletableFuture.runAsync(() -> {
        //更新本地DB
        log.info("[pssp]正在异步写入地区数据到本地数据库");
        regionRepository.save(regionList);
        log.info("[pssp]异步地区数据写入完成");
        //更新字典map
        log.info("[pssp] [定时任务] 开始更新字典Map任务,更新前的字典Map大小为,regionMap : {}", DictConst.REGION_MAP.size());
        DictConst.REGION_MAP = findDictMap();
        log.info("[pssp] [定时任务] 完成更新字典Map任务,更新后的字典Map大小为,regionMap : {}", DictConst.REGION_MAP.size());
        log.info("[pssp]正在异步写入网站数据到本地文件");
        String webJson = JacksonUtil.toJSon(regionList).replace("\n", "");
        log.info("[pssp]准备写入文件，要写入的文件字符长度为{}", webJson.length());
        fileCreator.createFileAndZip("pssp", "region", webJson);
        });
        }
        @Override
        public void fetchUpdatedData() {
        log.info("[pssp]正在执行地区数据的更新");
        //批量对比 首先获取远端DB所有区域，对比数量，数量一致任务结束,数量不一致获取自己没有的进行采集
        List<RemoteRegion> remoteRegionList = remoteRegionRepository.findAll();
        List<Region> localRegionList = regionRepository.findAll();
        int remoteSize = remoteRegionList.size();
        int localSize = localRegionList.size();
        if (remoteSize != localSize) {
        log.info("[pssp] 本次地区更新采集没有要更新的数据,本地区域size : {},远端区域size : {}", localSize, remoteSize);
        } else {
        CompletableFuture.runAsync(() -> {
        //映射为Map , key : code ,value : region object ,如果远程的区域对象code 在此get不到则，证明需要插入
        Map<String, Region> localRegionMap = localRegionList.stream().collect(toMap(Region::getCode, Function.identity()));
        List<Region> regionList = remoteRegionList.stream()
        .filter(remoteRegion -> nonNull(localRegionMap.get(remoteRegion.getCode())))
        .map(RemoteRegion::toDo)
        .collect(toList());
        log.info("[pssp] 发现需要同步更新的地区对象,对象内容为 : {}", regionList);
        log.info("[pssp]正在异步写入地区更新数据到本地数据库");
        regionRepository.save(regionList);
        log.info("[pssp]异步地区更新数据写入完成");
        //更新字典map
        log.info("[pssp] [定时任务] 开始更新字典Map任务,更新前的字典Map大小为,regionMap : {}", DictConst.REGION_MAP.size());
        DictConst.REGION_MAP = findDictMap();
        log.info("[pssp] [定时任务] 完成更新字典Map任务,更新后的字典Map大小为,regionMap : {}", DictConst.REGION_MAP.size());
        });
        }
        }
        @Override
        public List<Region> findAllData() {
        return regionRepository.findAll();
        }
        @Override
        public Map<String, Region> findDictMap() {
        return regionRepository.findAll()
        .stream()
        .collect(Collectors
        .toMap(Region::getCode, Function.identity()));
        log.info("[pssp] 本次地区更新采集没有要更新的数据");
        //区域数据不用更新
        // //批量对比 首先获取远端DB所有区域，对比数量，数量一致任务结束,数量不一致获取自己没有的进行采集
        // List<RemoteRegion> remoteRegionList = remoteRegionRepository.findAll();
        // List<Region> localRegionList = regionRepository.findAll();
        //
        // int remoteSize = remoteRegionList.size();
        // int localSize = localRegionList.size();
        // if (remoteSize != localSize) {
        // log.info("[pssp] 本次地区更新采集没有要更新的数据,本地区域size : {},远端区域size : {}", localSize, remoteSize);
        // } else {
        // CompletableFuture.runAsync(() -> {
        // //映射为Map , key : code ,value : region object ,如果远程的区域对象code 在此get不到则，证明需要插入
        // Map<String, Region> localRegionMap = localRegionList.stream().collect(toMap(Region::getCode, Function.identity()));
        //
        // List<Region> regionList = remoteRegionList.stream()
        // .filter(remoteRegion -> nonNull(localRegionMap.get(remoteRegion.getCode())))
        // .map(RemoteRegion::toDo)
        // .collect(toList());
        //
        // log.info("[pssp] 发现需要同步更新的地区对象,对象内容为 : {}", regionList);
        //
        // log.info("[pssp]正在异步写入地区更新数据到本地数据库");
        // // TODO: 2020/7/29 这里改到写入文件
        // regionRepository.save(regionList);
        // log.info("[pssp]异步地区更新数据写入完成");
        // });
        }
        }
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/service/impl/WebsiteServiceImpl.java
        浏览文件 @ 07447b81
        package com.zjty.fp.acq.pssp.subject.service.impl;
        import com.zjty.fp.acq.pssp.base.constant.DictConst;
        import com.zjty.fp.acq.misc.utils.FileCreator;
        import com.zjty.fp.acq.misc.utils.JacksonUtil;
        import com.zjty.fp.acq.pssp.subject.entity.location.Website;
        import com.zjty.fp.acq.pssp.subject.entity.remote.RemoteWebsite;
        import com.zjty.fp.acq.pssp.subject.repository.location.WebsiteRepository;
        import com.zjty.fp.acq.pssp.subject.repository.remote.RemoteWebsiteRepository;
        import com.zjty.fp.acq.pssp.subject.service.WebsiteService;
        import com.zjty.fp.acq.pssp.utils.DicMapUtil;
        import lombok.extern.slf4j.Slf4j;
        import org.springframework.beans.factory.annotation.Autowired;
        import org.springframework.stereotype.Service;
        import org.springframework.util.CollectionUtils;
        import javax.transaction.Transactional;
        import java.util.HashMap;
        import java.util.List;
        import java.util.Map;
        import java.util.concurrent.CompletableFuture;
        import java.util.function.Function;
        import java.util.stream.Collectors;
        import static com.zjty.fp.acq.pssp.task.CollectDataTask.dynamicNow;
        import static java.util.stream.Collectors.toList;
        /**
        ......@@ -32,11 +34,13 @@ import static java.util.stream.Collectors.toList;
        public class WebsiteServiceImpl implements WebsiteService {
        @Autowired
        private WebsiteRepository websiteRepository;
        FileCreator fileCreator;
        @Autowired
        private RemoteWebsiteRepository remoteWebsiteRepository;
        private static final String websiteName = "website";
        @Override
        public void fetchAllData() {
        log.info("[pssp]准备抓取全部源网站数据");
        ......@@ -46,27 +50,26 @@ public class WebsiteServiceImpl implements WebsiteService {
        log.info("[pssp]源网站数据抓取完成,size = {},准备写入融合平台数据库", websiteList.size());
        // 异步写入数据库
        CompletableFuture.runAsync(() -> {
        log.info("[pssp]正在异步写入网站数据到本地数据库");
        websiteRepository.save(websiteList);
        log.info("[pssp]正在异步写入网站数据到本地文件");
        String webJson = JacksonUtil.toJSon(websiteList).replace("\n", "");
        log.info("[pssp]准备写入文件，要写入的文件字符长度为{}", webJson.length());
        fileCreator.createFileAndZip("pssp", "website", webJson);
        log.info("[pssp]异步网站数据写入完成");
        //更新字典map
        log.info("[pssp] [定时任务] 开始更新字典Map任务,更新前的字典Map大小为,websiteMap : {}", DictConst.WEBSITE_MAP.size());
        DictConst.WEBSITE_MAP = findDictMap();
        log.info("[pssp] [定时任务] 完成更新字典Map任务,更新后的字典Map大小为,websiteMap : {}", DictConst.WEBSITE_MAP.size());
        });
        }
        @Override
        public void fetchUpdatedData() {
        log.info("[pssp]正在执行网站数据的增量更新");
        Website lastedWebsite = websiteRepository.findFirstByOrderByIdDesc();
        if (lastedWebsite == null) {
        log.info("[pssp] 当前DB中没有任何网站,请先执行网站历史采集任务");
        //读取文件中的最新id , 获得当前月的时间
        String key = dynamicNow.getYear() + "-" + dynamicNow.getMonthValue();
        Integer lastedId = DicMapUtil.readDictFile(websiteName).getOrDefault(key, 0);
        if (lastedId == 0) {
        log.info("[pssp] 当前文件中没有任何网站,请先执行网站历史采集任务");
        return;
        }
        Long lastedDataId = lastedWebsite.getId();
        Long lastedDataId = Long.valueOf(lastedId);
        log.info("[pssp]当前本机网站表数据的最新id为 {} ,向源数据库采集在这id之后的数据....", lastedDataId);
        List<Website> updatedData = remoteWebsiteRepository.findDataFromId(lastedDataId).stream()
        .map(RemoteWebsite::toDo)
        ......@@ -78,40 +81,20 @@ public class WebsiteServiceImpl implements WebsiteService {
        log.info("[pssp] 网站数据采集完成,本次采集了 {}条数据,数据写入本机数据库", updatedData.size());
        // 异步写入数据库
        CompletableFuture.runAsync(() -> {
        log.info("[pssp]正在异步写入网站更新数据到本地数据库");
        websiteRepository.save(updatedData);
        log.info("[pssp]正在异步写入网站更新数据到本地文件");
        String webJson = JacksonUtil.toJSon(updatedData).replace("\n", "");
        fileCreator.createFileAndZip("pssp", "website", webJson);
        //写完要把最新的id给记录住
        long maxId = updatedData.stream().mapToLong(Website::getId).max().orElse(0L);
        log.info("[pssp]更新后的最大网站数据id为:{},记录到文件中", maxId);
        Map<String, Integer> map = new HashMap<>();
        map.put(key, (int) maxId);
        DicMapUtil.createDictFile(websiteName, map);
        log.info("[pssp]异步网站更新数据写入完成");
        //更新字典map
        log.info("[pssp] [定时任务] 开始更新字典Map任务,更新前的字典Map大小为,websiteMap : {}", DictConst.WEBSITE_MAP.size());
        DictConst.WEBSITE_MAP = findDictMap();
        log.info("[pssp] [定时任务] 完成更新字典Map任务,更新后的字典Map大小为,websiteMap : {}", DictConst.WEBSITE_MAP.size());
        });
        }
        }
        @Override
        public List<Website> findAllData() {
        return websiteRepository.findAll();
        }
        @Override
        public long getCount() {
        return websiteRepository.count();
        }
        /**
        * 网站数据以字典的形式返回
        *
        * @return key : id ,value : {@link Website}
        */
        @Override
        public Map<Long, Website> findDictMap() {
        return websiteRepository.findAll()
        .stream()
        .collect(Collectors
        .toMap(Website::getId, Function.identity()));
        }
        }
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/task/CollectDataTask.java
        浏览文件 @ 07447b81
        ......@@ -4,7 +4,6 @@ package com.zjty.fp.acq.pssp.task;
        import com.zjty.fp.acq.misc.utils.DateTimeUtil;
        import com.zjty.fp.acq.pssp.base.aspect.EnablePsspSchProtect;
        import com.zjty.fp.acq.pssp.base.aspect.PsspScheduleTrigger;
        import com.zjty.fp.acq.pssp.subject.repository.location.AlertRepository;
        import com.zjty.fp.acq.pssp.subject.repository.remote.RemoteAlertRepository;
        import com.zjty.fp.acq.pssp.subject.service.AlertService;
        import com.zjty.fp.acq.pssp.subject.service.RegionService;
        ......@@ -37,10 +36,12 @@ public class CollectDataTask {
        * 用于控制计划任务的开关,一般来讲开关默认开启,当进行历史数据同步的时候,开关关闭
        */
        public static boolean trigger = true;
        public static LocalDate dynamicNow;
        private static String dynamicTableMonth;
        static {
        dynamicTableMonth = DateTimeUtil.formatDateTimetoString(new Date(), "yyyyMM");
        dynamicNow = LocalDate.now();
        }
        @Autowired
        ......@@ -54,9 +55,6 @@ public class CollectDataTask {
        private int count = 0;
        @Autowired
        private AlertRepository localPsspRep;
        @Autowired
        private RemoteAlertRepository remotePsspRep;
        ......@@ -66,6 +64,7 @@ public class CollectDataTask {
        public static void setDynamicTableMonth(LocalDate time) {
        dynamicTableMonth = time.format(DateTimeFormatter.ofPattern("yyyyMM"));
        dynamicNow = time;
        }
        public static void setDynamicTableMonth(Date time) {
        ......@@ -73,23 +72,11 @@ public class CollectDataTask {
        }
        /**
        * This method is for debug
        */
        // @EnablePsspSchProtect
        // @Scheduled(cron = "0 0/1 * * * ?")
        public void checkPsspDataCount() {
        long simcCount = remotePsspRep.count();
        long apiCount = localPsspRep.count();
        log.debug("[pssp] [count_check] simc={} api={}",
        simcCount, apiCount);
        }
        /**
        * 抓取更新的网站的数据,当前为1h/次
        */
        @EnablePsspSchProtect
        // @Scheduled(cron = "0 0 0/1 * * ?")
        @Scheduled(cron = "0 0 0/1 * * ?")
        public void collectWebsiteData() {
        log.info("[pssp] [定时任务]抓取更新的网站数据");
        websiteService.fetchUpdatedData();
        ......@@ -109,7 +96,7 @@ public class CollectDataTask {
        * 抓取更新的报警数据,当前为1min/次
        */
        @EnablePsspSchProtect
        // @Scheduled(cron = "30 0/1 * * * ?")
        @Scheduled(cron = "30 0/1 * * * ?")
        public void collectAlertData() {
        log.info("[pssp] [定时任务] 抓取更新的报警数据");
        alertService.fetchUpdatedData();
        ......@@ -122,7 +109,7 @@ public class CollectDataTask {
        * <li>3.执行数据更新任务</li>
        * <li>4.更正时间,开启其他计划任务></li>
        */
        // @Scheduled(cron = "0 10 0 1 * ?")
        @Scheduled(cron = "0 10 0 1 * ?")
        @EnablePsspSchProtect
        public void replenishMysqlDataMonthly() {
        log.info("[pssp] [定时任务] 正在执行月末数据补偿任务,暂停其他计划任务,要补偿的月份数据是 {}", LocalDate.now().minusMonths(1L).toString());
        ......@@ -133,5 +120,4 @@ public class CollectDataTask {
        }
        }
        acq-pssp/src/main/java/com/zjty/fp/acq/pssp/utils/DicMapUtil.java 0 → 100644
        浏览文件 @ 07447b81
        package com.zjty.fp.acq.pssp.utils;
        import com.google.common.collect.Lists;
        import lombok.extern.slf4j.Slf4j;
        import java.io.File;
        import java.io.IOException;
        import java.nio.file.Files;
        import java.nio.file.Path;
        import java.nio.file.Paths;
        import java.util.ArrayList;
        import java.util.HashMap;
        import java.util.List;
        import java.util.Map;
        /**
        * DicMapUtil.
        *
        * @author Matrix <xhyrzldf@gmail.com>
        * @since 2020/7/29 at 10:54 上午
        */
        @Slf4j
        public class DicMapUtil {
        /**
        * 将MAP转为形如k,v的字符串保存到文件中去,文件名为files/map/website/dictMap.txt
        */
        public static File createDictFile(String catalog, Map<String, Integer> idMap) {
        String pathname = "files/map/" + catalog + "/";
        String filename = "dictMap.txt";
        Path path = Paths.get(pathname + filename);
        //每次写入前先删除文件从而达到更新的效果
        try {
        log.info("[pssp]正在删除原始dict文件:{}", pathname + filename);
        Files.deleteIfExists(path);
        log.info("[pssp]删除成功，删除文件名为:{}", pathname + filename);
        } catch (IOException e) {
        log.warn("[pssp]删除文件时发生异常,信息为:{}", e.toString());
        }
        boolean make = createFilePath(new File(pathname));
        List<String> stringList = new ArrayList<>();
        idMap.forEach((k, v) -> stringList.add(k + "," + v));
        log.info("[pssp] 正在重新生成dict文件 : {}",path);
        if (make) {
        try {
        Files.write(path, stringList);
        log.info("[pssp] 重新生成 {} 文件成功",path);
        } catch (IOException e) {
        log.info("[pssp]生成文件时出现异常：{}", e.toString());
        }
        }
        return new File(pathname + filename);
        }
        public static Map<String, Integer> readDictFile(String catalog) {
        List<String> data = Lists.newArrayList();
        String localFilePath = "files/map/" + catalog + "/";
        String fileName = "dictMap.txt";
        File file = new File(localFilePath + fileName);
        Path path = Paths.get(localFilePath + fileName);
        try {
        data = Files.readAllLines(path);
        } catch (IOException e) {
        log.info("读取文件时出现异常：" + e);
        }
        //得到的data是形如2018-10,5的字符串
        Map<String, Integer> map = new HashMap<>();
        for (String d : data) {
        String[] strings = d.split(",");
        map.put(strings[0], Integer.valueOf(strings[1]));
        }
        return map;
        }
        /**
        * 生成指定的文件夹路径
        * 若文件夹不存在则创建
        */
        private static boolean createFilePath(File file) {
        return file.exists() || file.mkdirs();
        }
        }
        acq-union/pom.xml
        浏览文件 @ 07447b81
        ......@@ -76,12 +76,6 @@
        <artifactId>acq-vomp</artifactId>
        </dependency>
        <dependency>
        <groupId>com.zjty.fp</groupId>
        <artifactId>acq-jwt</artifactId>
        <version>0.0.1-SNAPSHOT</version>
        </dependency>
        <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
        ......
        acq-union/src/main/java/com/zjty/fp/acq/union/UnionApplication.java
        浏览文件 @ 07447b81
        ......@@ -18,8 +18,7 @@ import org.springframework.scheduling.annotation.EnableScheduling;
        "com.zjty.fp.acq.union",
        "com.zjty.fp.acq.pssp",
        "com.zjty.fp.acq.vomp",
        "com.zjty.fp.acq.misc",
        "com.zjty.fp.acq.jwt"
        "com.zjty.fp.acq.misc"
        })
        @EnableScheduling
        public class UnionApplication {
        ......
        acq-union/src/main/java/com/zjty/fp/acq/union/config/DataSourceConfig.java
        浏览文件 @ 07447b81
        ......@@ -52,120 +52,4 @@ public class DataSourceConfig {
        public DataSource vompDataSource() {
        return DruidDataSourceBuilder.create().build();
        }
        /**
        * fp-api数据源,用于正常提供api服务
        */
        @Bean(name = "locationDataSource")
        @Qualifier("locationDataSource")
        public DataSource secondaryDataSource() throws SQLException {
        ShardingRuleConfiguration shardingRuleConfig = new ShardingRuleConfiguration();
        shardingRuleConfig.getTableRuleConfigs().add(getAlertTableRuleConfiguration());
        shardingRuleConfig.getBindingTableGroups().add("pssp_alert");
        return createDataSource(
        createDataSourceMap(),
        shardingRuleConfig,
        new HashMap<>(),
        new Properties());
        }
        /**
        * fp-api数据源服务于{@link ##secondaryDataSource()}方法
        */
        private Map<String, DataSource> createDataSourceMap() throws SQLException {
        DruidDataSource dataSource = new DruidDataSource();
        dataSource.setDriverClassName(env.getProperty("spring.datasource.location.driver-class-name"));
        dataSource.setUrl(env.getProperty("spring.datasource.location.url"));
        dataSource.setUsername(env.getProperty("spring.datasource.location.username"));
        dataSource.setPassword(env.getProperty("spring.datasource.location.password"));
        /* 配置初始化大小、最小、最大 */
        dataSource.setDbType("mysql");
        dataSource.setInitialSize(4);
        dataSource.setMinIdle(4);
        dataSource.setMaxActive(20);
        /* 配置获取连接等待超时的时间 */
        dataSource.setMaxWait(60000);
        /* 配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒 */
        dataSource.setTimeBetweenEvictionRunsMillis(60000);
        /* 配置一个连接在池中最小生存的时间，单位是毫秒 */
        dataSource.setMinEvictableIdleTimeMillis(300000);
        dataSource.setValidationQuery("SELECT 'x'");
        dataSource.setTestWhileIdle(true);
        dataSource.setTestOnBorrow(true);
        dataSource.setTestOnReturn(false);
        /* 打开PSCache，并且指定每个连接上PSCache的大小。
        如果用Oracle，则把poolPreparedStatements配置为true，
        mysql可以配置为false。分库分表较多的数据库，建议配置为false */
        dataSource.setPoolPreparedStatements(false);
        dataSource.setMaxPoolPreparedStatementPerConnectionSize(20);
        /* 配置监控统计拦截的filters */
        dataSource.setFilters("stat,wall,log4j");
        Map<String, DataSource> result = new HashMap<>();
        result.put("ds0", dataSource);
        return result;
        }
        /**
        * 设置表规则
        */
        private TableRuleConfiguration getAlertTableRuleConfiguration() {
        String actualDataNodes = "" +
        "ds0.pssp_alert_201712," +
        "ds0.pssp_alert_201807," +
        "ds0.pssp_alert_201808," +
        "ds0.pssp_alert_201809," +
        "ds0.pssp_alert_201810," +
        "ds0.pssp_alert_201811," +
        "ds0.pssp_alert_201812," +
        "ds0.pssp_alert_201901," +
        "ds0.pssp_alert_201902," +
        "ds0.pssp_alert_201903," +
        "ds0.pssp_alert_201904," +
        "ds0.pssp_alert_201905," +
        "ds0.pssp_alert_201906," +
        "ds0.pssp_alert_201907," +
        "ds0.pssp_alert_201908," +
        "ds0.pssp_alert_201909," +
        "ds0.pssp_alert_201910," +
        "ds0.pssp_alert_201911," +
        "ds0.pssp_alert_201912," +
        "ds0.pssp_alert_202001," +
        "ds0.pssp_alert_202002," +
        "ds0.pssp_alert_202003," +
        "ds0.pssp_alert_202004," +
        "ds0.pssp_alert_202005," +
        "ds0.pssp_alert_202006," +
        "ds0.pssp_alert_202007," +
        "ds0.pssp_alert_202008," +
        "ds0.pssp_alert_202009," +
        "ds0.pssp_alert_202010," +
        "ds0.pssp_alert_202011," +
        "ds0.pssp_alert_202012," +
        "ds0.pssp_alert_202101," +
        "ds0.pssp_alert_202102," +
        "ds0.pssp_alert_202103," +
        "ds0.pssp_alert_202104," +
        "ds0.pssp_alert_202105," +
        "ds0.pssp_alert_202106," +
        "ds0.pssp_alert_202107," +
        "ds0.pssp_alert_202108";
        TableRuleConfiguration result = new TableRuleConfiguration();
        result.setLogicTable("pssp_alert");
        result.setActualDataNodes(actualDataNodes);
        result.setKeyGeneratorColumnName("primary_id");
        result.setTableShardingStrategyConfig(new StandardShardingStrategyConfiguration(
        "tm_fetch",
        new AlertPreShardAlgo("pssp_alert_"),
        new AlertRangeShardAlgo("pssp_alert_")));
        return result;
        }
        }
        acq-union/src/main/java/com/zjty/fp/acq/union/config/LocationConfig.java deleted 100644 → 0
        浏览文件 @ d3f8c6c3
        package com.zjty.fp.acq.union.config;
        import org.springframework.beans.factory.annotation.Autowired;
        import org.springframework.beans.factory.annotation.Qualifier;
        import org.springframework.boot.autoconfigure.orm.jpa.JpaProperties;
        import org.springframework.boot.context.properties.EnableConfigurationProperties;
        import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
        import org.springframework.context.annotation.Bean;
        import org.springframework.context.annotation.Configuration;
        import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
        import org.springframework.orm.jpa.JpaTransactionManager;
        import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
        import org.springframework.transaction.PlatformTransactionManager;
        import org.springframework.transaction.annotation.EnableTransactionManagement;
        import javax.persistence.EntityManager;
        import javax.sql.DataSource;
        import java.util.Map;
        @Configuration
        @EnableTransactionManagement
        @EnableJpaRepositories(
        entityManagerFactoryRef = "entityManagerFactoryLocation",
        transactionManagerRef = "transactionManagerLocation",
        basePackages = {
        "com.zjty.fp.acq.pssp.subject.repository.location",
        "com.zjty.fp.acq.vomp.subject.repository.local",
        }
        )
        @EnableConfigurationProperties(JpaProperties.class)
        public class LocationConfig {
        @Autowired
        @Qualifier("locationDataSource")
        private DataSource locationDataSource;
        @Autowired
        private JpaProperties jpaProperties;
        @Bean(name = "entityManageLocation")
        public EntityManager entityManager(EntityManagerFactoryBuilder builder) {
        return entityManagerFactoryLocation(builder).getObject().createEntityManager();
        }
        @Bean(name = "entityManagerFactoryLocation")
        public LocalContainerEntityManagerFactoryBean entityManagerFactoryLocation(EntityManagerFactoryBuilder builder) {
        return builder
        .dataSource(locationDataSource)
        .properties(getVendorProperties(locationDataSource))
        .packages(
        "com.zjty.fp.acq.pssp.subject.entity.location",
        "com.zjty.fp.acq.vomp.subject.entity.localdo")
        .persistenceUnit("locationPersistenceUnit")
        .build();
        }
        private Map<String, String> getVendorProperties(DataSource dataSource) {
        return jpaProperties.getHibernateProperties(dataSource);
        }
        // @Primary
        @Bean(name = "transactionManagerLocation")
        public PlatformTransactionManager transactionManagerLocation(EntityManagerFactoryBuilder builder) {
        return new JpaTransactionManager(entityManagerFactoryLocation(builder).getObject());
        }
        }
        acq-union/src/main/resources/application-dev.properties
        浏览文件 @ 07447b81
        # suppress inspection "SpringBootApplicationProperties" for whole file
        spring.application.name=fp-acq-wz
        # 数据库one相关配置 21.18.29.98:3306/db_secret_alert.
        # 数据库one相关配置 21.18.29.98:3306/db_secret_alert. 这里接入的是PSSP的数据源
        spring.datasource.remote.driver-class-name=com.mysql.jdbc.Driver
        spring.datasource.remote.url=jdbc:mysql://localhost:3306/fp_simc?useSSL=false&serverTimezone=UTC&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&allowPublicKeyRetrieval=true
        spring.datasource.remote.username=fp
        ......@@ -27,33 +27,8 @@ spring.datasource.remote.test-on-borrow=true
        spring.datasource.remote.test-on-return=false
        spring.datasource.remote.test-while-idle=true
        # 数据库two相关配置
        spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
        spring.datasource.location.driver-class-name=com.mysql.jdbc.Driver
        spring.datasource.location.url=jdbc:mysql://localhost:3306/fp_api_empty?useSSL=false&serverTimezone=UTC&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&allowPublicKeyRetrieval=true
        spring.datasource.location.username=fp
        spring.datasource.location.password=fp123456
        ## StatFilter
        spring.datasource.location.filter.stat.db-type=mysql
        spring.datasource.location.initial-size=4
        spring.datasource.location.min-idle=4
        spring.datasource.location.max-active=20
        ## 配置获取连接等待超时的时间 ms
        spring.datasource.location.max-wait=60000
        ## 打开PSCache，并且指定每个连接上PSCache的大小
        spring.datasource.location.pool-prepared-statements=true
        spring.datasource.location.max-open-prepared-statements=20
        ## 配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒
        spring.datasource.location.time-between-eviction-runs-millis=60000
        ## 配置一个连接在池中最小生存的时间，单位是毫秒
        spring.datasource.location.min-evictable-idle-time-millis=3000
        ## 配置查询语句验证(用于查询是否还在连接的语句)
        spring.datasource.location.validation-query=SELECT 1
        spring.datasource.location.test-on-borrow=true
        spring.datasource.location.test-on-return=false
        spring.datasource.location.test-while-idle=true
        # 数据库three相关配置
        # 数据库three相关配置 这里接入的是VOMP的数据源
        spring.datasource.remote2.driver-class-name=com.mysql.jdbc.Driver
        spring.datasource.remote2.url=jdbc:mysql://localhost:3306/fp_simc?useSSL=false&serverTimezone=UTC&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&allowPublicKeyRetrieval=true
        spring.datasource.remote2.username=fp
        ......
        acq-union/src/main/resources/application-wz.properties
        浏览文件 @ 07447b81
        # suppress inspection "SpringBootApplicationProperties" for whole file
        spring.application.name=fp-acq-wz
        # 数据库one相关配置 21.18.29.98:3306/db_secret_alert.
        # 数据库one相关配置 21.18.29.98:3306/db_secret_alert. 这里接入的是PSSP的数据源
        spring.datasource.remote.driver-class-name=com.mysql.jdbc.Driver
        spring.datasource.remote.url=jdbc:mysql://localhost:3306/fp_simc?useSSL=false&serverTimezone=UTC&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&allowPublicKeyRetrieval=true
        spring.datasource.remote.username=fp
        ......@@ -27,33 +27,8 @@ spring.datasource.remote.test-on-borrow=true
        spring.datasource.remote.test-on-return=false
        spring.datasource.remote.test-while-idle=true
        # 数据库two相关配置
        spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
        spring.datasource.location.driver-class-name=com.mysql.jdbc.Driver
        spring.datasource.location.url=jdbc:mysql://localhost:3306/fp_api?useSSL=false&serverTimezone=UTC&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&allowPublicKeyRetrieval=true
        spring.datasource.location.username=fp
        spring.datasource.location.password=fp123456
        ## StatFilter
        spring.datasource.location.filter.stat.db-type=mysql
        spring.datasource.location.initial-size=4
        spring.datasource.location.min-idle=4
        spring.datasource.location.max-active=20
        ## 配置获取连接等待超时的时间 ms
        spring.datasource.location.max-wait=60000
        ## 打开PSCache，并且指定每个连接上PSCache的大小
        spring.datasource.location.pool-prepared-statements=true
        spring.datasource.location.max-open-prepared-statements=20
        ## 配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒
        spring.datasource.location.time-between-eviction-runs-millis=60000
        ## 配置一个连接在池中最小生存的时间，单位是毫秒
        spring.datasource.location.min-evictable-idle-time-millis=3000
        ## 配置查询语句验证(用于查询是否还在连接的语句)
        spring.datasource.location.validation-query=SELECT 1
        spring.datasource.location.test-on-borrow=true
        spring.datasource.location.test-on-return=false
        spring.datasource.location.test-while-idle=true
        # 数据库three相关配置
        # 数据库three相关配置 这里接入的是VOMP的数据源
        spring.datasource.remote2.driver-class-name=com.mysql.jdbc.Driver
        spring.datasource.remote2.url=jdbc:mysql://localhost:3306/fp_simc?useSSL=false&serverTimezone=UTC&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&allowPublicKeyRetrieval=true
        spring.datasource.remote2.username=fp
        ......
        pom.xml
        浏览文件 @ 07447b81
        ......@@ -29,7 +29,6 @@
        <module>acq-vomp</module>
        <module>acq-union</module>
        <module>acq-misc</module>
        <module>acq-jwt</module>
        </modules>
        <dependencyManagement>
        ......@@ -58,12 +57,6 @@
        <version>0.0.1-SNAPSHOT</version>
        </dependency>
        <dependency>
        <groupId>com.zjty.fp</groupId>
        <artifactId>acq-jwt</artifactId>
        <version>0.0.1-SNAPSHOT</version>
        </dependency>
        </dependencies>
        </dependencyManagement>
        ......
        Markdown 格式
        0% 或
        您添加了 0 人 到此讨论。请谨慎行事。
        请先完成此评论的编辑！
        请 注册 或者 后发表评论