一些BUG修复与优化

acq-misc/src/main/java/com/zjty/fp/acq/misc/entity/ImportAds.java

+package com.zjty.fp.acq.misc.entity;
+
+/**
+ * ImportAds.
+ *
+ * @author Matrix <xhyrzldf@gmail.com>
+ * @since 2020/7/29 at 6:31 下午
+ */
+public class ImportAds {
+
+    public static final String IMPORT_URL = "/Users/matrix/code/test/";
+}

acq-misc/src/main/java/com/zjty/fp/acq/misc/entity/PsspCount.java

@@ -9,12 +9,20 @@ package com.zjty.fp.acq.misc.entity;
 public class PsspCount {
 
     /**
-     * 代表下一个文件应当用的后缀号
+     * 代表下一个报警文件应当用的后缀号
      */
-    public static int count = -1;
+    public static int alert_count = -1;
+
+    /**
+     * 代表下一个网站文件应当用的后缀号
+     */
+    public static int website_count = -1;
 
     public static final String COUNT_PATH = "files/pssp/count";
 
+    public static final String GLOBAL_COUNT_PATH = ImportAds.IMPORT_URL + "files/pssp/count";
+
+    public static final String COUNT_ADDRESS_ALERT = "files/pssp/count/alert.txt";
 
-    public static final String COUNT_ADDRESS = "files/pssp/count/count.txt";
+    public static final String COUNT_ADDRESS_WEBSITE = "files/pssp/count/website.txt";
 }

acq-misc/src/main/java/com/zjty/fp/acq/misc/utils/FileCreator.java

 package com.zjty.fp.acq.misc.utils;
 
 import com.google.common.collect.Lists;
-import com.zjty.fp.acq.misc.entity.PsspCount;
+import com.zjty.fp.acq.misc.entity.ImportAds;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.context.annotation.Scope;
 import org.springframework.stereotype.Component;
@@ -22,7 +22,7 @@ import static com.zjty.fp.acq.misc.entity.PsspCount.*;
 @Component
 public class FileCreator {
     public File createFile(String subName, String catalog, String data) {
-        String pathname = "files/" + subName + "/" + catalog + "/";
+        String pathname = ImportAds.IMPORT_URL + "files/" + subName + "/" + catalog + "/";
         String filename = getFileNameByDate(subName, catalog);
         Path path = Paths.get(pathname + filename);
         boolean make = createFilePath(new File(pathname));
@@ -36,10 +36,10 @@ public class FileCreator {
         return new File(pathname + filename);
     }
 
-    public File createFileAndZip(String subName, String catalog, String data) {
+    public File createFileAndZip(String countAddress, String subName, String catalog, String data) {
         String zipData = DeflaterUtils.zipString(data);
-        String pathname = "files/" + subName + "/" + catalog + "/";
-        String filename = getPsspFileName(subName, catalog);
+        String pathname = ImportAds.IMPORT_URL + "files/" + subName + "/" + catalog + "/";
+        String filename = getPsspFileName(countAddress, subName, catalog);
         Path path = Paths.get(pathname + filename);
         boolean make = createFilePath(new File(pathname));
         if (make) {
@@ -51,12 +51,29 @@ public class FileCreator {
         }
 
         //生成完毕后需要更新一下count的值
-        count += 1;
+        updateCount(countAddress);
+        return new File(pathname + filename);
+    }
+
+    /**
+     * 更新Count值，Count+1
+     */
+    private void updateCount(String countAddress) {
+        if (countAddress.equals(COUNT_ADDRESS_ALERT)) {
+            alert_count += 1;
+        } else if (countAddress.equals(COUNT_ADDRESS_WEBSITE)) {
+            website_count += 1;
+        }
         //写入到count文件中去
-        Path countPath = Paths.get(COUNT_ADDRESS);
+        Path countPath = Paths.get(countAddress);
+
+        //再写入一份到单导另外一端供API端参考
+        String globalPath = ImportAds.IMPORT_URL + countAddress;
+        Path globalCountPath = Paths.get(globalPath);
         //先删除原有的count文件，再写入现有的
         try {
             Files.deleteIfExists(countPath);
+            Files.deleteIfExists(countPath);
         } catch (IOException e) {
             log.warn("删除原count文件失败!原因:{}", e.toString());
         }
@@ -65,13 +82,22 @@ public class FileCreator {
 
         if (countMake) {
             try {
-                Files.write(Paths.get(COUNT_ADDRESS), String.valueOf(count).getBytes());
-                log.info("[pssp]磁盘count值更新成功:count = {}", count);
+                if (countAddress.equals(COUNT_ADDRESS_ALERT)) {
+                    Files.write(countPath, String.valueOf(alert_count).getBytes());
+                    log.info("[pssp] 磁盘 count_alert 值更新成功:count = {}", alert_count);
+                    Files.write(globalCountPath, String.valueOf(alert_count).getBytes());
+                    log.info("[pssp] 全局 count_alert 值更新成功:count = {}", alert_count);
+
+                } else if (countAddress.equals(COUNT_ADDRESS_WEBSITE)) {
+                    Files.write(countPath, String.valueOf(website_count).getBytes());
+                    log.info("[pssp] 磁盘 count_website 值更新成功:count = {}", website_count);
+                    Files.write(globalCountPath, String.valueOf(website_count).getBytes());
+                    log.info("[pssp] 全局 count_website 值更新成功:count = {}", website_count);
+                }
             } catch (IOException e) {
                 log.info("[pssp]生成文件时出现异常：" + e);
             }
         }
-        return new File(pathname + filename);
     }
 
     public File createFileNoTime(String subName, String catalog, String data) {
@@ -183,22 +209,28 @@ public class FileCreator {
         return subName + "-" + catalog + ".txt";
     }
 
-    private String getPsspFileName(String subName, String catalog) {
-
+    private String getPsspFileName(String countAddress, String subName, String catalog) {
 
         //count 先从内存里取读 内存里读不到从磁盘读
-        int count = PsspCount.count;
+        int count = -1;
+        if (countAddress.equals(COUNT_ADDRESS_ALERT)) {
+            count = alert_count;
+
+        } else if (countAddress.equals(COUNT_ADDRESS_WEBSITE)) {
+            count = website_count;
+
+        }
 
         //如果count为初始值-1,则代表还没有从文件中读取之前的记录
         if (count == -1) {
             //读取文件获得文件编号后缀值
-            Path path = Paths.get(COUNT_ADDRESS);
+            Path path = Paths.get(countAddress);
             List<String> data = Lists.newArrayList();
 
             try {
                 data = Files.readAllLines(path);
             } catch (IOException e) {
-                log.info("读取文件时出现异常：" + e);
+                log.info("[pssp]读取文件时出现异常：" + e);
             }
             count = data.get(0) == null ? 0 : Integer.parseInt(data.get(0));
         }
@@ -206,11 +238,9 @@ public class FileCreator {
         if (count <= 0) count = 0;
 
         String psspFileName = subName + "-" + catalog + count + ".txt";
-        log.info("成功生成文件名:{}", psspFileName);
-
+        log.info("[pssp]成功生成文件名:{}", psspFileName);
 
         return psspFileName;
-
     }
 
     /**

acq-pssp/src/main/java/com/zjty/fp/acq/pssp/base/beans/PsspInitialRunner.java

 package com.zjty.fp.acq.pssp.base.beans;
 
 import com.google.common.collect.Lists;
+import com.zjty.fp.acq.misc.entity.ImportAds;
 import com.zjty.fp.acq.pssp.subject.service.RegionService;
 import com.zjty.fp.acq.pssp.subject.service.WebsiteService;
 import lombok.extern.slf4j.Slf4j;
@@ -40,40 +41,70 @@ public class PsspInitialRunner {
     @Bean
     public CommandLineRunner initializeDictMap() {
         return args -> {
-            log.info("[pssp] 正在尝试初始化 count 文件");
+            initCount("COUNT_ALERT", COUNT_ADDRESS_ALERT);
+            initCount("COUNT_WEBSITE", COUNT_ADDRESS_WEBSITE);
+        };
+    }
+
+    private void initCount(String countFileName, String CountAddress) {
+        log.info("[pssp] 正在尝试初始化 {} 文件", countFileName);
+
+        boolean countExists = Files.exists(Paths.get(CountAddress));
+        String GlobalCountAddress = ImportAds.IMPORT_URL + CountAddress;
 
+        boolean globalCountExists = Files.exists(Paths.get(GlobalCountAddress));
 
-            boolean countExists = Files.exists(Paths.get(COUNT_ADDRESS));
+        log.info("[pssp] 创建本地Count文件");
+        createCountFile(countFileName, CountAddress, COUNT_PATH, countExists);
 
-            if (countExists) {
-                log.info("[pssp]磁盘已存在count文件，跳过初始化流程，将磁盘的count读取入内存中");
-                //读取文件获得文件编号后缀值
-                Path path = Paths.get(COUNT_ADDRESS);
-                List<String> data = Lists.newArrayList();
+        log.info("[pssp] 创建全局Count文件");
+        createCountFile(countFileName, GlobalCountAddress, GLOBAL_COUNT_PATH, globalCountExists);
+
+    }
+
+    private void createCountFile(String countFileName, String CountAddress, String dirPath, boolean countExists) {
+        int countVar;
+        if (countExists) {
+            log.info("[pssp]磁盘已存在 {} 文件，跳过初始化流程，将磁盘的 {} 读取入内存中", countFileName, countFileName);
+            //读取文件获得文件编号后缀值
+            Path path = Paths.get(CountAddress);
+            List<String> data = Lists.newArrayList();
+
+            try {
+                data = Files.readAllLines(path);
+            } catch (IOException e) {
+                log.info("读取文件时出现异常：" + e);
+            }
+            countVar = data.get(0) == null ? 0 : Integer.parseInt(data.get(0));
 
+            if (CountAddress.equals(COUNT_ADDRESS_ALERT)) {
+                alert_count = countVar;
+            } else if (CountAddress.equals(COUNT_ADDRESS_WEBSITE)) {
+                website_count = countVar;
+            }
+
+        } else {
+            log.info("[pssp] 磁盘不存在 {} 文件，初始化 {} 文件", countFileName, countFileName);
+            boolean countMake = createFilePath(new File(dirPath));
+
+            countVar = 0;
+            if (countMake) {
                 try {
-                    data = Files.readAllLines(path);
+                    Files.write(Paths.get(CountAddress), String.valueOf(countVar).getBytes());
+                    log.info("[pssp]磁盘 {} 值更新成功:count = {}", countFileName, countVar);
                 } catch (IOException e) {
-                    log.info("读取文件时出现异常：" + e);
-                }
-                count = data.get(0) == null ? 0 : Integer.parseInt(data.get(0));
-            } else {
-                log.info("[pssp] 磁盘不存在count文件，初始化count文件");
-                boolean countMake = createFilePath(new File(COUNT_PATH));
-
-                count = 0;
-                if (countMake) {
-                    try {
-                        Files.write(Paths.get(COUNT_ADDRESS), String.valueOf(count).getBytes());
-                        log.info("[pssp]磁盘count值更新成功:count = {}", count);
-                    } catch (IOException e) {
-                        log.info("[pssp]生成文件时出现异常：" + e);
-                    }
+                    log.info("[pssp]生成文件时出现异常：" + e);
                 }
-                log.info("[pssp]初始化count文件完毕!");
             }
+            log.info("[pssp]初始化 {} 文件完毕!", countFileName);
 
-        };
+            if (CountAddress.equals(COUNT_ADDRESS_ALERT)) {
+                alert_count = countVar;
+            } else if (CountAddress.equals(COUNT_ADDRESS_WEBSITE)) {
+                website_count = countVar;
+            }
+
+        }
     }
 
     private boolean createFilePath(File file) {

acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/service/impl/AlertServiceImpl.java

@@ -18,6 +18,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static com.zjty.fp.acq.misc.entity.PsspCount.COUNT_ADDRESS_ALERT;
 import static com.zjty.fp.acq.pssp.task.CollectDataTask.dynamicNow;
 import static java.util.stream.Collectors.toList;
 
@@ -85,7 +86,7 @@ public class AlertServiceImpl implements AlertService {
                     .orElse(0L);
             //1.更新写入本地文件
             String webJson = JacksonUtil.toJSon(updatedData).replace("\n", "");
-            fileCreator.createFileAndZip("pssp", alertName, webJson);
+            fileCreator.createFileAndZip(COUNT_ADDRESS_ALERT,"pssp", alertName, webJson);
             //2.记住最大ID值
             long maxId = updatedData.stream().mapToLong(Alert::getId).max().orElse(0L);
             log.info("[pssp]更新后的最大报警数据id为:{},记录到文件中", maxId);

acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/service/impl/RegionServiceImpl.java

@@ -37,22 +37,7 @@ public class RegionServiceImpl implements RegionService {
     @Override
     public void fetchAllData() {
         log.info("[pssp]准备抓取全部源地区数据");
-
-        List<Region> regionList = remoteRegionRepository.findAll()
-                .stream()
-                .map(RemoteRegion::toDo)
-                .collect(toList());
-
-        log.info("[pssp]源地区数据抓取完成,size = {},准备写入融合平台数据库", regionList.size());
-
-        // 异步
-        CompletableFuture.runAsync(() -> {
-            //更新本地DB
-            log.info("[pssp]正在异步写入网站数据到本地文件");
-            String webJson = JacksonUtil.toJSon(regionList).replace("\n", "");
-            log.info("[pssp]准备写入文件，要写入的文件字符长度为{}", webJson.length());
-            fileCreator.createFileAndZip("pssp", "region", webJson);
-        });
+        log.info("[pssp]本次更新没有抓取地区数据");
     }
 
     @Override

acq-pssp/src/main/java/com/zjty/fp/acq/pssp/subject/service/impl/WebsiteServiceImpl.java

@@ -19,6 +19,7 @@ import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 
+import static com.zjty.fp.acq.misc.entity.PsspCount.COUNT_ADDRESS_WEBSITE;
 import static com.zjty.fp.acq.pssp.task.CollectDataTask.dynamicNow;
 import static java.util.stream.Collectors.toList;
 
@@ -33,14 +34,12 @@ import static java.util.stream.Collectors.toList;
 @Transactional(rollbackOn = Exception.class)
 public class WebsiteServiceImpl implements WebsiteService {
 
+    private static final String websiteName = "website";
     @Autowired
     FileCreator fileCreator;
-
     @Autowired
     private RemoteWebsiteRepository remoteWebsiteRepository;
 
-    private static final String websiteName = "website";
-
     @Override
     public void fetchAllData() {
         log.info("[pssp]准备抓取全部源网站数据");
@@ -53,7 +52,7 @@ public class WebsiteServiceImpl implements WebsiteService {
             log.info("[pssp]正在异步写入网站数据到本地文件");
             String webJson = JacksonUtil.toJSon(websiteList).replace("\n", "");
             log.info("[pssp]准备写入文件，要写入的文件字符长度为{}", webJson.length());
-            fileCreator.createFileAndZip("pssp", "website", webJson);
+            fileCreator.createFileAndZip(COUNT_ADDRESS_WEBSITE, "pssp", "website", webJson);
             log.info("[pssp]异步网站数据写入完成");
         });
     }
@@ -62,8 +61,7 @@ public class WebsiteServiceImpl implements WebsiteService {
     public void fetchUpdatedData() {
         log.info("[pssp]正在执行网站数据的增量更新");
         //读取文件中的最新id , 获得当前月的时间
-        String key = dynamicNow.getYear() + "-" + dynamicNow.getMonthValue();
-        Integer lastedId = DicMapUtil.readDictFile(websiteName).getOrDefault(key, 0);
+        Integer lastedId = DicMapUtil.readDictFile(websiteName).getOrDefault("web", 0);
         if (lastedId == 0) {
             log.info("[pssp] 当前文件中没有任何网站,请先执行网站历史采集任务");
             return;
@@ -83,12 +81,12 @@ public class WebsiteServiceImpl implements WebsiteService {
             CompletableFuture.runAsync(() -> {
                 log.info("[pssp]正在异步写入网站更新数据到本地文件");
                 String webJson = JacksonUtil.toJSon(updatedData).replace("\n", "");
-                fileCreator.createFileAndZip("pssp", "website", webJson);
+                fileCreator.createFileAndZip(COUNT_ADDRESS_WEBSITE, "pssp", "website", webJson);
                 //写完要把最新的id给记录住
                 long maxId = updatedData.stream().mapToLong(Website::getId).max().orElse(0L);
                 log.info("[pssp]更新后的最大网站数据id为:{},记录到文件中", maxId);
                 Map<String, Integer> map = new HashMap<>();
-                map.put(key, (int) maxId);
+                map.put("web", (int) maxId);
                 DicMapUtil.createDictFile(websiteName, map);
                 log.info("[pssp]异步网站更新数据写入完成");
             });

acq-pssp/src/main/java/com/zjty/fp/acq/pssp/task/CollectDataTask.java

@@ -55,6 +55,8 @@ public class CollectDataTask {
 
     private int count = 0;
 
+    private int checkCount = 0;
+
     @Autowired
     private RemoteAlertRepository remotePsspRep;
 
@@ -76,7 +78,7 @@ public class CollectDataTask {
      * 抓取更新的网站的数据,当前为1h/次
      */
     @EnablePsspSchProtect
-    @Scheduled(cron = "0 0 0/1 * * ?")
+    @Scheduled(cron = "0 0/1 * * * ?")
     public void collectWebsiteData() {
         log.info("[pssp] [定时任务]抓取更新的网站数据");
         websiteService.fetchUpdatedData();
@@ -102,6 +104,26 @@ public class CollectDataTask {
         alertService.fetchUpdatedData();
     }
 
+    /**
+     * 用于检查计划任务开关的任务，如果开关连续5次检查失败，则手动打开
+     * 该任务每30分钟执行一次
+     */
+    @EnablePsspSchProtect
+    @Scheduled(cron = "0 0/30 * * * ?")
+    public void checkTrigger() {
+        log.info("[pssp] 开关检查任务，当前trigger为 {}", trigger);
+        if (trigger == false) {
+            checkCount++;
+            log.info("[pssp] 记录一次开关关闭次数,当前次数为 : {}", checkCount);
+        }
+
+        if (checkCount >= 5) {
+            log.info("[pssp] 检测到开关关闭时间过长,当前关闭次数为 {} ,重新打开计划任务开关", checkCount);
+            trigger = true;
+            checkCount = 0;
+        }
+    }
+
     /**
      * 在每个月月初的第10分钟，执行一次上个月的数据丢失补偿任务,大体逻辑如下
      * <li>1.关闭其他计划任务</li>

acq-union/src/main/resources/application-wz.properties

@@ -3,10 +3,10 @@ spring.application.name=fp-acq-wz
 
 # 数据库one相关配置 21.18.29.98:3306/db_secret_alert. 这里接入的是PSSP的数据源
 spring.datasource.remote.driver-class-name=com.mysql.jdbc.Driver
-spring.datasource.remote.url=jdbc:mysql://localhost:3306/fp_simc?useSSL=false&serverTimezone=UTC&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&allowPublicKeyRetrieval=true
-spring.datasource.remote.username=fp
+spring.datasource.remote.url=jdbc:mysql://21.28.120.10:3306/db_secret_alert?useSSL=false&serverTimezone=UTC&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&allowPublicKeyRetrieval=true
+spring.datasource.remote.username=root
 #spring.datasource.remote.password=efield-tech
-spring.datasource.remote.password=fp123456
+spring.datasource.remote.password=efield-tech
 ## StatFilter
 spring.datasource.remote.filter.stat.db-type=mysql
 spring.datasource.remote.initial-size=4
@@ -29,10 +29,10 @@ spring.datasource.remote.test-while-idle=true
 
 
 # 数据库three相关配置 这里接入的是VOMP的数据源
-spring.datasource.remote2.driver-class-name=com.mysql.jdbc.Driver
-spring.datasource.remote2.url=jdbc:mysql://localhost:3306/fp_simc?useSSL=false&serverTimezone=UTC&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&allowPublicKeyRetrieval=true
-spring.datasource.remote2.username=fp
-spring.datasource.remote2.password=fp123456
+spring.datasource.driver-class-name=com.microsoft.sqlserver.jdbc.SQLServerDriver
+spring.datasource.url=jdbc:sqlserver://21.28.120.2:1433;DatabaseName=PowerMonJava;
+spring.datasource.username=ms
+spring.datasource.password=ms123456
 # ftp
 ftp.server.ip=192.168.1.159
 ftp.server.port=2121

acq-union/src/main/resources/application.properties

 # suppress inspection "SpringBootApplicationProperties" for whole file
 # 服务器端口配置
-server.port=8087
+server.port=8086
 spring.profiles.active=@activatedProperties@
 # jpa
 # spring.jpa.properties.hibernate.format_sql=true