提交 33c5aa96 authored 作者: wyl's avatar wyl

inspect

上级 c841611b
......@@ -24,6 +24,12 @@
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!--webSocket 后台向前端推送消息-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-websocket</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
......
......@@ -3,6 +3,7 @@ package com.zjty.adaptationmaster.adaptor.enginer;
import com.zjty.adaptationmaster.adaptor.entity.AdaptationDetailsLogEntity;
import com.zjty.adaptationmaster.adaptor.entity.OriginalFile;
import com.zjty.adaptationmaster.adaptor.entity.Rule;
import com.zjty.adaptationmaster.base.enums.Const;
import com.zjty.adaptationmaster.managerment.dao.AdaptationDetailLogEntityDao;
import com.zjty.adaptationmaster.managerment.dao.OriginalFileDao;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -24,8 +25,8 @@ public class Adaptor {
//private AdaptationDetailLogEntityDao adaptationDetailLogEntityDao;
@Autowired
private OriginalFileDao originalFileDao;
@Value("${base.path}")
private String basePath;
//@Value("${base.path}")
private String basePath = Const.CONSOLE;
//线程池数量,合适的线程数量能让程序更快
private static final int poolSize = 20;
//为避免内存溢出,对于超大文件,切分成部分分别读入处理,最后拼接写出
......@@ -105,7 +106,7 @@ public class Adaptor {
if (linesRepository.put(s)) {
List<ReadedFile> readedFiles = new ArrayList<>();
readedFiles.add(new ReadedFile(linesRepository.getReadedFiles(), attrs, file,thisFileMatched));
ReadedFileTask apacheTask = new ReadedFileTask(readedFiles,responseWriter);
ReadedFileTask apacheTask = new ReadedFileTask(readedFiles);
apacheTask.setBySort(bySort);
apacheTask.setIndex(i);
pool.submit(apacheTask);
......@@ -114,7 +115,7 @@ public class Adaptor {
}
List<ReadedFile> readedFiles = new ArrayList<>();
readedFiles.add(new ReadedFile(linesRepository.getReadedFiles(), attrs, file,thisFileMatched));
ReadedFileTask apacheTask = new ReadedFileTask(readedFiles,responseWriter);
ReadedFileTask apacheTask = new ReadedFileTask(readedFiles);
apacheTask.setBySort(bySort);
apacheTask.setIndex(i);
pool.submit(apacheTask);
......@@ -124,7 +125,7 @@ public class Adaptor {
String s = new String(Files.readAllBytes(originalPath));
ReadedFile readedFile = new ReadedFile(s, attrs, file,thisFileMatched);
if (repository.put(readedFile)) {
ReadedFileTask apacheTask = new ReadedFileTask(repository.getReadedFiles(), responseWriter);
ReadedFileTask apacheTask = new ReadedFileTask(repository.getReadedFiles());
pool.submit(apacheTask);
}
}
......@@ -134,7 +135,7 @@ public class Adaptor {
return FileVisitResult.CONTINUE;
}
});
ReadedFileTask apacheTask = new ReadedFileTask(repository.getReadedFiles(), responseWriter);
ReadedFileTask apacheTask = new ReadedFileTask(repository.getReadedFiles());
pool.submit(apacheTask);
pool.shutdown();
try {
......
package com.zjty.adaptationmaster.adaptor.enginer;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class AdaptorEntity {
/**
* 匹配模式及处理方式
* 文件匹配模式
* https://blog.csdn.net/qq_36263601/article/details/74260553
* 文本位置匹配模式
* 全文匹配
* 文本处理模式
* 全文替换
*/
private MatchType matchType;//匹配方式
private String parent;//文件路径匹配
private String matching;//文本匹配
private String replacing;//更改方式
}
package com.zjty.adaptationmaster.adaptor.enginer;
public class Counter {
private int i = 0;
public void plus(){
i++;
}
public int getI() {
return i;
}
public void reset(){
i = 0;
}
}
package com.zjty.adaptationmaster.adaptor.enginer;
import org.apache.commons.io.FileUtils;
import java.io.*;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.*;
public class Engine {
//判断一个文件够不够称作大文件的阈值,大文件单独占用一个线程,小文件凑够阈值合用一个线程
private final static long LIMIT = 200*1024;
private String path;
private String projectName;
private List<AdaptorEntity> adaptorEntities;
private ThreadPoolExecutor pool;
public int threadNum;
public void setPath(String path) {
this.path = path;
}
public void setProjectName(String name){
this.projectName = name;
}
public void initAdaptorEntities(List adaptorEntities){
this.adaptorEntities = adaptorEntities;
}
public void setThreadNum(int threadNum) {
this.threadNum = threadNum;
}
public void addAdaptorEntities(List adaptorEntities){
if(this.adaptorEntities == null)this.adaptorEntities = new ArrayList<>();
this.adaptorEntities.addAll(adaptorEntities);
}
long beginTime ;
public void enginer(){
pool = new ThreadPoolExecutor(threadNum, threadNum, 10, TimeUnit.SECONDS, new ArrayBlockingQueue<>(400));
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
beginTime = System.currentTimeMillis();
System.out.println(beginTime+"开始时间,线程池数量"+threadNum);
doDir(path);
pool.shutdown();
try {
//该方法会被阻塞,直到子线程全部执行完毕,并且shutdown()方法被调用,或者等待时间超时。这里等待时间设置一小时
pool.awaitTermination(1,TimeUnit.HOURS);
} catch (InterruptedException e) {
e.printStackTrace();
}
long l = System.currentTimeMillis();
System.out.println("所有线程执行完毕"+l+"总时间差"+(l-beginTime));
}
final Counter i = new Counter();
SmallFileRepository repository = new SmallFileRepository();
private void doDir(String path){
//final PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher("glob:**/*.{java,txt,properties}");
//Collection<File> files = FileUtils.listFiles(new File(path), null, true);
Iterator<File> fileIterator = FileUtils.iterateFiles(new File(path), new String[]{"java","txt","properties"}, true);
while (fileIterator.hasNext()){
File next = fileIterator.next();
long length = next.length();
if(length>LIMIT) {
System.out.println("大文件出没");
List<File> files = new ArrayList<>();
files.add(next);
execute(files);
} else {
repository.fileList.add(next);
repository.length+=length;
if(repository.length>LIMIT){
execute(repository.fileList);
repository.length=0;
repository.fileList = new ArrayList<>();
}
}
i.plus();
//List<String> stringList = FileUtils.readLines(fileIterator.next(), "utf-8");
//execute(stringFactory);
}
// try {
// Files.walkFileTree(Paths.get(path),new SimpleFileVisitor<Path>(){
// @Override
// public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
//
// StringTask task = new StringTask(file,pathMatcher);
// execute(task);
// i.plus();
// //过滤文件名后缀
//// if(pathMatcher.matches(file)){
//// i.plus();
//// List<String> list = Files.readAllLines(file);
//// if(list.size()>200){
//// //System.out.println("文件够大,开线程"+file);
//// execute(new StringFactory().setProduct(list));
//// //new StringFactory().setProduct(list).run();
//// }else {
//// //System.out.println("文件不够大,存起来"+file);
//// stringList.addAll(list);
//// if(stringList.size()>200){
//// //System.out.println("存够了,开线程");
//// ArrayList<String> strings = new ArrayList<>();
//// strings.addAll(stringList);
//// execute(new StringFactory().setProduct(strings));
//// //new StringFactory().setProduct(strings).run();
//// stringList.clear();
//// }
//// }
//// }
// return FileVisitResult.CONTINUE;
// }
// @Override
// public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
// return FileVisitResult.CONTINUE;
// }
//};
// if(stringList.size()>0){
// ArrayList<String> strings = new ArrayList<>();
// strings.addAll(stringList);
// execute(new StringFactory().setProduct(strings));
// //new StringFactory().setProduct(strings).run();
// }
long endTime = System.currentTimeMillis();
System.out.println(endTime+"结束时间");
System.out.println(i.getI()+"处理总数量");
System.out.println((endTime-beginTime)+"时间差");
i.reset();
}
List<Runnable> rejectedTask = new ArrayList<>();
private void execute(List<File> files){
// ApacheTask stringFactory = new ApacheTask();
// stringFactory.setFile(files);
// pool.submit(stringFactory);
}
/**
* 文件如果过小,先存起来,大小存够了再交给一个线程去处理
*/
private class SmallFileRepository{
public long length = 0;
public List<File> fileList = new ArrayList<>();
}
public static void main(String[] args) {
Engine engine = new Engine();
engine.setPath("C:\\Users\\wyl\\Desktop\\testReplace");
//engine.setPath("C:\\Users\\wyl\\Desktop\\testReplace\\testReplace1\\java\\com\\zjty\\hrmanager\\base\\request");
engine.setProjectName("testReplace");
List<AdaptorEntity> adaptorEntities = new ArrayList<>();
adaptorEntities.add(new AdaptorEntity(MatchType.ALL,"**/resources/*.properties","log4j","log5j"));
engine.initAdaptorEntities(adaptorEntities);
for(int i = 10;i<200;i++){
engine.setThreadNum(i);
engine.enginer();
}
}
}
package com.zjty.adaptationmaster.adaptor.enginer;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
public class Engine1 {
private static final String path = "C:\\Users\\wyl\\Desktop\\testReplace";
private long totalSize;
private ReadedFileRepository repository = new ReadedFileRepository();
public void readFile(){
//List<>
try {
Files.walkFileTree(Paths.get(path),new SimpleFileVisitor<Path>(){
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
String s = new String(Files.readAllBytes(file));
ReadedFile readedFile = new ReadedFile(s,attrs,file,new ArrayList<>());
if(repository.put(readedFile)){
// ApacheTask apacheTask = new ApacheTask();
// apacheTask.setFile(repository.getReadedFiles());
}
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
e.printStackTrace();
}
}
//先把文件读取到内存中
private class ReadedFileRepository{
private static final long LIMIT = 200*1024*1024;//字节*k*m
private List<ReadedFile> readedFiles;
private long count;
public List<ReadedFile> getReadedFiles() {
return readedFiles;
}
public boolean put(ReadedFile readedFile){
if(readedFile==null)readedFiles = new ArrayList<>();
readedFiles.add(readedFile);
count+=readedFile.getAttributes().size();
if(count>LIMIT){
return true;
}
return false;
}
}
}
package com.zjty.adaptationmaster.adaptor.enginer;
public enum MatchType {
ALL,PATTERN
public class Inspector {
}
package com.zjty.adaptationmaster.adaptor.enginer;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
public class KMP {
/**
* 求出一个字符数组的next数组
* @param t 字符数组
* @return next数组
*/
public static int[] getNextArray(char[] t) {
int[] next = new int[t.length];
next[0] = -1;
next[1] = 0;
int k;
for (int j = 2; j < t.length; j++) {
k=next[j-1];
while (k!=-1) {
if (t[j - 1] == t[k]) {
next[j] = k + 1;
break;
}
else {
k = next[k];
}
next[j] = 0; //当k==-1而跳出循环时,next[j] = 0,否则next[j]会在break之前被赋值
}
}
return next;
}
/**
* 对主串s和模式串t进行KMP模式匹配
* @param s 主串
* @param t 模式串
* @return 若匹配成功,返回t在s中的位置(第一个相同字符对应的位置),若匹配失败,返回-1
*/
public static int kmpMatch(String s, String t){
char[] s_arr = s.toCharArray();
char[] t_arr = t.toCharArray();
int[] next = getNextArray(t_arr);
int i = 0, j = 0;
while (i<s_arr.length && j<t_arr.length){
if(j == -1 || s_arr[i]==t_arr[j]){
i++;
j++;
}
else
j = next[j];
}
if(j == t_arr.length)
return i-j;
else
return -1;
}
public static void main(String[] args) {
String path = "C:\\Users\\wyl\\Desktop\\testReplace\\testReplace1\\java\\soapui.properties";
try {
System.out.println(kmpMatch(new String(Files.readAllBytes(Paths.get(path))), "2019-11-27 13:55:36,631 INFO [WorkspaceImpl] Saved workspace to [C:\\Users\\wyl\\intellij-soapui-workspace.xml]\n" +
"2019-11-27 13:55:36,648 INFO [DefaultSoapUICore] Settings saved to [C:\\Users\\wyl\\soapui-setting中华s.xml]"));
} catch (IOException e) {
e.printStackTrace();
}
}
}
\ No newline at end of file
package com.zjty.adaptationmaster.adaptor.enginer;
import com.zjty.adaptationmaster.adaptor.entity.Rule;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.messaging.simp.SimpMessagingTemplate;
import java.io.*;
import java.util.List;
public class ReadedFileTask implements Runnable {
@Autowired
private SimpMessagingTemplate template;
private List<ReadedFile> readedFiles;
private Writer responseWriter;
public ReadedFileTask(List<ReadedFile> readedFiles,Writer responseWriter){
//private Writer responseWriter;
public ReadedFileTask(List<ReadedFile> readedFiles/*,Writer responseWriter*/) {
this.readedFiles = readedFiles;
this.responseWriter = responseWriter;
//this.responseWriter = responseWriter;
}
private Adaptor.WriterBySort bySort;
......@@ -51,11 +55,10 @@ public class ReadedFileTask implements Runnable {
}
}
//container.add(readedFile.getPath().getFileName()+"文本替换:"+entity.getTextMatching()+"|"+entity.getReplacing());
try {
responseWriter.write("===="+readedFile.getPath().getFileName()+"////"+entity.getTextMatching()+"||||"+entity.getReplacing());
} catch (IOException e) {
e.printStackTrace();
}
template.convertAndSend("===="+readedFile.getPath().getFileName()+"////"+entity.getTextMatching()+"||||"+entity.getReplacing(),"1L");
//responseWriter.write("===="+readedFile.getPath().getFileName()+"////"+entity.getTextMatching()+"||||"+entity.getReplacing());
}
try {
if(bySort!=null){
......
package com.zjty.adaptationmaster.adaptor.enginer;
import java.util.List;
public class StringFactory implements Runnable {
private List<String> product;
public StringFactory setProduct(List<String> product) {
this.product = product;
return this;
}
@Override
public void run() {
//System.out.println("我开始处理了+++"+Thread.currentThread().getName()+System.currentTimeMillis());
for(String s:product){
if(s.contains("一二三四五")){
System.out.println("包含");
}
int i = 0;
while (true){
i++;
}
// try {
// Object o = new Object();
// synchronized (o) {
// o.wait(20);
// }
// } catch (InterruptedException e) {
// e.printStackTrace();
// }
}
product.clear();
//System.out.println("我处理完了---"+Thread.currentThread().getName()+"\t"+System.currentTimeMillis());
}
}
package com.zjty.adaptationmaster.adaptor.enginer;
import lombok.AllArgsConstructor;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.util.List;
@AllArgsConstructor
public class StringTask implements Runnable {
private Path path;
private PathMatcher pathMatcher;
@Override
public void run() {
//过滤文件名后缀
if(pathMatcher.matches(path)){
List<String> list = null;
try {
list = Files.readAllLines(path);
for (String s:list){
if(s.contains("一二三四五")){
System.out.println("包含");
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
package com.zjty.adaptationmaster.adaptor.enginer;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.commons.io.FileUtils;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class TimeAreaEngine {
public static void main(String[] args) {
TimeAreaEngine engine = new TimeAreaEngine();
engine.setPath("C:\\Users\\wyl\\Desktop\\testReplace");
engine.setProjectName("testReplace");
List<AdaptorEntity> adaptorEntities = new ArrayList<>();
adaptorEntities.add(new AdaptorEntity(null,"log4j",null,"log5j"));
engine.setAdaptorEntities(adaptorEntities);
engine.enginer();
}
private String path;
private List<AdaptorEntity> adaptorEntities;
private static final int THREADCOUNT = 4;
private String projectName;
public void enginer(){
try {
Iterator<File> fileIterator = FileUtils.iterateFiles(new File(path), new String[]{"java", "txt", "properties"}, true);
while (fileIterator.hasNext()){
List<String> stringList = FileUtils.readLines(fileIterator.next(), "utf-8");
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
package com.zjty.adaptationmaster.adaptor.examination;
import com.zjty.adaptationmaster.adaptor.enginer.Counter;
import com.zjty.adaptationmaster.adaptor.enginer.util.StringCompareUtil;
import java.io.IOException;
......@@ -178,4 +177,18 @@ public class Inspectors {
}
return null;
}
private static class Counter {
private int i = 0;
public void plus(){
i++;
}
public int getI() {
return i;
}
public void reset(){
i = 0;
}
}
}
......@@ -11,8 +11,13 @@ public class Report {
public String type;//语言类型
public String structure;//项目架构
public String sqlType;//数据库类型
public DependenceManagement dependenceManagement;
@Override
public String toString(){
return ""+type+structure+sqlType;
}
private static enum DependenceManagement{
MAVEN,GRADLE,ANT
}
}
package com.zjty.adaptationmaster.base.configuration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.messaging.simp.config.MessageBrokerRegistry;
import org.springframework.web.socket.config.annotation.AbstractWebSocketMessageBrokerConfigurer;
import org.springframework.web.socket.config.annotation.StompEndpointRegistry;
import org.springframework.web.socket.server.standard.ServerEndpointExporter;
/**
* 开启websocket支持
*/
@Configuration
public class WebSocketConfig extends AbstractWebSocketMessageBrokerConfigurer {
@Override
public void configureMessageBroker(MessageBrokerRegistry registry) {
registry.enableSimpleBroker("/topic","/user");
}
@Override
public void registerStompEndpoints(StompEndpointRegistry registry) {
registry.addEndpoint("/webServer").withSockJS();
//registry.addEndpoint("/queueServer").withSockJS();
}
}
......@@ -19,8 +19,13 @@ public class Const {
*/
private static final Logger logger = LoggerFactory.getLogger(Const.class);
/**
* 操作台
* 适配对象项目基础地址
*/
public static final String CONSOLE = "/root/project/adaptationMaster/";
/** 上传文件地址 */
public static final String UPLOAD_LOCATION = "/root/projects/bservice/uploads/";
public static final String UPLOAD_LOCATION = "/root/projects/adaptationMaster/uploads/";
// System.getProperty("user.dir") + "\\src\\main\\resources\\uploads\\";
/** 返回值样例常量,用于swagger
* 页面的example显示 */
......
......@@ -8,7 +8,8 @@ server.port=8078
# 日志文件相关配置
# logging.level.org.hibernate.sql=debug
logging.file=./log/master.log
logging.file.path=./log/master.log
#logging.file=./log/master.log
#spring.datasource.url=jdbc:mysql://localhost:3306/adapt?useSSL=false&serverTimezone=Asia/Shanghai&autoReconnect=true&characterEncoding=utf-8
#spring.datasource.username=root
#spring.datasource.password=root
......@@ -32,6 +33,4 @@ spring.servlet.multipart.max-request-size=100MB
spring.thymeleaf.cache=false
spring.mvc.static-path-pattern=/**
spring.resources.static-locations=classpath:/uploads/
base.path=C:\\home\\master\\
\ No newline at end of file
spring.resources.static-locations=classpath:/uploads/
\ No newline at end of file
package com.zjty.adaptationmaster;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
public class Compile {
private static final String USER_DIR = System.getProperty("user.dir");
private static final String FILE_SEPARATOR = System.getProperty("file.separator");
private static final String OS_NAME = System.getProperty("os.name");
public static void main(String[] args) throws IOException {
String sourcePath = USER_DIR;
if (args.length > 0) {
sourcePath = getPath(args[0]);
}
Set<String> dirs = getSourceDirs(new File(sourcePath));
String input = dirs.toString()
.replace(",", "")
.replace("[", "")
.replace("]", "");
System.out.println(input);
String output;
if (args.length >= 2) {
output = createDestDirectory(args[1]);
} else {
output = createDestDirectory("out");
}
// 打印将要执行的javac 命令
System.out.println("javac -d " + output + " " + input);
try {
// 执行javac编译操作
Runtime.getRuntime().exec("javac -d " + output + " " + input);
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException("Failure");
}
System.out.println("SUCCESS, The dest path is [" + output + "]");
}
// 创建class文件的存放目录 默认路径为当前工作路径下的out文件夹
private static String createDestDirectory(String args) {
String destPath = getPath(args);
File file = new File(destPath);
if (!file.exists()) {
file.mkdirs();
}
return file.getAbsolutePath();
}
// 获取要编译的java源文件的目录, 并在目录的末尾添加上了 *.java
private static Set<String> getSourceDirs(File file) {
Set<String> resultDirs = new HashSet<>();
if (file.isFile() && file.getName().endsWith(".java")) {
resultDirs.add(file.getAbsolutePath().replace(file.getName(), "*.java"));
return resultDirs;
} else if (file.isDirectory()) {
File[] files = file.listFiles();
for (File f : files) {
resultDirs.addAll(getSourceDirs(f));
}
}
return resultDirs;
}
// 根据传入的参数获取当前项目工作路径
private static String getPath(String sourcePath) {
String path;
if (sourcePath == null) {
return USER_DIR;
}
path = sourcePath.trim();
if (!path.contains(":") && OS_NAME.toLowerCase().contains("windows")) {
if (path.startsWith("\\") || path.startsWith("/")) {
path = USER_DIR + path;
} else {
path = USER_DIR + FILE_SEPARATOR + path;
}
if (!(path.endsWith("\\") || path.endsWith("/"))) {
path += FILE_SEPARATOR;
}
}
if (OS_NAME.toLowerCase().contains("linux") && !path.startsWith("/")) {
path = USER_DIR + FILE_SEPARATOR + path;
}
return path.replace("\\", FILE_SEPARATOR).replace("/", FILE_SEPARATOR);
}
}
package com.zjty.adaptationmaster;
import org.junit.Test;
public class KMPTest {
@Test
public void test(){
String pattern = "adfghjkadfg";
String trext = "adfghadfghadfghjk";
}
private int[] anaylize(String pattern){
char[] chars = pattern.toCharArray();
int length = chars.length;
int[] result = new int[length];
for(int i=1;i<length;i++){
}
return result;
}
}
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论