Skip to content

Commit

Permalink
Merge pull request #42 from owent/dev
Browse files Browse the repository at this point in the history
Prepare v2.17.0
  • Loading branch information
owent authored Jul 12, 2024
2 parents 9c3c768 + 5e0c54a commit d4783d6
Show file tree
Hide file tree
Showing 6 changed files with 191 additions and 42 deletions.
5 changes: 5 additions & 0 deletions HISTORY.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@

## Unrelease

## 2.17.0

1. 增加 `--disable-data-validator` 允许跳过数据验证。
2. 增加正则表达式验证器 `Regex("正则表达式")`

## 2.16.1

1. 修复字段别名的一处错误
Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ echo "
| --javascript-global | 导出javascript全局空间 | 导出数据到全局时,可以指定写入的名字空间 |
| --ignore-unknown-dependency | 忽略未知的依赖项 | 忽略未知的输入协议的依赖项(>=2.9.0版本) |
| --validator-rules | 指定自定义验证器配置文件路径 | 指定自定义验证器配置文件路径 |
| --disable-data-validator | 允许忽略数据验证错误 | (>=2.17.0版本) |
| --data-source-lru-cache-rows | 数据源的LRU Cache行数 | 仅缓存流式索引 |
| --tolerate-max-empty-rows | 连续空行检测的行数 | 设置连续空行检测的行数(>=2.14.1版本) ,大量的连续空行通常是误操作 |

Expand Down Expand Up @@ -287,6 +288,7 @@ Excel里的Key使用@后缀的字段名,@后面的部分都属于验证器。
+ 函数: `InText("文件名"[, 第几个字段[, \"字段分隔正则表达式\"]])` : 从文本文件(UTF-8编码),可以指定读第几个字段和用于字段分隔的正则表达式
+ 函数: `InTableColumn("文件名", "Sheet名", 从第几行开始, 从第几列开始)` : 从Excel数据列读取可用值,指定数据行和数据列
+ 函数: `InTableColumn("文件名", "Sheet名", 从第几行开始, KeyRow, KeyValue)` : 从Excel数据列读取可用值,指定数据行并通过某一行的的值获取数据列
+ 函数: `Regex("正则表达式")` : 验证匹配正则表达式(>=2.17.0版本)
+ 自定义验证器名(通过 `--validator-rules` 加载)
+ 协议类型(对应protobuf的message里的每个field,excel里可以填field number或者field name)
+ 枚举类型(对应protobuf的enum里的每个number,excel里可以填enum number或者enum name)
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

<groupId>org.xresloader</groupId>
<artifactId>xresloader</artifactId>
<version>2.16.1</version>
<version>2.17.0</version>
<packaging>jar</packaging>
<name>xresloader</name>

Expand Down
9 changes: 9 additions & 0 deletions src/org/xresloader/core/ProgramOptions.java
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ public class RenameRule {
public int prettyIndent = 0;
public boolean enableStdin = false;
public String[] customValidatorRules = null;
public boolean enableDataValidator = true;

public String protoDumpFile = "";
public ProtoDumpType protoDumpType = ProtoDumpType.NONE;
Expand Down Expand Up @@ -231,6 +232,8 @@ private static synchronized Options get_options_group() {
options.addOption(null, "lua-module", true, "module(MODULE_NAME, package.seeall) if in lua mode");
options.addOption(Option.builder().longOpt("validator-rules")
.desc("set file to load custom validator").hasArg().argName("FILE PATH").build());
options.addOption(null, "disable-data-validator", false,
"disable data validator, so it will not show warnings when data checking failed.");
options.addOption(Option.builder().longOpt("data-source-lru-cache-rows")
.desc("set row number for LRU cache").hasArg().argName("NUMBER").build());
options.addOption(Option.builder().longOpt("tolerate-max-empty-rows")
Expand Down Expand Up @@ -492,6 +495,12 @@ public int init(String[] args) {
// custom validator rule file
customValidatorRules = cmd.getOptionValues("validator-rules");

if (cmd.hasOption("disable-data-validator")) {
enableDataValidator = false;
} else {
enableDataValidator = true;
}

return 0;
}

Expand Down
126 changes: 85 additions & 41 deletions src/org/xresloader/core/data/vfy/DataVerifyImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.xresloader.core.ProgramOptions;
import org.xresloader.core.data.err.ConvException;

/**
Expand Down Expand Up @@ -149,10 +150,10 @@ static public double getAndVerify(List<DataVerifyImpl> verifyEngine, String path
return n;
}

try {
DataVerifyResult verify_cache = new DataVerifyResult();
DataVerifyResult verify_cache = new DataVerifyResult();

for (DataVerifyImpl vfy : verifyEngine) {
for (DataVerifyImpl vfy : verifyEngine) {
try {
if (vfy.get(n, verify_cache)) {
if (verify_cache.value == null) {
return 0;
Expand All @@ -165,16 +166,22 @@ static public double getAndVerify(List<DataVerifyImpl> verifyEngine, String path
}
return doubleValueOf(verify_cache.value.toString());
}
} catch (Exception e) {
String value;
if (n == (long) n) {
value = String.format("%d", (long) n);
} else {
value = String.format("%g", n);
}
String message = String.format("Check %s for %s with validator %s failed, %s", value, path,
vfy.getDescription(), e.getMessage());
if (ProgramOptions.getInstance().enableDataValidator) {
throw new ConvException(
message);
} else {
ProgramOptions.getLoger().warn(message);
}
}
} catch (Exception e) {
String value;
if (n == (long) n) {
value = String.format("%d", (long) n);
} else {
value = String.format("%g", n);
}
throw new ConvException(String.format("Check %s for %s with %s %s failed, %s", value, path,
getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine), e.getMessage()));
}

String value;
Expand All @@ -183,9 +190,16 @@ static public double getAndVerify(List<DataVerifyImpl> verifyEngine, String path
} else {
value = String.format("%g", n);
}
throw new ConvException(
String.format("Check %s for %s with %s %s failed, check data failed.", value, path,
getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine)));

String message = String.format("Check %s for %s with %s %s failed, check data failed.", value, path,
getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine));
if (ProgramOptions.getInstance().enableDataValidator) {
throw new ConvException(
message);
} else {
ProgramOptions.getLoger().warn(message);
return 0.0;
}
}

static public double getAndVerifyToDouble(List<DataVerifyImpl> verifyEngine, String path, String val)
Expand Down Expand Up @@ -222,42 +236,60 @@ static public double getAndVerifyToDouble(List<DataVerifyImpl> verifyEngine, Str
DataVerifyResult verify_cache = new DataVerifyResult();

for (DataVerifyImpl vfy : verifyEngine) {
if (vfy.get(val, verify_cache)) {
if (verify_cache.value == null) {
return 0;
}
if (verify_cache.value instanceof Double) {
return (double) verify_cache.value;
try {
if (vfy.get(val, verify_cache)) {
if (verify_cache.value == null) {
return 0;
}
if (verify_cache.value instanceof Double) {
return (double) verify_cache.value;
}
if (verify_cache.value instanceof Long) {
return ((Long) verify_cache.value).doubleValue();
}
return doubleValueOf(verify_cache.value.toString());
}
if (verify_cache.value instanceof Long) {
return ((Long) verify_cache.value).doubleValue();
} catch (Exception e) {
String message = String.format("Check %s for %s with validator %s failed, %s", val, path,
vfy.getDescription(), e.getMessage());
if (ProgramOptions.getInstance().enableDataValidator) {
throw new ConvException(
message);
} else {
ProgramOptions.getLoger().warn(message);
}
return doubleValueOf(verify_cache.value.toString());
}
}
} catch (Exception e) {
if (verifyEngine == null || verifyEngine.isEmpty()) {
throw new ConvException(String.format("Convert %s for %s failed, %s", val, path, e.getMessage()));
String message = String.format("Convert %s for %s failed, %s", val, path, e.getMessage());
if (ProgramOptions.getInstance().enableDataValidator) {
throw new ConvException(
message);
} else {
throw new ConvException(String.format("Convert %s for %s with %s %s failed, %s", val, path,
getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine), e.getMessage()));
ProgramOptions.getLoger().warn(message);
}
}

throw new ConvException(String.format("Convert %s for %s with %s %s failed, check data failed.", val,
path, getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine)));
String message = String.format("Convert %s for %s with %s %s failed, check data failed.", val,
path, getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine));
if (ProgramOptions.getInstance().enableDataValidator) {
throw new ConvException();
} else {
ProgramOptions.getLoger().warn(message);
return 0.0;
}
}

static public String getAndVerifyToString(List<DataVerifyImpl> verifyEngine, String path, String val)
throws ConvException {
try {
if (verifyEngine == null || verifyEngine.isEmpty()) {
return val;
}
if (verifyEngine == null || verifyEngine.isEmpty()) {
return val;
}

DataVerifyResult verify_cache = new DataVerifyResult();
DataVerifyResult verify_cache = new DataVerifyResult();

for (DataVerifyImpl vfy : verifyEngine) {
for (DataVerifyImpl vfy : verifyEngine) {
try {
if (vfy.get(val, verify_cache)) {
if (verify_cache.value == null) {
return "";
Expand All @@ -276,14 +308,26 @@ static public String getAndVerifyToString(List<DataVerifyImpl> verifyEngine, Str
}
return verify_cache.value.toString();
}
} catch (Exception e) {
String message = String.format("Check %s for %s with validator %s failed, %s", val, path,
vfy.getDescription(), e.getMessage());
if (ProgramOptions.getInstance().enableDataValidator) {
throw new ConvException(
message);
} else {
ProgramOptions.getLoger().warn(message);
}
}
} catch (Exception e) {
throw new ConvException(String.format("Convert %s for %s with %s %s failed, %s", val, path,
getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine), e.getMessage()));
}

throw new ConvException(String.format("Convert %s for %s with %s %s failed, check data failed.", val,
path, getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine)));
String message = String.format("Convert %s for %s with %s %s failed, check data failed.", val,
path, getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine));
if (ProgramOptions.getInstance().enableDataValidator) {
throw new ConvException(message);
} else {
ProgramOptions.getLoger().warn(message);
return "";
}
}

static public long getAndVerifyToLong(List<DataVerifyImpl> verifyEngine, String path, String val)
Expand Down
89 changes: 89 additions & 0 deletions src/org/xresloader/core/data/vfy/DataVerifyRegex.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package org.xresloader.core.data.vfy;

import java.util.ArrayList;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import org.xresloader.core.ProgramOptions;

public class DataVerifyRegex extends DataVerifyImpl {
private boolean valid = false;
private ArrayList<Pattern> rules = new ArrayList<>();

static Pattern SPACE_SPLITOR = Pattern.compile("\\s+");

public DataVerifyRegex(ValidatorTokens tokens) {
super(tokens);

this.valid = false;
if (tokens.parameters.size() < 2) {
ProgramOptions.getLoger().error("Invalid in regex validator %s", tokens.name);
return;
}

this.valid = true;
for (int i = 1; i < tokens.parameters.size(); ++i) {
try {
this.rules.add(Pattern.compile(tokens.parameters.get(i)));
} catch (PatternSyntaxException e) {
ProgramOptions.getLoger().error("Can not parse regex %s for validator %s : %s",
tokens.parameters.get(i),
tokens.name,
e.getMessage());
this.valid = false;
}
}
}

public boolean isValid() {
return this.valid;
}

@Override
public boolean get(double number, DataVerifyResult res) {
// 0 值永久有效,因为空数据项会被填充默认值
if (0 == number) {
res.success = true;
res.value = number;
return true;
}

String value;
if (number == (long) number) {
value = String.format("%d", (long) number);
} else {
value = String.format("%g", number);
}

for (Pattern rule : this.rules) {
if (rule.matcher(value).matches()) {
res.success = true;
res.value = number;
return true;
}
}
res.success = false;
return false;
}

@Override
public boolean get(String input, DataVerifyResult res) {
// 空值永久有效,因为空数据项会被填充默认值
if (input.isEmpty()) {
res.success = true;
res.value = "";
return true;
}

for (Pattern rule : this.rules) {
if (rule.matcher(input.trim()).matches()) {
res.success = true;
res.value = input;
return true;
}
}

res.success = false;
return false;
}
}

0 comments on commit d4783d6

Please sign in to comment.