Skip to content

Commit c6e10a8

Browse files
robinhood-jimrobinhood-jim
committed
vue back server modify
1 parent dbef11b commit c6e10a8

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+972
-173
lines changed

‎common/src/main/java/com/robin/comm/sql/CommRecordGenerator.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ public static void doAsyncCalculator(SqlSegment segment,Map<String,Object> input
9292
calculator.setOutputRecord(newRecord);
9393
calculator.setSegment(segment);
9494
ListenableFuture<Boolean> future = pool.submit(new CalculatorCallable(calculator));
95-
Futures.addCallback(future, new FutureCallback<Boolean>() {
95+
Futures.addCallback(future, new FutureCallback<>() {
9696
@Override
9797
public void onSuccess(Boolean aBoolean) {
9898
caPool.returnObject(calculator);

‎common/src/main/java/com/robin/core/fileaccess/fs/FileSystemAccessorFactory.java

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,13 @@ public static AbstractFileSystemAccessor getResourceAccessorByType(@NonNull Stri
3333
try {
3434
Class<? extends IFileSystemAccessor> clazz = accessorMap.get(resType);
3535
if (!ObjectUtils.isEmpty(clazz)) {
36-
accessor = (AbstractFileSystemAccessor) clazz.getConstructor().newInstance();
37-
if(!ObjectUtils.isEmpty(colmeta)){
38-
accessor.init(colmeta);
36+
if(LocalFileSystemAccessor.class.isAssignableFrom(clazz)){
37+
return LocalFileSystemAccessor.getInstance();
38+
}else {
39+
accessor = (AbstractFileSystemAccessor) clazz.getConstructor().newInstance();
40+
if (!ObjectUtils.isEmpty(colmeta)) {
41+
accessor.init(colmeta);
42+
}
3943
}
4044
}
4145
} catch (Exception ex) {

‎common/src/main/java/com/robin/core/fileaccess/fs/LocalFileSystemAccessor.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
public class LocalFileSystemAccessor extends AbstractFileSystemAccessor {
1111
private static LocalFileSystemAccessor accessor=new LocalFileSystemAccessor();
12-
public LocalFileSystemAccessor(){
12+
private LocalFileSystemAccessor(){
1313
this.identifier= Const.FILESYSTEM.LOCAL.getValue();
1414
}
1515
public static LocalFileSystemAccessor getInstance(){

‎core/src/main/java/com/robin/core/base/service/AbstractMybatisService.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ public abstract class AbstractMybatisService<M extends BaseMapper<T>, T extends
6262
protected Map<String, Method> getMethods;
6363
protected Map<String, Method> setMethods;
6464

65-
protected String defaultOrderField = "create_time";
65+
protected String defaultOrderField = "create_tm";
6666
protected Boolean defaultOrder = false;
6767
protected String statusColumn = "status";
6868
protected Method setStatusMethod = null;

‎core/src/main/java/com/robin/core/base/service/util/QueryWrapperUtils.java

Lines changed: 27 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import com.robin.core.convert.util.ConvertUtil;
1212
import com.robin.core.query.util.Condition;
1313
import lombok.extern.slf4j.Slf4j;
14+
import org.springframework.beans.BeanUtils;
1415
import org.springframework.util.Assert;
1516
import org.springframework.util.CollectionUtils;
1617
import org.springframework.util.ObjectUtils;
@@ -34,6 +35,9 @@ public static <T> void getWrapperByReq(Map<String, String> fieldMap, Map<String,
3435
for (Map.Entry<String, Object> entry : valueMap.entrySet()) {
3536
String key = entry.getKey();
3637
String columnName = fieldMap.get(entry.getKey());
38+
if(ObjectUtils.isEmpty(columnName)){
39+
continue;
40+
}
3741
//or 组合，支持全字段查询
3842
if ("or".equalsIgnoreCase(key)) {
3943
Object valueObj = entry.getValue();
@@ -44,7 +48,7 @@ public static <T> void getWrapperByReq(Map<String, String> fieldMap, Map<String,
4448
wrapper.and(f -> {
4549
for (String column : columns) {
4650
if(fieldMap.containsKey(column)) {
47-
queryConditionWrap(fieldTypeMap, f.or(), fieldMap.get(column), orMap.get("value").toString());
51+
queryConditionWrap(fieldTypeMap, f.or(), fieldMap.get(column), orMap.get("value").toString(),false);
4852
}else{
4953
throw new MissingConfigException("column "+column+" not exists in table");
5054
}
@@ -56,9 +60,9 @@ public static <T> void getWrapperByReq(Map<String, String> fieldMap, Map<String,
5660
}
5761
} else if (entry.getValue().getClass().isAssignableFrom(String.class)) {
5862
if (!StringUtils.isEmpty(columnName)) {
59-
queryConditionWrap(fieldTypeMap,wrapper, columnName, entry.getValue().toString());
63+
queryConditionWrap(fieldTypeMap,wrapper, columnName, entry.getValue().toString(),true);
6064
} else {
61-
queryConditionWrap(fieldTypeMap,wrapper, key, entry.getValue().toString());
65+
queryConditionWrap(fieldTypeMap,wrapper, key, entry.getValue().toString(),false);
6266
}
6367
} else {
6468
if (!StringUtils.isEmpty(columnName)) {
@@ -74,7 +78,7 @@ public static <T> void getWrapperByReq(Map<String, String> fieldMap, Map<String,
7478
}
7579
}
7680

77-
protected static <T> void queryConditionWrap(Map<String, Class<?>> fieldTypeMap,QueryWrapper<T> wrapper, String columnName, String value) {
81+
protected static <T> void queryConditionWrap(Map<String, Class<?>> fieldTypeMap,QueryWrapper<T> wrapper, String columnName, String value,boolean defaultUseLike) {
7882
if (value.contains("%")) {
7983
if (value.startsWith("%")) {
8084
if (value.endsWith("%")) {
@@ -110,7 +114,11 @@ protected static <T> void queryConditionWrap(Map<String, Class<?>> fieldTypeMap,
110114
} else if ("NULL".equalsIgnoreCase(value)) {
111115
wrapper.isNull(columnName);
112116
} else {
113-
wrapper.eq(columnName, retValue(fieldTypeMap,columnName, value));
117+
if(defaultUseLike){
118+
wrapper.like(columnName,retValue(fieldTypeMap, columnName, value));
119+
}else {
120+
wrapper.eq(columnName, retValue(fieldTypeMap, columnName, value));
121+
}
114122
}
115123
}
116124

@@ -176,19 +184,9 @@ protected static Object returnTimeColumn(String value, Class<?> clazz) {
176184
}
177185
private static Map<String, Object> returnValueMap(PageDTO targetObj) throws Exception {
178186
Map<String, Object> getMap = new HashMap<>();
179-
Map<String, Field> fieldMap = ReflectUtils.getAllField(targetObj.getClass());
180-
for (Map.Entry<String, Field> entry : fieldMap.entrySet()) {
181-
String name = entry.getValue().getName();
182-
if (name.startsWith("get") && !"getClass".equals(name) && !"getOffset".equals(name) && !"getLimit".equals(name)) {
183-
Object val = entry.getValue().get(targetObj);
184-
if (null != val) {
185-
getMap.put(org.springframework.util.StringUtils.uncapitalize(name.substring(3)), val);
186-
}
187-
}
188-
}
189-
Map<String, Object> paramMap = targetObj.getParam();
190-
if (!CollectionUtils.isEmpty(paramMap)) {
191-
getMap.putAll(paramMap);
187+
ConvertUtil.objectToMapObj(getMap,targetObj,"param","order","page","limit","size","orderField","orderBy");
188+
if (!CollectionUtils.isEmpty(targetObj.getParam())) {
189+
getMap.putAll(targetObj.getParam());
192190
}
193191
return getMap;
194192
}
@@ -202,17 +200,25 @@ public static <T> QueryWrapper<T> wrapWithEntity(Map<String,String> fieldMap,Ma
202200
wrapWithValue(fieldMap, getMethod, queryWrapper, iter);
203201
if (tmpMap.get(Const.ORDER) != null && !org.apache.commons.lang3.StringUtils.isEmpty(tmpMap.get(Const.ORDER).toString())
204202
&& tmpMap.get(Const.ORDER_FIELD) != null && !org.apache.commons.lang3.StringUtils.isEmpty(tmpMap.get(Const.ORDER_FIELD).toString())) {
205-
queryWrapper.orderBy(true, tmpMap.get(Const.ORDER).equals(Const.ASC), tmpMap.get(Const.ORDER_FIELD).toString());
203+
if(tmpMap.get(Const.ORDER).equals(Const.ASC)){
204+
queryWrapper.orderByAsc(defaultOrderField);
205+
}else{
206+
queryWrapper.orderByDesc(defaultOrderField);
207+
}
206208
} else {
207-
queryWrapper.orderBy(true, defaultOrder, defaultOrderField);
209+
if(defaultOrder){
210+
queryWrapper.orderByAsc(defaultOrderField);
211+
}else{
212+
queryWrapper.orderByDesc(defaultOrderField);
213+
}
208214
}
209215
}
210216
//PageDTO paramMap
211217
else if (queryObject.getClass().getSuperclass().isAssignableFrom(PageDTO.class)) {
212218
PageDTO pageDTO = (PageDTO) queryObject;
213219
QueryWrapperUtils.getWrapperByReq(fieldMap,fieldTypeMap,getStatusMethod,statusColumn,pageDTO, queryWrapper, true);
214220
if (!org.apache.commons.lang3.StringUtils.isEmpty(pageDTO.getOrderField())) {
215-
queryWrapper.orderBy(true, pageDTO.getOrder(), pageDTO.getOrderField());
221+
queryWrapper.orderBy(true, pageDTO.getOrder().booleanValue(), pageDTO.getOrderField());
216222
} else {
217223
queryWrapper.orderBy(true, defaultOrder, defaultOrderField);
218224
}

‎core/src/main/java/com/robin/core/base/util/ResourceConst.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ public class ResourceConst {
1818
public static final int DEFAULTSTORAGEUPLOADTHREAD=4;
1919
public static final String STORAGEFILTERSQL="storage.FilterSql";
2020
public static final String PARQUETFILEFORMAT="parquet.file.format";
21+
public static final String USEADMINTG="useAdmin";
2122

2223
public enum IngestType {
2324
TYPE_HDFS(1L,"HDFS"),

‎core/src/main/java/com/robin/core/convert/util/ConvertUtil.java

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
*/
1616
package com.robin.core.convert.util;
1717

18-
import com.google.common.base.Function;
1918
import com.google.common.collect.Lists;
19+
import com.google.common.collect.Sets;
2020
import com.robin.core.base.datameta.DataBaseColumnMeta;
2121
import com.robin.core.base.exception.GenericException;
2222
import com.robin.core.base.model.BaseObject;
@@ -43,7 +43,6 @@
4343
import java.time.format.DateTimeFormatter;
4444
import java.time.format.DateTimeParseException;
4545
import java.util.*;
46-
import java.util.stream.Collectors;
4746

4847
@Slf4j
4948
public class ConvertUtil {
@@ -106,7 +105,25 @@ public static void objectToMapObj(Map<String, Object> target, Object src) throws
106105
for (Map.Entry<String, Method> entry : getMetholds.entrySet()) {
107106
if (entry.getValue().getParameterTypes().length == 0) {
108107
Object value = entry.getValue().invoke(src);
109-
target.put(entry.getKey(), value);
108+
if(!ObjectUtils.isEmpty(value)) {
109+
target.put(entry.getKey(), value);
110+
}
111+
}
112+
}
113+
}
114+
public static void objectToMapObj(Map<String, Object> target, Object src,String... ignoreColumns) throws IllegalAccessException, IllegalArgumentException,
115+
InvocationTargetException {
116+
Set<String> ignoreSets=ignoreColumns.length>0? Sets.newHashSet(ignoreColumns):null;
117+
if (ObjectUtils.isEmpty(src)) {
118+
return;
119+
}
120+
Map<String, Method> getMetholds = ReflectUtils.returnGetMethods(src.getClass());
121+
for (Map.Entry<String, Method> entry : getMetholds.entrySet()) {
122+
if (entry.getValue().getParameterTypes().length == 0 && (CollectionUtils.isEmpty(ignoreSets) || !ignoreSets.contains(entry.getKey()))) {
123+
Object value = entry.getValue().invoke(src);
124+
if(!ObjectUtils.isEmpty(value)) {
125+
target.put(entry.getKey(), value);
126+
}
110127
}
111128
}
112129
}

‎core/src/main/java/com/robin/core/fileaccess/meta/DataCollectionMeta.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
*/
1616
package com.robin.core.fileaccess.meta;
1717

18+
import cn.hutool.setting.yaml.YamlUtil;
1819
import com.robin.core.base.datameta.BaseDataBaseMeta;
1920
import com.robin.core.base.datameta.DataBaseColumnMeta;
2021
import com.robin.core.base.datameta.DataBaseParam;
@@ -27,6 +28,8 @@
2728
import org.springframework.util.CollectionUtils;
2829
import org.springframework.util.ObjectUtils;
2930

31+
import java.io.FileInputStream;
32+
import java.io.IOException;
3033
import java.io.Serializable;
3134
import java.util.*;
3235
import java.util.stream.Collectors;
@@ -183,6 +186,13 @@ public DataCollectionMeta build(){
183186
return meta;
184187
}
185188
}
189+
public static DataCollectionMeta fromYamlConfig(String yamlConfigPath) throws IOException {
190+
if(yamlConfigPath.startsWith("classpath:")){
191+
return YamlUtil.load(DataCollectionMeta.class.getClassLoader().getResourceAsStream(yamlConfigPath.substring(10,yamlConfigPath.length())),DataCollectionMeta.class);
192+
}else{
193+
return YamlUtil.load(new FileInputStream(yamlConfigPath),DataCollectionMeta.class);
194+
}
195+
}
186196
public String constructUrl() {
187197
VfsParam param = new VfsParam();
188198
try {

‎hadooptool/pom.xml

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,7 @@
445445
<groupId>org.apache.arrow</groupId>
446446
<artifactId>arrow-memory-core</artifactId>
447447
<version>${arrow.version}</version>
448+
<optional>true</optional>
448449
<exclusions>
449450
<exclusion>
450451
<groupId>org.slf4j</groupId>
@@ -456,16 +457,19 @@
456457
<groupId>org.apache.arrow</groupId>
457458
<artifactId>arrow-memory-netty</artifactId>
458459
<version>${arrow.version}</version>
460+
<optional>true</optional>
459461
</dependency>
460462
<dependency>
461463
<groupId>org.apache.arrow</groupId>
462464
<artifactId>arrow-vector</artifactId>
463465
<version>${arrow.version}</version>
466+
<optional>true</optional>
464467
</dependency>
465468
<dependency>
466469
<groupId>io.netty</groupId>
467470
<artifactId>netty-handler</artifactId>
468471
<version>4.1.110.Final</version>
472+
<optional>true</optional>
469473
</dependency>
470474
<!-- FileSystem dependency begin -->
471475
<dependency>
@@ -647,7 +651,7 @@
647651
<dependency>
648652
<groupId>io.minio</groupId>
649653
<artifactId>minio</artifactId>
650-
<version>8.4.0</version>
654+
<version>8.5.17</version>
651655
<optional>true</optional>
652656
<exclusions>
653657
<exclusion>
@@ -660,6 +664,11 @@
660664
</exclusion>
661665
</exclusions>
662666
</dependency>
667+
<dependency>
668+
<groupId>io.minio</groupId>
669+
<artifactId>minio-admin</artifactId>
670+
<version>8.5.17</version>
671+
</dependency>
663672
<dependency>
664673
<groupId>com.google.cloud</groupId>
665674
<artifactId>google-cloud-storage</artifactId>
@@ -781,10 +790,12 @@
781790
<dependency>
782791
<groupId>ch.qos.logback</groupId>
783792
<artifactId>logback-classic</artifactId>
793+
<optional>true</optional>
784794
</dependency>
785795
<dependency>
786796
<groupId>ch.qos.logback</groupId>
787797
<artifactId>logback-core</artifactId>
798+
<optional>true</optional>
788799
</dependency>
789800
<dependency>
790801
<groupId>cn.hutool</groupId>
@@ -795,6 +806,7 @@
795806
<groupId>io.etcd</groupId>
796807
<artifactId>jetcd-core</artifactId>
797808
<version>0.7.3</version>
809+
<optional>true</optional>
798810
</dependency>
799811

800812

@@ -825,7 +837,7 @@
825837
</archive>
826838
</configuration>
827839
</plugin>
828-
<plugin><!-- 拷贝依赖的jar包到指定路径 -->
840+
<!--<plugin>
829841
<groupId>org.apache.maven.plugins</groupId>
830842
<artifactId>maven-dependency-plugin</artifactId>
831843
<executions>
@@ -842,7 +854,7 @@
842854
</configuration>
843855
</execution>
844856
</executions>
845-
</plugin>
857+
</plugin>-->
846858

847859
<plugin>
848860
<groupId>org.apache.maven.plugins</groupId>

‎hadooptool/src/main/java/com/robin/comm/fileaccess/fs/AbstractCloudStorageFileSystemAccessor.java

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.springframework.util.ObjectUtils;
1111

1212
import java.io.*;
13+
import java.util.Map;
1314

1415
/**
1516
* Cloud Storage FileSystemAccessor Abstract super class,not singleton,must init individual
@@ -18,12 +19,18 @@
1819
public abstract class AbstractCloudStorageFileSystemAccessor extends AbstractFileSystemAccessor {
1920
protected String bucketName;
2021
protected String tmpFilePath;
21-
22+
protected boolean useAdmin=false;
2223
protected MemorySegment segment;
2324
protected boolean useFileCache = false;
25+
public void init(){
2426

25-
public void init() {
27+
}
2628

29+
public void init(DataCollectionMeta meta) {
30+
super.init(meta);
31+
if(meta.getResourceCfgMap().containsKey(ResourceConst.USEADMINTG) && "true".equalsIgnoreCase(meta.getResourceCfgMap().get(ResourceConst.USEADMINTG).toString())){
32+
useAdmin=true;
33+
}
2734
}
2835

2936
@Override
@@ -92,7 +99,8 @@ protected String getContentType(DataCollectionMeta meta) {
9299
return !ObjectUtils.isEmpty(meta.getContent()) && !ObjectUtils.isEmpty(meta.getContent().getContentType()) ? meta.getContent().getContentType() : ResourceConst.DEFAULTCONTENTTYPE;
93100
}
94101

95-
protected abstract boolean putObject(String bucketName, DataCollectionMeta meta, InputStream inputStream, long size) throws IOException;
102+
public abstract boolean putObject(String bucketName, DataCollectionMeta meta, InputStream inputStream, long size) throws IOException;
96103

97-
protected abstract InputStream getObject(String bucketName, String objectName);
104+
public abstract InputStream getObject(String bucketName, String objectName);
105+
public abstract boolean createBucket(String name, Map<String,String> paramMap, Map<String,Object> retMap) throws Exception;
98106
}

0 commit comments

Comments
 (0)