diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HbaseMapper.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HbaseMapper.java deleted file mode 100644 index 8314d39d..00000000 --- a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HbaseMapper.java +++ /dev/null @@ -1,104 +0,0 @@ -package io.github.dunwu.javadb.hbase; - -import io.github.dunwu.javadb.hbase.entity.BaseHbaseEntity; -import org.apache.hadoop.hbase.client.Connection; - -import java.util.Collection; -import java.util.List; - -/** - * Hbase Mapper - * - * @author Zhang Peng - * @date 2023-11-15 - */ -public interface HbaseMapper { - - /** - * 获取 Hbase 官方客户端实体 - */ - Connection getClient(); - - /** - * 获取命名空间 - */ - String getNamespace(); - - /** - * 获取表名 - */ - String getTableName(); - - /** - * 判断表是否存在 - */ - boolean existsTable(); - - /** - * 获取列族 - */ - String getFamily(); - - /** - * 获取实体类型 - */ - Class getEntityClass(); - - /** - * 根据 ID 查数据 - * - * @param id 即 Hbase rowkey - * @return / - */ - T pojoByRowKey(String id); - - /** - * 根据 ID 列表批量查数据 - * - * @param ids 即 Hbase rowkey - * @return / - */ - List pojoListByRowKeys(Collection ids); - - /** - * 根据 ID 滚动分页查询 - * - * @param scrollId 为空值时，默认查第一页 - * @param size 每页记录数 - * @return / - */ - List scroll(String scrollId, int size); - - /** - * 保存实体 - * - * @param entity 实体（存于默认的 Hbase 列族 f） - * @return / - */ - T save(T entity); - - /** - * 保存实体列表，每条记录将作为一行保存 - * - * @param list 实体列表（存于默认的 Hbase 列族 f） - * @return / - */ - boolean batchSave(Collection list); - - /** - * 根据 ID 删除记录 - * - * @param id 即 Hbase rowkey - * @return / - */ - boolean delete(String id); - - /** - * 根据 ID 列表批量删除记录 - * - * @param ids 即 Hbase rowkey - * @return / - */ - boolean batchDelete(Collection ids); - -} diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HbaseTemplate.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HbaseTemplate.java index 4a7f447c..018bbddd 100644 --- a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HbaseTemplate.java +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HbaseTemplate.java @@ -1,8 +1,8 @@ package io.github.dunwu.javadb.hbase; -import cn.hutool.core.bean.BeanUtil; -import cn.hutool.core.bean.copier.CopyOptions; import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateUtil; import cn.hutool.core.io.IoUtil; import cn.hutool.core.map.MapUtil; import cn.hutool.core.util.ArrayUtil; @@ -10,13 +10,14 @@ import cn.hutool.core.util.ReflectUtil; import cn.hutool.core.util.StrUtil; import io.github.dunwu.javadb.hbase.entity.BaseHbaseEntity; -import io.github.dunwu.javadb.hbase.entity.ColumnDo; -import io.github.dunwu.javadb.hbase.entity.FamilyDo; -import io.github.dunwu.javadb.hbase.entity.PageData; -import io.github.dunwu.javadb.hbase.entity.RowDo; -import io.github.dunwu.javadb.hbase.entity.ScrollData; +import io.github.dunwu.javadb.hbase.entity.common.ColumnDo; +import io.github.dunwu.javadb.hbase.entity.common.FamilyDo; +import io.github.dunwu.javadb.hbase.entity.common.PageData; +import io.github.dunwu.javadb.hbase.entity.common.RowDo; +import io.github.dunwu.javadb.hbase.entity.common.ScrollData; import io.github.dunwu.javadb.hbase.entity.scan.MultiFamilyScan; import io.github.dunwu.javadb.hbase.entity.scan.SingleFamilyScan; +import io.github.dunwu.javadb.hbase.util.JsonUtil; import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; @@ -43,8 +44,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; +import java.util.Date; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -221,7 +223,7 @@ public static Put newPut(String row, Long timestamp, String family, String colum if (StrUtil.isBlank(row) || StrUtil.isBlank(family) || StrUtil.isBlank(column) || StrUtil.isBlank(value)) { return null; } - Map columnMap = new HashMap<>(1); + Map columnMap = new LinkedHashMap<>(1); columnMap.put(column, value); return newPut(row, timestamp, family, columnMap); } @@ -230,22 +232,16 @@ public static Put newPut(String row, Long timestamp, String family, Map> familyMap = new HashMap<>(1); + Map> familyMap = new LinkedHashMap<>(1); familyMap.put(family, columnMap); return newPut(row, timestamp, familyMap); } - @SuppressWarnings("unchecked") public static Put newPut(String row, Long timestamp, String family, Object obj) { if (obj == null) { return null; } - Map columnMap; - if (obj instanceof Map) { - columnMap = (Map) obj; - } else { - columnMap = BeanUtil.beanToMap(obj); - } + Map columnMap = JsonUtil.toMap(obj); return newPut(row, timestamp, family, columnMap); } @@ -267,20 +263,26 @@ public static Put newPut(String row, Long timestamp, Map entry : columnMap.entrySet()) { String column = entry.getKey(); Object value = entry.getValue(); - if (ObjectUtil.isEmpty(value)) { continue; } - put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column), timestamp, - Bytes.toBytes(String.valueOf(value))); + if (value instanceof String) { + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column), timestamp, + Bytes.toBytes(value.toString())); + } else if (value instanceof Date) { + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column), timestamp, + Bytes.toBytes(DateUtil.format((Date) value, DatePattern.NORM_DATETIME_PATTERN))); + } else { + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column), + timestamp, Bytes.toBytes(JsonUtil.toString(value))); + } } } } return put; } - private static List newPutList(String family, Collection list) - throws IOException { + private static List newPutList(String family, Collection list) { long timestamp = System.currentTimeMillis(); List puts = new ArrayList<>(); for (T entity : list) { @@ -387,7 +389,10 @@ public T getEntity(String tableName, String row, String family, Class cla Map fieldMap = ReflectUtil.getFieldMap(clazz); String[] columns = fieldMap.keySet().toArray(new String[0]); Map columnMap = getColumnMap(tableName, row, family, columns); - return toEntity(columnMap, clazz); + if (MapUtil.isEmpty(columnMap)) { + return null; + } + return toEntity(ColumnDo.toKvMap(columnMap), clazz); } /** @@ -402,6 +407,33 @@ public T getEntity(String tableName, String row, String family, Class cla */ public List getEntityList(String tableName, String[] rows, String family, Class clazz) throws IOException { + Map map = getEntityMap(tableName, rows, family, clazz); + if (MapUtil.isEmpty(map)) { + return new ArrayList<>(0); + } + return new ArrayList<>(map.values()); + } + + /** + * 指定多行、列族，以实体 {@link T} 列表形式返回数据 + * + * @param tableName 表名 + * @param rows 指定多行 + * @param family 列族 + * @param clazz 返回实体类型 + * @param 实体类型 + * @return / + */ + public List getEntityList(String tableName, Collection rows, String family, Class clazz) + throws IOException { + if (CollectionUtil.isEmpty(rows)) { + return new ArrayList<>(0); + } + return getEntityList(tableName, rows.toArray(new String[0]), family, clazz); + } + + public Map getEntityMap(String tableName, String[] rows, String family, Class clazz) + throws IOException { if (StrUtil.isBlank(tableName) || ArrayUtil.isEmpty(rows) || StrUtil.isBlank(family) || clazz == null) { return null; @@ -413,26 +445,27 @@ public List getEntityList(String tableName, String[] rows, String family, Result[] results = batchGet(tableName, gets); if (ArrayUtil.isEmpty(results)) { - return new ArrayList<>(); + return new LinkedHashMap<>(0); } - List list = new ArrayList<>(); + Map map = new LinkedHashMap<>(results.length); for (Result result : results) { Map columnMap = getColumnsFromResult(result, tableName, family, CollectionUtil.newArrayList(columns)); if (MapUtil.isNotEmpty(columnMap)) { - T entity = toEntity(columnMap, clazz); - list.add(entity); + T entity = toEntity(ColumnDo.toKvMap(columnMap), clazz); + map.put(Bytes.toString(result.getRow()), entity); } } - return list; + return map; } - private T toEntity(Map columnMap, Class clazz) { - if (MapUtil.isEmpty(columnMap)) { - return null; + public Map getEntityMap(String tableName, Collection rows, String family, Class clazz) + throws IOException { + if (CollectionUtil.isEmpty(rows)) { + return new LinkedHashMap<>(0); } - return BeanUtil.mapToBean(ColumnDo.toKvMap(columnMap), clazz, true, CopyOptions.create().ignoreError()); + return getEntityMap(tableName, rows.toArray(new String[0]), family, clazz); } /** @@ -492,6 +525,9 @@ public Map getColumnMap(String tableName, String row, String f */ public FamilyDo getFamily(String tableName, String row, String family) throws IOException { Map columnMap = getColumnMap(tableName, row, family); + if (MapUtil.isEmpty(columnMap)) { + return null; + } return new FamilyDo(tableName, row, family, columnMap); } @@ -507,13 +543,13 @@ public Map getFamilyMap(String tableName, String row, Map> familyColumnMap) throws IOException { if (StrUtil.isBlank(tableName) || StrUtil.isBlank(row)) { - return new HashMap<>(0); + return new LinkedHashMap<>(0); } if (MapUtil.isEmpty(familyColumnMap)) { RowDo rowDo = getRow(tableName, row); if (rowDo == null) { - return new HashMap<>(0); + return new LinkedHashMap<>(0); } return rowDo.getFamilyMap(); } @@ -567,9 +603,9 @@ public Map getRowMap(String tableName, String... rows) throws IOE } Result[] results = batchGet(tableName, rows); if (ArrayUtil.isEmpty(results)) { - return new HashMap<>(0); + return new LinkedHashMap<>(0); } - Map map = new HashMap<>(results.length); + Map map = new LinkedHashMap<>(results.length); for (Result result : results) { String row = Bytes.toString(result.getRow()); RowDo rowDo = getRowFromResult(result, tableName); @@ -674,7 +710,7 @@ public PageData getEntityPage(SingleFamilyScan scan, Class clazz) thro List list = data.getContent().stream().map(rowDo -> { Map> familyKvMap = rowDo.getFamilyKvMap(); Map columnKvMap = familyKvMap.get(scan.getFamily()); - return BeanUtil.mapToBean(columnKvMap, clazz, true, CopyOptions.create().ignoreError()); + return toEntity(columnKvMap, clazz); }).collect(Collectors.toList()); return new PageData<>(scan.getPage(), scan.getSize(), data.getTotal(), list); } @@ -693,7 +729,7 @@ public ScrollData getEntityScroll(SingleFamilyScan scan, Class clazz) List list = data.getContent().stream().map(rowDo -> { Map> familyKvMap = rowDo.getFamilyKvMap(); Map columnKvMap = familyKvMap.get(scan.getFamily()); - return BeanUtil.mapToBean(columnKvMap, clazz, true, CopyOptions.create().ignoreError()); + return toEntity(columnKvMap, clazz); }).collect(Collectors.toList()); return new ScrollData<>(data.getStartRow(), data.getStopRow(), data.getScrollRow(), 0, list); } @@ -712,7 +748,7 @@ public ScrollData getEntityScroll(String tableName, String family, String private PageData getPageData(String tableName, Integer page, Integer size, Scan scan, Map> familyColumnMap) throws IOException { Table table = getTable(tableName); - Map rowMap = new HashMap<>(size); + Map rowMap = new LinkedHashMap<>(size); try { int pageIndex = 1; byte[] lastRow = null; @@ -728,7 +764,9 @@ private PageData getPageData(String tableName, Integer page, Integer size Result result = it.next(); if (page == pageIndex) { RowDo rowDo = getRowFromResult(result, tableName, familyColumnMap); - rowMap.put(rowDo.getRow(), rowDo); + if (rowDo != null) { + rowMap.put(rowDo.getRow(), rowDo); + } } lastRow = result.getRow(); count++; @@ -751,12 +789,14 @@ private ScrollData getScrollData(String tableName, int size, Scan scan, Map> familyColumnMap) throws IOException { Table table = getTable(tableName); ResultScanner scanner = null; - Map rowMap = new HashMap<>(size); + Map rowMap = new LinkedHashMap<>(size); try { scanner = table.getScanner(scan); for (Result result : scanner) { RowDo rowDo = getRowFromResult(result, tableName, familyColumnMap); - rowMap.put(rowDo.getRow(), rowDo); + if (rowDo != null) { + rowMap.put(rowDo.getRow(), rowDo); + } } String scrollRow = null; @@ -829,24 +869,25 @@ private static RowDo getRowFromResult(Result result, String tableName) { } String row = Bytes.toString(result.getRow()); - Map> familyColumnMap = new HashMap<>(result.size()); + Map> familyColumnMap = new LinkedHashMap<>(result.size()); for (Cell cell : result.listCells()) { String family = Bytes.toString(CellUtil.cloneFamily(cell)); if (!familyColumnMap.containsKey(family)) { - familyColumnMap.put(family, new HashMap<>(0)); + familyColumnMap.put(family, new LinkedHashMap<>(0)); } String column = Bytes.toString(CellUtil.cloneQualifier(cell)); - String value = Bytes.toString(CellUtil.cloneValue(cell)); - long timestamp = cell.getTimestamp(); - ColumnDo columnDo = new ColumnDo(tableName, row, family, timestamp, column, value); + ColumnDo columnDo = getColumnFromResult(result, tableName, family, column); familyColumnMap.get(family).put(column, columnDo); } - Map familyMap = new HashMap<>(familyColumnMap.size()); + Map familyMap = new LinkedHashMap<>(familyColumnMap.size()); familyColumnMap.forEach((family, columnMap) -> { FamilyDo familyDo = new FamilyDo(tableName, row, family, columnMap); familyMap.put(family, familyDo); }); + if (MapUtil.isEmpty(familyMap)) { + return null; + } return new RowDo(tableName, row, familyMap); } @@ -857,6 +898,9 @@ private static RowDo getRowFromResult(Result result, String tableName, } String row = Bytes.toString(result.getRow()); Map familyMap = getFamiliesFromResult(result, tableName, familyColumnMap); + if (MapUtil.isEmpty(familyMap)) { + return null; + } return new RowDo(tableName, row, familyMap); } @@ -877,23 +921,24 @@ private static Map getFamiliesFromResult(Result result, String Map> familyColumnMap) { if (result == null || StrUtil.isBlank(tableName) || MapUtil.isEmpty(familyColumnMap)) { - return new HashMap<>(0); + return new LinkedHashMap<>(0); } String row = Bytes.toString(result.getRow()); - Map familyMap = new HashMap<>(familyColumnMap.size()); + Map familyMap = new LinkedHashMap<>(familyColumnMap.size()); familyColumnMap.forEach((family, columns) -> { - Map columnMap; + FamilyDo familyDo; if (CollectionUtil.isNotEmpty(columns)) { - columnMap = new HashMap<>(columns.size()); + Map columnMap = new LinkedHashMap<>(columns.size()); for (String column : columns) { ColumnDo columnDo = getColumnFromResult(result, tableName, family, column); columnMap.put(column, columnDo); } + familyDo = new FamilyDo(tableName, row, family, columnMap); } else { - columnMap = new HashMap<>(0); + familyDo = getFamilyFromResult(result, tableName, family); } - familyMap.put(family, new FamilyDo(tableName, row, family, columnMap)); + familyMap.put(family, familyDo); }); return familyMap; } @@ -918,9 +963,12 @@ private static Map getColumnsFromResult(Result result, String Collection columns) { if (CollectionUtil.isEmpty(columns)) { RowDo rowDo = getRowFromResult(result, tableName); + if (rowDo == null) { + return new LinkedHashMap<>(0); + } return rowDo.getFamilyMap().get(family).getColumnMap(); } - Map columnMap = new HashMap<>(columns.size()); + Map columnMap = new LinkedHashMap<>(columns.size()); for (String column : columns) { ColumnDo columnDo = getColumnFromResult(result, tableName, family, column); if (columnDo != null) { @@ -930,4 +978,34 @@ private static Map getColumnsFromResult(Result result, String return columnMap; } + private static T toEntity(Map kvMap, Class clazz) { + + if (MapUtil.isEmpty(kvMap)) { + return null; + } + + MapUtil.removeNullValue(kvMap); + T obj; + try { + Map> typeMap = new LinkedHashMap<>(); + Field[] fields = ReflectUtil.getFields(clazz); + for (Field f : fields) { + typeMap.put(f.getName(), f.getType()); + } + obj = clazz.newInstance(); + for (Map.Entry entry : kvMap.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + Class filedType = typeMap.get(key); + if (filedType != null) { + Object fieldObj = JsonUtil.toBean(value, filedType); + ReflectUtil.setFieldValue(obj, key, fieldObj); + } + } + return obj; + } catch (InstantiationException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + } diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/annotation/RowKeyRule.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/annotation/RowKeyRule.java index 1b4a7db1..6c91c0a5 100644 --- a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/annotation/RowKeyRule.java +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/annotation/RowKeyRule.java @@ -16,13 +16,13 @@ */ @Documented @Retention(RetentionPolicy.RUNTIME) -@Target({ ElementType.FIELD, ElementType.ANNOTATION_TYPE }) +@Target({ ElementType.TYPE, ElementType.ANNOTATION_TYPE }) public @interface RowKeyRule { /** - * 字段名（该值可无） + * 主键 */ - String value() default ""; + String pk(); /** * 主键类型 {@link RowType} diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/annotation/RowKeyRuleParser.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/annotation/RowKeyRuleParser.java new file mode 100644 index 00000000..9912d141 --- /dev/null +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/annotation/RowKeyRuleParser.java @@ -0,0 +1,111 @@ +package io.github.dunwu.javadb.hbase.annotation; + +import cn.hutool.core.util.HashUtil; +import cn.hutool.core.util.IdUtil; +import cn.hutool.core.util.ReflectUtil; +import cn.hutool.core.util.StrUtil; +import io.github.dunwu.javadb.hbase.entity.BaseHbaseEntity; + +import java.lang.reflect.Field; + +/** + * {@link RowKeyRule} 解析器 + * + * @author Zhang Peng + * @date 2023-11-20 + */ +public class RowKeyRuleParser { + + /** + * 获取主键 + */ + public static String getRowKey(T entity) throws IllegalArgumentException { + + String row = null; + Class clazz = entity.getClass(); + RowKeyRule rule = clazz.getAnnotation(RowKeyRule.class); + if (rule == null) { + throw new IllegalArgumentException(StrUtil.format("实体定义错误！未定义 @RowKeyRule", entity.getClass(), + BaseHbaseEntity.class.getCanonicalName())); + } + + Field field = ReflectUtil.getField(clazz, rule.pk()); + if (field == null) { + throw new IllegalArgumentException(StrUtil.format("实体定义错误！@RowKeyRule 中未指定 value", entity.getClass(), + BaseHbaseEntity.class.getCanonicalName())); + } + + switch (rule.type()) { + case ORIGIN_ID: + row = getRowKeyForOriginId(entity, field, rule); + break; + case TIMESTAMP: + row = getRowKeyForTimestamp(); + break; + case UUID: + row = IdUtil.fastSimpleUUID(); + break; + case BUCKET: + row = getRowKeyForBucket(entity, field, rule); + default: + break; + } + + if (StrUtil.isBlank(row)) { + throw new IllegalArgumentException(StrUtil.format("实体定义错误！未定义 @RowKeyRule", entity.getClass(), + BaseHbaseEntity.class.getCanonicalName())); + } + return row; + } + + private static String getRowKeyForOriginId(T entity, Field field, RowKeyRule rule) + throws IllegalArgumentException { + String originId; + Object value = ReflectUtil.getFieldValue(entity, field); + if (value instanceof String) { + originId = (String) value; + } else { + originId = String.valueOf(value); + } + if (rule.length() == 0) { + throw new IllegalArgumentException( + StrUtil.format("实体定义错误！{} 选择 @RowKey 的 type 为 {}，必须指定 length 且不能为 0", + entity.getClass(), rule.type())); + } + return StrUtil.padPre(originId, rule.length(), "0"); + } + + private static String getRowKeyForTimestamp() { + String timestamp = String.valueOf(System.currentTimeMillis() / 1000); + return StrUtil.padPre(timestamp, 10, "0"); + } + + private static String getRowKeyForBucket(T entity, Field field, RowKeyRule rowKeyRule) + throws IllegalArgumentException { + if (rowKeyRule.bucket() == 0) { + throw new IllegalArgumentException( + StrUtil.format("实体定义错误！{} 选择 @RowKey 的 type 为 {}，必须指定 bucket 且不能为 0", + entity.getClass(), rowKeyRule.type())); + } + + String originId = getRowKeyForOriginId(entity, field, rowKeyRule); + int bucketLength = getBucketIdLength(rowKeyRule.bucket()); + String bucketId = String.valueOf(HashUtil.fnvHash(originId) % rowKeyRule.bucket()); + return StrUtil.padPre(bucketId, bucketLength, "0") + originId; + } + + private static int getBucketIdLength(int bucket) { + bucket = bucket - 1; + if (bucket <= 0) { + return 1; + } + + int length = 0; + while (bucket > 0) { + length++; + bucket = bucket / 10; + } + return length; + } + +} diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/BaseHbaseEntity.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/BaseHbaseEntity.java index 8cf30737..5a79debd 100644 --- a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/BaseHbaseEntity.java +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/BaseHbaseEntity.java @@ -1,16 +1,7 @@ package io.github.dunwu.javadb.hbase.entity; -import cn.hutool.core.util.HashUtil; -import cn.hutool.core.util.IdUtil; -import cn.hutool.core.util.ReflectUtil; -import cn.hutool.core.util.StrUtil; -import com.alibaba.fastjson.annotation.JSONField; import com.fasterxml.jackson.annotation.JsonIgnore; -import io.github.dunwu.javadb.hbase.annotation.RowKeyRule; - -import java.io.IOException; -import java.io.Serializable; -import java.lang.reflect.Field; +import io.github.dunwu.javadb.hbase.annotation.RowKeyRuleParser; /** * HBase 基础实体 @@ -18,104 +9,14 @@ * @author Zhang Peng * @date 2023-11-15 */ -public abstract class BaseHbaseEntity implements Serializable { - - @JsonIgnore - @JSONField(serialize = false, deserialize = false) - protected String rowKey; +public interface BaseHbaseEntity { /** * 获取主键 */ - public String getRowKey() throws IOException { - if (StrUtil.isNotBlank(rowKey)) { - return rowKey; - } - - String row = null; - Field[] fields = ReflectUtil.getFields(this.getClass()); - for (Field field : fields) { - RowKeyRule rule = field.getAnnotation(RowKeyRule.class); - if (rule != null) { - switch (rule.type()) { - case ORIGIN_ID: - row = getRowKeyForOriginId(this, field, rule); - break; - case TIMESTAMP: - row = getRowKeyForTimestamp(); - break; - case UUID: - row = IdUtil.fastSimpleUUID(); - break; - case BUCKET: - row = getRowKeyForBucket(this, field, rule); - default: - break; - } - } - } - - if (StrUtil.isBlank(row)) { - throw new IOException(StrUtil.format("实体定义错误！未定义 @RowKeyRule", - this.getClass(), BaseHbaseEntity.class.getCanonicalName())); - } - this.rowKey = row; - return row; - } - - private static String getRowKeyForOriginId(T entity, Field field, RowKeyRule rowKeyRule) - throws IOException { - String originId; - Object value = ReflectUtil.getFieldValue(entity, field); - if (value instanceof String) { - originId = (String) value; - } else { - originId = String.valueOf(value); - } - if (rowKeyRule.length() == 0) { - throw new IOException( - StrUtil.format("实体定义错误！{} 选择 @RowKey 的 type 为 {}，必须指定 length 且不能为 0", - entity.getClass(), rowKeyRule.type())); - } - return StrUtil.padPre(originId, rowKeyRule.length(), "0"); - } - - private static String getRowKeyForTimestamp() { - String timestamp = String.valueOf(System.currentTimeMillis() / 1000); - return StrUtil.padPre(timestamp, 10, "0"); - } - - private static String getRowKeyForBucket(T entity, Field field, RowKeyRule rowKeyRule) - throws IOException { - if (rowKeyRule.bucket() == 0) { - throw new IOException( - StrUtil.format("实体定义错误！{} 选择 @RowKey 的 type 为 {}，必须指定 bucket 且不能为 0", - entity.getClass(), rowKeyRule.type())); - } - - String originId = getRowKeyForOriginId(entity, field, rowKeyRule); - int bucketLength = getBucketIdLength(rowKeyRule.bucket()); - String bucketId = String.valueOf(HashUtil.fnvHash(originId) % rowKeyRule.bucket()); - String timestamp = String.valueOf(System.currentTimeMillis() / 1000); - return StrUtil.padPre(bucketId, bucketLength, "0") - + StrUtil.padPre(timestamp, 10, "0") - + originId; - } - - private static int getBucketIdLength(int bucket) { - bucket = bucket - 1; - if (bucket <= 0) { - return 1; - } - - int length = 0; - while (bucket > 0) { - length++; - bucket = bucket / 10; - } - return length; + @JsonIgnore + default String getRowKey() { + return RowKeyRuleParser.getRowKey(this); } - private static final long serialVersionUID = 5075127328254616085L; - } diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/ColumnDo.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/ColumnDo.java similarity index 97% rename from codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/ColumnDo.java rename to codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/ColumnDo.java index a2ba66a3..95167743 100644 --- a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/ColumnDo.java +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/ColumnDo.java @@ -1,4 +1,4 @@ -package io.github.dunwu.javadb.hbase.entity; +package io.github.dunwu.javadb.hbase.entity.common; import cn.hutool.core.map.MapUtil; import cn.hutool.core.util.StrUtil; diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/FamilyDo.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/FamilyDo.java similarity index 97% rename from codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/FamilyDo.java rename to codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/FamilyDo.java index 61d89c75..8d59bd8e 100644 --- a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/FamilyDo.java +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/FamilyDo.java @@ -1,4 +1,4 @@ -package io.github.dunwu.javadb.hbase.entity; +package io.github.dunwu.javadb.hbase.entity.common; import cn.hutool.core.map.MapUtil; import cn.hutool.core.util.StrUtil; diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/PageData.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/PageData.java similarity index 93% rename from codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/PageData.java rename to codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/PageData.java index 18c755a5..f044aab5 100644 --- a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/PageData.java +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/PageData.java @@ -1,4 +1,4 @@ -package io.github.dunwu.javadb.hbase.entity; +package io.github.dunwu.javadb.hbase.entity.common; import lombok.Data; diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/RowDo.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/RowDo.java similarity index 98% rename from codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/RowDo.java rename to codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/RowDo.java index 5432aca3..f9625d3f 100644 --- a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/RowDo.java +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/RowDo.java @@ -1,4 +1,4 @@ -package io.github.dunwu.javadb.hbase.entity; +package io.github.dunwu.javadb.hbase.entity.common; import cn.hutool.core.map.MapUtil; import cn.hutool.core.util.StrUtil; diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/ScrollData.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/ScrollData.java similarity index 89% rename from codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/ScrollData.java rename to codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/ScrollData.java index 1465027c..96cd4412 100644 --- a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/ScrollData.java +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/common/ScrollData.java @@ -1,4 +1,4 @@ -package io.github.dunwu.javadb.hbase.entity; +package io.github.dunwu.javadb.hbase.entity.common; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/scan/SingleFamilyScan.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/scan/SingleFamilyScan.java index 9858265c..614b689a 100644 --- a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/scan/SingleFamilyScan.java +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/entity/scan/SingleFamilyScan.java @@ -42,12 +42,18 @@ public Scan toScan() throws IOException { } public Map> getFamilyColumnMap() { - if (StrUtil.isNotBlank(family) && CollectionUtil.isNotEmpty(columns)) { - Map> familyColumnMap = new HashMap<>(1); + + if (StrUtil.isBlank(family)) { + return new HashMap<>(0); + } + + Map> familyColumnMap = new HashMap<>(1); + if (CollectionUtil.isNotEmpty(columns)) { familyColumnMap.put(family, columns); - return familyColumnMap; + } else { + familyColumnMap.put(family, new ArrayList<>()); } - return new HashMap<>(0); + return familyColumnMap; } } diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/BaseHbaseMapper.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/mapper/BaseHbaseMapper.java similarity index 51% rename from codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/BaseHbaseMapper.java rename to codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/mapper/BaseHbaseMapper.java index f2ac11af..5714f743 100644 --- a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/BaseHbaseMapper.java +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/mapper/BaseHbaseMapper.java @@ -1,19 +1,23 @@ -package io.github.dunwu.javadb.hbase; +package io.github.dunwu.javadb.hbase.mapper; import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.util.StrUtil; +import io.github.dunwu.javadb.hbase.HbaseTemplate; import io.github.dunwu.javadb.hbase.entity.BaseHbaseEntity; -import io.github.dunwu.javadb.hbase.entity.ScrollData; +import io.github.dunwu.javadb.hbase.entity.common.ScrollData; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.util.Bytes; import java.io.IOException; +import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; /** * HBase Mapper 基础类 @@ -27,8 +31,6 @@ public abstract class BaseHbaseMapper implements Hbas protected final HbaseTemplate hbaseTemplate; - protected final HbaseAdmin hbaseAdmin; - @Override public Connection getClient() { return hbaseTemplate.getConnection(); @@ -36,19 +38,7 @@ public Connection getClient() { @Override public String getNamespace() { - return "test"; - } - - @Override - public boolean existsTable() { - byte[] namespace = Bytes.toBytes(getNamespace()); - byte[] tableName = Bytes.toBytes(getTableName()); - try { - return hbaseAdmin.existsTable(TableName.valueOf(namespace, tableName)); - } catch (IOException e) { - log.error("【Hbase】existsTable 异常", e); - return false; - } + return "default"; } @Override @@ -57,95 +47,121 @@ public String getFamily() { } @Override - public T pojoByRowKey(String rowKey) { + public int deleteById(Serializable id) { + + String rowKey = getIdStr(id); if (StrUtil.isBlank(rowKey)) { - return null; + return 0; } + try { - return hbaseTemplate.getEntity(getFullTableName(), rowKey, getFamily(), getEntityClass()); + hbaseTemplate.delete(getFullTableName(), rowKey); + return 1; } catch (IOException e) { - log.error("【Hbase】pojoById 异常", e); - return null; + log.error("【Hbase】deleteById 异常", e); + return 0; } } @Override - public List pojoListByRowKeys(Collection rowKeys) { - if (CollectionUtil.isEmpty(rowKeys)) { - return null; - } - try { - return hbaseTemplate.getEntityList(getFullTableName(), rowKeys.toArray(new String[0]), - getFamily(), getEntityClass()); - } catch (IOException e) { - log.error("【Hbase】getEntityList 异常", e); - return new ArrayList<>(); + public int deleteById(T entity) { + if (entity == null) { + return 0; } + return deleteById(entity.getRowKey()); } @Override - public List scroll(String scrollRowKey, int size) { + public int deleteBatchIds(Collection ids) { + + if (CollectionUtil.isEmpty(ids)) { + return 0; + } + + List rowKeys = getIdStrList(ids); try { - ScrollData scrollData = - hbaseTemplate.getEntityScroll(getFullTableName(), getFamily(), scrollRowKey, size, getEntityClass()); - if (scrollData == null || CollectionUtil.isEmpty(scrollData.getContent())) { - return new ArrayList<>(); - } - return new ArrayList<>(scrollData.getContent()); - } catch (IOException e) { - log.error("【Hbase】getEntityScroll 异常", e); - return new ArrayList<>(); + hbaseTemplate.batchDelete(getFullTableName(), rowKeys.toArray(new String[0])); + return rowKeys.size(); + } catch (IOException | InterruptedException e) { + log.error("【Hbase】deleteBatchIds 异常", e); + return 0; } } @Override - public T save(T entity) { + public int save(T entity) { try { String rowKey = entity.getRowKey(); hbaseTemplate.put(getFullTableName(), rowKey, getFamily(), entity); - return entity; + return 1; } catch (IOException e) { - log.error("【Hbase】put 异常", e); - return null; + log.error("【Hbase】updateById 异常", e); + return 0; } } @Override - public boolean batchSave(Collection list) { + public int batchSave(Collection list) { + + if (CollectionUtil.isEmpty(list)) { + return 0; + } + try { hbaseTemplate.batchPut(getFullTableName(), getFamily(), list); - return true; + return list.size(); } catch (IOException | InterruptedException e) { - log.error("【Hbase】batchPut 异常", e); - return false; + log.error("【Hbase】batchSave 异常", e); + return 0; } } @Override - public boolean delete(String rowKey) { + public T getOneById(Serializable id) { + + String rowKey = getIdStr(id); if (StrUtil.isBlank(rowKey)) { - return true; + return null; } + try { - hbaseTemplate.delete(getFullTableName(), rowKey); - return true; + return hbaseTemplate.getEntity(getFullTableName(), rowKey, getFamily(), getEntityClass()); } catch (IOException e) { - log.error("【Hbase】delete 异常", e); - return false; + log.error("【Hbase】getOneById 异常", e); + return null; } } @Override - public boolean batchDelete(Collection rowKeys) { - if (CollectionUtil.isEmpty(rowKeys)) { - return true; + public Map getMapByIds(Collection ids) { + + if (CollectionUtil.isEmpty(ids)) { + return new LinkedHashMap<>(0); } + + List rowKeys = getIdStrList(ids); try { - hbaseTemplate.batchDelete(getFullTableName(), rowKeys.toArray(new String[0])); - return true; - } catch (IOException | InterruptedException e) { - log.error("【Hbase】batchDelete 异常", e); - return false; + return hbaseTemplate.getEntityMap(getFullTableName(), rowKeys.toArray(new String[0]), getFamily(), + getEntityClass()); + } catch (IOException e) { + log.error("【Hbase】getMapByIds 异常", e); + return new LinkedHashMap<>(0); + } + } + + @Override + public List scroll(Serializable scrollId, int size) { + String scrollRowKey = getIdStr(scrollId); + try { + ScrollData scrollData = + hbaseTemplate.getEntityScroll(getFullTableName(), getFamily(), scrollRowKey, size, getEntityClass()); + if (scrollData == null || CollectionUtil.isEmpty(scrollData.getContent())) { + return new ArrayList<>(); + } + return new ArrayList<>(scrollData.getContent()); + } catch (IOException e) { + log.error("【Hbase】getEntityScroll 异常", e); + return new ArrayList<>(); } } @@ -153,4 +169,26 @@ protected String getFullTableName() { return StrUtil.format("{}:{}", getNamespace(), getTableName()); } + protected String getIdStr(Serializable id) { + + if (id == null) { + return null; + } + + String rowKey; + if (id instanceof String) { + rowKey = (String) id; + } else { + rowKey = String.valueOf(id); + } + return rowKey; + } + + protected List getIdStrList(Collection ids) { + if (CollectionUtil.isEmpty(ids)) { + return new ArrayList<>(0); + } + return ids.stream().map(this::getIdStr).filter(Objects::nonNull).collect(Collectors.toList()); + } + } diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/mapper/CommonMapper.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/mapper/CommonMapper.java new file mode 100644 index 00000000..56749e11 --- /dev/null +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/mapper/CommonMapper.java @@ -0,0 +1,78 @@ +package io.github.dunwu.javadb.hbase.mapper; + +import java.io.Serializable; +import java.util.Collection; +import java.util.List; + +/** + * 通用 Mapper + * + * @author Zhang Peng + * @date 2023-11-22 + */ +public interface CommonMapper { + + /** + * 插入一条记录 + * + * @param entity 实体对象 + */ + int insert(T entity); + + /** + * 批量插入记录 + * + * @param list 实体对象列表 + */ + int insertBatch(Collection list); + + /** + * 根据 ID 删除 + * + * @param id 主键ID + */ + int deleteById(Serializable id); + + /** + * 根据实体(ID)删除 + * + * @param entity 实体对象 + */ + int deleteById(T entity); + + /** + * 删除（根据ID或实体 批量删除） + * + * @param idList 主键ID列表或实体列表(不能为 null 以及 empty) + */ + int deleteBatchIds(Collection idList); + + /** + * 根据 ID 修改 + * + * @param entity 实体对象 + */ + int updateById(T entity); + + /** + * 批量更新记录 + * + * @param list 实体对象列表 + */ + int updateBatchById(Collection list); + + /** + * 根据 ID 查询 + * + * @param id 主键ID + */ + T getOneById(Serializable id); + + /** + * 查询（根据ID 批量查询） + * + * @param idList 主键ID列表(不能为 null 以及 empty) + */ + List getListByIds(Collection idList); + +} diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/mapper/HbaseMapper.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/mapper/HbaseMapper.java new file mode 100644 index 00000000..b6418064 --- /dev/null +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/mapper/HbaseMapper.java @@ -0,0 +1,106 @@ +package io.github.dunwu.javadb.hbase.mapper; + +import cn.hutool.core.map.MapUtil; +import io.github.dunwu.javadb.hbase.entity.BaseHbaseEntity; +import org.apache.hadoop.hbase.client.Connection; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * Hbase Mapper + * + * @author Zhang Peng + * @date 2023-11-15 + */ +public interface HbaseMapper extends CommonMapper { + + /** + * 获取 Hbase 官方客户端实体 + */ + Connection getClient(); + + /** + * 获取命名空间 + */ + String getNamespace(); + + /** + * 获取表名 + */ + String getTableName(); + + /** + * 获取列族 + */ + String getFamily(); + + /** + * 获取实体类型 + */ + Class getEntityClass(); + + @Override + default int insert(T entity) { + return save(entity); + } + + @Override + default int updateById(T entity) { + return save(entity); + } + + /** + * 保存一条记录 + * + * @param entity 实体对象 + */ + int save(T entity); + + @Override + default int insertBatch(Collection list) { + return batchSave(list); + } + + @Override + default int updateBatchById(Collection list) { + return batchSave(list); + } + + /** + * 批量保存记录 + * + * @param list 实体对象列表 + */ + int batchSave(Collection list); + + @Override + default List getListByIds(Collection ids) { + Map map = getMapByIds(ids); + if (MapUtil.isEmpty(map)) { + return new ArrayList<>(); + } + return new ArrayList<>(map.values()); + } + + /** + * 根据 ID 列表批量查数据，以 Map 形式返回 + * + * @param ids 即 Hbase rowkey + * @return / + */ + Map getMapByIds(Collection ids); + + /** + * 根据 ID 滚动分页查询 + * + * @param scrollId 为空值时，默认查第一页 + * @param size 每页记录数 + * @return / + */ + List scroll(Serializable scrollId, int size); + +} diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/util/JsonUtil.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/util/JsonUtil.java new file mode 100644 index 00000000..17d57d03 --- /dev/null +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/util/JsonUtil.java @@ -0,0 +1,144 @@ +package io.github.dunwu.javadb.hbase.util; + +import cn.hutool.core.util.StrUtil; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.type.TypeFactory; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +public class JsonUtil { + + private JsonUtil() { } + + private static final ObjectMapper OBJECT_MAPPER; + private static final TypeFactory TYPE_FACTORY; + + static { + OBJECT_MAPPER = new ObjectMapper(); + OBJECT_MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL); + OBJECT_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + TYPE_FACTORY = OBJECT_MAPPER.getTypeFactory(); + } + + public static ObjectMapper getInstance() { + return OBJECT_MAPPER; + } + + /** + * 简单对象转换 + */ + @SuppressWarnings("unchecked") + public static T toBean(String json, Class clazz) { + if (StrUtil.isBlank(json)) { + return null; + } + if (clazz == String.class) { + return (T) json; + } + try { + return OBJECT_MAPPER.readValue(json, clazz); + } catch (IOException e) { + log.error("反序列化失败！json: {}, msg: {}", json, e.getMessage()); + } + return null; + } + + /** + * 复杂对象转换 + */ + public static T toBean(String json, TypeReference typeReference) { + if (StrUtil.isBlank(json)) { + return null; + } + try { + return (T) OBJECT_MAPPER.readValue(json, typeReference); + } catch (Exception e) { + log.error("反序列化失败！json: {}, msg: {}", json, e.getMessage()); + } + return null; + } + + public static T toBean(Map map, Class clazz) { + return OBJECT_MAPPER.convertValue(toString(map), clazz); + } + + public static String toString(Object obj) { + if (obj == null) { + return null; + } + if (obj instanceof String) { + return (String) obj; + } + try { + return OBJECT_MAPPER.writeValueAsString(obj); + } catch (JsonProcessingException e) { + log.error("序列化失败！obj: {}, msg: {}", obj, e.getMessage()); + } + return null; + } + + public static Map toMap(String json) { + if (StrUtil.isBlank(json)) { + return new HashMap<>(0); + } + try { + return OBJECT_MAPPER.readValue(json, new TypeReference>() { }); + } catch (Exception e) { + log.error("反序列化失败！json: {}, msg: {}", json, e.getMessage()); + } + return Collections.emptyMap(); + } + + public static Map toMap(Object obj) { + + if (obj == null) { + return null; + } + + try { + return OBJECT_MAPPER.readValue(toString(obj), new TypeReference>() { }); + } catch (IOException e) { + log.error("反序列化失败！json: {}, msg: {}", toString(obj), e.getMessage()); + } + return null; + } + + public static List toList(String json, Class clazz) { + if (StrUtil.isBlank(json)) { + return null; + } + JavaType javaType = TYPE_FACTORY.constructParametricType(List.class, clazz); + try { + return OBJECT_MAPPER.readValue(json, javaType); + } catch (IOException e) { + log.error("反序列化失败！json: {}, msg: {}", json, e.getMessage()); + } + return null; + } + + public static List toList(String json, TypeReference typeReference) { + if (StrUtil.isBlank(json)) { + return null; + } + JavaType elementType = TYPE_FACTORY.constructType(typeReference); + JavaType javaType = TYPE_FACTORY.constructParametricType(List.class, elementType); + try { + return OBJECT_MAPPER.readValue(json, javaType); + } catch (IOException e) { + log.error("反序列化失败！json: {}, msg: {}", json, e.getMessage()); + } + return null; + } + +} \ No newline at end of file diff --git a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseMapperTest.java b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseMapperTest.java index b1ac3a46..1722cba8 100644 --- a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseMapperTest.java +++ b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseMapperTest.java @@ -1,7 +1,6 @@ package io.github.dunwu.javadb.hbase; import cn.hutool.core.collection.CollectionUtil; -import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.DisplayName; @@ -9,8 +8,11 @@ import java.io.IOException; import java.math.BigDecimal; -import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; /** * @author Zhang Peng @@ -18,70 +20,57 @@ */ public class HbaseMapperTest { - private static ProductMapper mapper; + private static final OrderMapper mapper; static { HbaseTemplate hbaseTemplate = null; try { hbaseTemplate = HbaseFactory.newHbaseTemplate(); - HbaseAdmin hbaseAdmin = HbaseFactory.newHbaseAdmin(); - mapper = new ProductMapper(hbaseTemplate, hbaseAdmin); + mapper = new OrderMapper(hbaseTemplate); } catch (IOException e) { throw new RuntimeException(e); } } @Test - @DisplayName("批量保存、查询、删除测试") - public void batchSave() throws IOException { - Product product1 = new Product("test-key-8", "product8", new BigDecimal(4000.0)); - Product product2 = new Product("test-key-9", "product9", new BigDecimal(5000.0)); + @DisplayName("批量保存、查询、删除 BaseHbaseEntity 实体") + public void batchSave() { + + Date now = new Date(); + Product product1 = new Product("1", "product1", new BigDecimal(4000.0)); + Product product2 = new Product("2", "product2", new BigDecimal(5000.0)); List products = CollectionUtil.newArrayList(product1, product2); - mapper.batchSave(products); + User user1 = new User(1, "user1"); + Map tags = new LinkedHashMap<>(); + tags.put("type", "tool"); + tags.put("color", "red"); + + Order originOrder = Order.builder() + .id("1") + .user(user1) + .products(products) + .desc("测试订单") + .date(now) + .tags(tags) + .build(); + mapper.batchSave(Collections.singleton(originOrder)); - List list = mapper.pojoListByRowKeys(Arrays.asList(product1.getRowKey(), product2.getRowKey())); + List list = mapper.getListByIds(Collections.singleton(originOrder.getRowKey())); + Assertions.assertThat(list).isNotEmpty(); + Order order = list.get(0); + Assertions.assertThat(order).isNotNull(); + Assertions.assertThat(order.getDate()).isNotNull().isEqualTo(now); + Assertions.assertThat(order.getTags()).isNotNull().isEqualTo(tags); + Assertions.assertThat(order.getUser()).isNotNull().isEqualTo(user1); + Assertions.assertThat(order.getProducts()).isNotEmpty(); Assertions.assertThat(list).isNotEmpty(); - Assertions.assertThat(list.size()).isEqualTo(2); + Assertions.assertThat(list.size()).isEqualTo(1); System.out.println(JSONUtil.toJsonStr(list)); - mapper.batchDelete(Arrays.asList(product1.getRowKey(), product2.getRowKey())); + mapper.deleteBatchIds(Collections.singletonList(originOrder.getRowKey())); - List list2 = mapper.pojoListByRowKeys(Arrays.asList(product1.getRowKey(), product2.getRowKey())); + List list2 = mapper.getListByIds(Collections.singletonList(originOrder.getRowKey())); Assertions.assertThat(list2).isEmpty(); } - @Test - @DisplayName("scroll") - public void scroll() throws IOException { - Product product1 = new Product("test-key-8", "product8", new BigDecimal(4000.0)); - Product product2 = new Product("test-key-9", "product9", new BigDecimal(5000.0)); - List products = CollectionUtil.newArrayList(product1, product2); - mapper.batchSave(products); - - int size = 1; - String lastRowKey = null; - List list = mapper.scroll(null, size); - if (CollectionUtil.isNotEmpty(list)) { - Product last = CollectionUtil.getLast(list); - System.out.println("entity: " + JSONUtil.toJsonPrettyStr(last)); - lastRowKey = last.getRowKey(); - } - - while (true) { - List tempList = mapper.scroll(lastRowKey, size); - if (CollectionUtil.isEmpty(list)) { - break; - } - Product last = CollectionUtil.getLast(tempList); - if (last == null) { - break; - } - System.out.println("entity: " + JSONUtil.toJsonPrettyStr(last)); - lastRowKey = last.getRowKey(); - if (StrUtil.isBlank(lastRowKey)) { - break; - } - } - } - } diff --git a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseTemplateDeleteTest.java b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseTemplateDeleteTest.java deleted file mode 100644 index cffb5b18..00000000 --- a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseTemplateDeleteTest.java +++ /dev/null @@ -1,47 +0,0 @@ -package io.github.dunwu.javadb.hbase; - -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * Hbase 删除测试 - * - * @author Zhang Peng - * @date 2023-11-13 - */ -public class HbaseTemplateDeleteTest { - - public static final String TABLE_NAME = "test:test"; - - private static final HbaseTemplate HBASE_TEMPLATE; - - static { - try { - HBASE_TEMPLATE = HbaseFactory.newHbaseTemplate(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Test - @DisplayName("删除单条记录") - public void testDelete() throws IOException { - String rowkey = "test-key-1"; - HBASE_TEMPLATE.delete(TABLE_NAME, rowkey); - } - - @Test - @DisplayName("批量删除记录") - public void testBatchDelete() throws IOException, InterruptedException { - List rowkeys = new ArrayList<>(); - for (int i = 1; i <= 13; i++) { - rowkeys.add("test-key-" + i); - } - HBASE_TEMPLATE.batchDelete(TABLE_NAME, rowkeys.toArray(new String[0])); - } - -} diff --git a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseTemplateGetTest.java b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseTemplateGetTest.java index 3dc250f3..a0e953ce 100644 --- a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseTemplateGetTest.java +++ b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseTemplateGetTest.java @@ -1,26 +1,28 @@ package io.github.dunwu.javadb.hbase; -import cn.hutool.core.util.StrUtil; -import cn.hutool.json.JSONUtil; -import io.github.dunwu.javadb.hbase.entity.ColumnDo; -import io.github.dunwu.javadb.hbase.entity.FamilyDo; -import io.github.dunwu.javadb.hbase.entity.RowDo; +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.util.ObjectUtil; +import io.github.dunwu.javadb.hbase.entity.BaseHbaseEntity; +import io.github.dunwu.javadb.hbase.entity.common.ColumnDo; +import io.github.dunwu.javadb.hbase.entity.common.FamilyDo; +import io.github.dunwu.javadb.hbase.entity.common.RowDo; +import io.github.dunwu.javadb.hbase.util.JsonUtil; +import org.apache.hadoop.hbase.client.Put; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; +import java.math.BigDecimal; +import java.util.Date; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** - * Get 测试集 - *

- * 测试前，先完整执行 {@link HbaseTemplatePutTest} + * Hbase Get 测试 * * @author Zhang Peng * @date 2023-11-13 @@ -40,85 +42,267 @@ public class HbaseTemplateGetTest { } @Test - @DisplayName("查询实体") - public void getEntity() throws IOException { - User user = HBASE_TEMPLATE.getEntity(TABLE_NAME, "test-key-3", "f1", User.class); - System.out.println(StrUtil.format("查询实体: {}", JSONUtil.toJsonStr(user))); - Assertions.assertThat(user).isNotNull(); + @DisplayName("put、get 单列数据") + public void test00() throws IOException { + long timestamp = System.currentTimeMillis(); + HBASE_TEMPLATE.put(TABLE_NAME, "test-key-0", "f1", "name", "user0"); + ColumnDo columnDo = HBASE_TEMPLATE.getColumn(TABLE_NAME, "test-key-0", "f1", "name"); + Assertions.assertThat(columnDo).isNotNull(); + Assertions.assertThat(columnDo.getColumn()).isEqualTo("name"); + Assertions.assertThat(columnDo.getValue()).isEqualTo("user0"); + + HBASE_TEMPLATE.put(TABLE_NAME, "test-key-0", timestamp, "f2", "姓名", "张三"); + ColumnDo columnDo2 = HBASE_TEMPLATE.getColumn(TABLE_NAME, "test-key-0", "f2", "姓名"); + Assertions.assertThat(columnDo2).isNotNull(); + Assertions.assertThat(columnDo2.getColumn()).isEqualTo("姓名"); + Assertions.assertThat(columnDo2.getValue()).isEqualTo("张三"); + Assertions.assertThat(columnDo2.getTimestamp()).isEqualTo(timestamp); + + HBASE_TEMPLATE.delete(TABLE_NAME, "test-key-0"); + columnDo = HBASE_TEMPLATE.getColumn(TABLE_NAME, "test-key-0", "f1", "name"); + Assertions.assertThat(columnDo).isNull(); + columnDo2 = HBASE_TEMPLATE.getColumn(TABLE_NAME, "test-key-0", "f2", "姓名"); + Assertions.assertThat(columnDo2).isNull(); } @Test - @DisplayName("查询实体列表") - public void getEntityList() throws IOException { - List rows = Arrays.asList("test-key-3", "test-key-4", "test-key-5"); - List list = HBASE_TEMPLATE.getEntityList(TABLE_NAME, rows.toArray(new String[0]), "f1", User.class); - System.out.println(StrUtil.format("查询实体列表: {}", JSONUtil.toJsonStr(list))); - Assertions.assertThat(list).isNotEmpty(); - Assertions.assertThat(list.size()).isEqualTo(rows.size()); + @DisplayName("put、get 多列数据") + public void test01() throws IOException { + + String row = "test-key-1"; + long timestamp = System.currentTimeMillis(); + Map map1 = new HashMap<>(2); + map1.put("id", 1); + map1.put("name", "zhangsan"); + Map map2 = new HashMap<>(2); + map2.put("编号", 1); + map2.put("姓名", "张三"); + + HBASE_TEMPLATE.put(TABLE_NAME, row, timestamp, "f1", map1); + HBASE_TEMPLATE.put(TABLE_NAME, row, timestamp, "f2", map2); + + Map f1ColumnMap = HBASE_TEMPLATE.getColumnMap(TABLE_NAME, row, "f1", "id", "name"); + Assertions.assertThat(f1ColumnMap).isNotEmpty(); + Assertions.assertThat(f1ColumnMap.get("id")).isNotNull(); + Assertions.assertThat(f1ColumnMap.get("id").getValue()).isEqualTo(String.valueOf(1)); + Assertions.assertThat(f1ColumnMap.get("name")).isNotNull(); + Assertions.assertThat(f1ColumnMap.get("name").getValue()).isEqualTo("zhangsan"); + + Map f2ColumnMap = HBASE_TEMPLATE.getColumnMap(TABLE_NAME, row, "f2", "编号", "姓名"); + Assertions.assertThat(f2ColumnMap).isNotEmpty(); + Assertions.assertThat(f2ColumnMap.get("编号")).isNotNull(); + Assertions.assertThat(f2ColumnMap.get("编号").getValue()).isEqualTo(String.valueOf(1)); + Assertions.assertThat(f2ColumnMap.get("姓名")).isNotNull(); + Assertions.assertThat(f2ColumnMap.get("姓名").getValue()).isEqualTo("张三"); + + HBASE_TEMPLATE.delete(TABLE_NAME, row); + f1ColumnMap = HBASE_TEMPLATE.getColumnMap(TABLE_NAME, row, "f1", "id", "name"); + Assertions.assertThat(f1ColumnMap).isEmpty(); + f2ColumnMap = HBASE_TEMPLATE.getColumnMap(TABLE_NAME, row, "f2", "编号", "姓名"); + Assertions.assertThat(f2ColumnMap).isEmpty(); } @Test - @DisplayName("查询列") - public void getColumn() throws IOException { - ColumnDo columnDo = HBASE_TEMPLATE.getColumn(TABLE_NAME, "test-key-1", "f1", "key"); - System.out.println(StrUtil.format("查询单列: {}", JSONUtil.toJsonStr(columnDo))); + @DisplayName("put、get 列族数据") + public void test02() throws IOException { + + String row = "test-key-2"; + long timestamp = System.currentTimeMillis(); + Map map1 = new HashMap<>(2); + map1.put("id", 1); + map1.put("name", "zhangsan"); + Map map2 = new HashMap<>(2); + map2.put("编号", 1); + map2.put("姓名", "张三"); + + HBASE_TEMPLATE.put(TABLE_NAME, row, timestamp, "f1", map1); + HBASE_TEMPLATE.put(TABLE_NAME, row, timestamp, "f2", map2); + + FamilyDo f1 = HBASE_TEMPLATE.getFamily(TABLE_NAME, row, "f1"); + Assertions.assertThat(f1).isNotNull(); + Assertions.assertThat(f1.getColumnMap().get("id")).isNotNull(); + Assertions.assertThat(f1.getColumnMap().get("id").getValue()).isEqualTo(String.valueOf(1)); + Assertions.assertThat(f1.getColumnMap().get("name")).isNotNull(); + Assertions.assertThat(f1.getColumnMap().get("name").getValue()).isEqualTo("zhangsan"); + + FamilyDo f2 = HBASE_TEMPLATE.getFamily(TABLE_NAME, row, "f2"); + Assertions.assertThat(f2).isNotNull(); + Assertions.assertThat(f2.getColumnMap().get("编号")).isNotNull(); + Assertions.assertThat(f2.getColumnMap().get("编号").getValue()).isEqualTo(String.valueOf(1)); + Assertions.assertThat(f2.getColumnMap().get("姓名")).isNotNull(); + Assertions.assertThat(f2.getColumnMap().get("姓名").getValue()).isEqualTo("张三"); + + HBASE_TEMPLATE.delete(TABLE_NAME, row); + f1 = HBASE_TEMPLATE.getFamily(TABLE_NAME, row, "f1"); + Assertions.assertThat(f1).isNull(); + f2 = HBASE_TEMPLATE.getFamily(TABLE_NAME, row, "f2"); + Assertions.assertThat(f2).isNull(); } @Test - @DisplayName("查询多列") - public void getColumnMap() throws IOException { - Map columnMap = HBASE_TEMPLATE.getColumnMap(TABLE_NAME, "test-key-3", "f1"); - System.out.println(StrUtil.format("查询多列: {}", JSONUtil.toJsonStr(columnMap))); - Assertions.assertThat(columnMap).isNotEmpty(); - - Map columnMap2 = HBASE_TEMPLATE.getColumnMap(TABLE_NAME, "test-key-3", "f1", "id", "name"); - System.out.println(StrUtil.format("查询多列: {}", JSONUtil.toJsonStr(columnMap2))); - Assertions.assertThat(columnMap2).isNotEmpty(); + @DisplayName("put、get 单行数据") + public void test03() throws IOException { + + String row = "test-key-3"; + long timestamp = System.currentTimeMillis(); + Map map1 = new HashMap<>(2); + map1.put("id", 1); + map1.put("name", "zhangsan"); + Map map2 = new HashMap<>(2); + map2.put("编号", 1); + map2.put("姓名", "张三"); + Map> familyMap = new HashMap<>(2); + familyMap.put("f1", map1); + familyMap.put("f2", map2); + + HBASE_TEMPLATE.put(TABLE_NAME, row, timestamp, familyMap); + + RowDo rowDo = HBASE_TEMPLATE.getRow(TABLE_NAME, row); + Assertions.assertThat(rowDo).isNotNull(); + + FamilyDo f1 = rowDo.getFamilyMap().get("f1"); + Assertions.assertThat(f1).isNotNull(); + Assertions.assertThat(f1.getColumnMap()).isNotEmpty(); + Assertions.assertThat(f1.getColumnMap().get("id")).isNotNull(); + Assertions.assertThat(f1.getColumnMap().get("id").getValue()).isEqualTo(String.valueOf(1)); + Assertions.assertThat(f1.getColumnMap().get("name")).isNotNull(); + Assertions.assertThat(f1.getColumnMap().get("name").getValue()).isEqualTo("zhangsan"); + + FamilyDo f2 = rowDo.getFamilyMap().get("f2"); + Assertions.assertThat(f2).isNotNull(); + Assertions.assertThat(f2.getColumnMap()).isNotEmpty(); + Assertions.assertThat(f2.getColumnMap().get("编号")).isNotNull(); + Assertions.assertThat(f2.getColumnMap().get("编号").getValue()).isEqualTo(String.valueOf(1)); + Assertions.assertThat(f2.getColumnMap().get("姓名")).isNotNull(); + Assertions.assertThat(f2.getColumnMap().get("姓名").getValue()).isEqualTo("张三"); + + HBASE_TEMPLATE.delete(TABLE_NAME, row); + rowDo = HBASE_TEMPLATE.getRow(TABLE_NAME, row); + Assertions.assertThat(rowDo).isNull(); } @Test - @DisplayName("查询列族") - public void getFamily() throws IOException { - FamilyDo familyDo = HBASE_TEMPLATE.getFamily(TABLE_NAME, "test-key-7", "f1"); - System.out.println(StrUtil.format("查询列族: {}", JSONUtil.toJsonStr(familyDo))); - Assertions.assertThat(familyDo).isNotNull(); - Assertions.assertThat(familyDo.getFamily()).isEqualTo("f1"); - - FamilyDo familyDo2 = HBASE_TEMPLATE.getFamily(TABLE_NAME, "test-key-7", "f2"); - System.out.println(StrUtil.format("查询列族: {}", JSONUtil.toJsonStr(familyDo2))); - Assertions.assertThat(familyDo2).isNotNull(); - Assertions.assertThat(familyDo2.getFamily()).isEqualTo("f2"); + @DisplayName("put get 多行数据") + public void test04() throws IOException, InterruptedException { + + long timestamp = System.currentTimeMillis(); + + Map columnMap1 = new HashMap<>(2); + columnMap1.put("id", 1); + columnMap1.put("name", "zhangsan"); + Put put = HbaseTemplate.newPut("test-key-1", timestamp, "f1", columnMap1); + + Map columnMap2 = new HashMap<>(2); + columnMap2.put("id", 2); + columnMap2.put("name", "lisi"); + Put put2 = HbaseTemplate.newPut("test-key-2", timestamp, "f1", columnMap2); + + List puts = CollectionUtil.newArrayList(put, put2); + + HBASE_TEMPLATE.batchPut(TABLE_NAME, puts); + + Map rowMap = HBASE_TEMPLATE.getRowMap(TABLE_NAME, "test-key-1", "test-key-2"); + + RowDo rowDo1 = rowMap.get("test-key-1"); + Assertions.assertThat(rowDo1).isNotNull(); + FamilyDo f1 = rowDo1.getFamilyMap().get("f1"); + Assertions.assertThat(f1).isNotNull(); + Assertions.assertThat(f1.getColumnMap()).isNotEmpty(); + Assertions.assertThat(f1.getColumnMap().get("id")).isNotNull(); + Assertions.assertThat(f1.getColumnMap().get("id").getValue()).isEqualTo(String.valueOf(1)); + Assertions.assertThat(f1.getColumnMap().get("name")).isNotNull(); + Assertions.assertThat(f1.getColumnMap().get("name").getValue()).isEqualTo("zhangsan"); + + RowDo rowDo2 = rowMap.get("test-key-2"); + FamilyDo f2 = rowDo2.getFamilyMap().get("f1"); + Assertions.assertThat(f2).isNotNull(); + Assertions.assertThat(f2.getColumnMap()).isNotEmpty(); + Assertions.assertThat(f2.getColumnMap().get("id")).isNotNull(); + Assertions.assertThat(f2.getColumnMap().get("id").getValue()).isEqualTo(String.valueOf(2)); + Assertions.assertThat(f2.getColumnMap().get("name")).isNotNull(); + Assertions.assertThat(f2.getColumnMap().get("name").getValue()).isEqualTo("lisi"); + + HBASE_TEMPLATE.batchDelete(TABLE_NAME, "test-key-1", "test-key-2"); + rowDo1 = HBASE_TEMPLATE.getRow(TABLE_NAME, "test-key-1"); + Assertions.assertThat(rowDo1).isNull(); + rowDo2 = HBASE_TEMPLATE.getRow(TABLE_NAME, "test-key-2"); + Assertions.assertThat(rowDo2).isNull(); } @Test - @DisplayName("查询多列族") - public void getFamilyMap() throws IOException { - Map> familyColumnMap = new HashMap<>(); - familyColumnMap.put("f1", Collections.singleton("id")); - familyColumnMap.put("f2", Collections.singleton("name")); - Map familyMap = HBASE_TEMPLATE.getFamilyMap(TABLE_NAME, "test-key-7", familyColumnMap); - System.out.println(StrUtil.format("查询多列族: {}", JSONUtil.toJsonStr(familyMap))); - Assertions.assertThat(familyMap).isNotEmpty(); - Assertions.assertThat(familyMap.size()).isEqualTo(familyColumnMap.size()); + @DisplayName("put get 简单 Java 实体数据") + public void test05() throws IOException, InterruptedException { + + User originUser1 = new User(1, "user1"); + HBASE_TEMPLATE.put(TABLE_NAME, "test-key-1", "f1", originUser1); + User user1 = HBASE_TEMPLATE.getEntity(TABLE_NAME, "test-key-1", "f1", User.class); + Assertions.assertThat(user1).isNotNull(); + Assertions.assertThat(ObjectUtil.equals(originUser1, user1)).isTrue(); + + HBASE_TEMPLATE.batchDelete(TABLE_NAME, "test-key-1", "test-key-2"); + user1 = HBASE_TEMPLATE.getEntity(TABLE_NAME, "test-key-1", "f1", User.class); + Assertions.assertThat(user1).isNull(); } @Test - @DisplayName("查询行") - public void getRow() throws IOException { - RowDo rowDo = HBASE_TEMPLATE.getRow(TABLE_NAME, "test-key-7"); - System.out.println(StrUtil.format("查询行: {}", JSONUtil.toJsonStr(rowDo))); - Assertions.assertThat(rowDo).isNotNull(); - Assertions.assertThat(rowDo.getRow()).isEqualTo("test-key-7"); + @DisplayName("put get 实现 BaseHbaseEntity 的简单 Java 实体数据") + public void test06() throws IOException, InterruptedException { + + Product product1 = new Product("1", "product1", new BigDecimal(4000.0)); + Product product2 = new Product("2", "product2", new BigDecimal(5000.0)); + List products = CollectionUtil.newArrayList(product1, product2); + HBASE_TEMPLATE.batchPut(TABLE_NAME, "f1", products); + + List rows = products.stream().map(BaseHbaseEntity::getRowKey).collect(Collectors.toList()); + List list = HBASE_TEMPLATE.getEntityList(TABLE_NAME, rows, "f1", Product.class); + Assertions.assertThat(list).isNotEmpty(); + Assertions.assertThat(list.size()).isEqualTo(rows.size()); + + HBASE_TEMPLATE.batchDelete(TABLE_NAME, rows.toArray(new String[0])); + product1 = HBASE_TEMPLATE.getEntity(TABLE_NAME, "test-key-1", "f1", Product.class); + Assertions.assertThat(product1).isNull(); + product2 = HBASE_TEMPLATE.getEntity(TABLE_NAME, "test-key-2", "f1", Product.class); + Assertions.assertThat(product2).isNull(); + list = HBASE_TEMPLATE.getEntityList(TABLE_NAME, rows, "f1", Product.class); + Assertions.assertThat(list).isEmpty(); } @Test - @DisplayName("批量查询行记录") - public void getRowMap() throws IOException { - String[] rows = new String[] { "test-key-3", "test-key-4", "test-key-7" }; - Map rowMap = HBASE_TEMPLATE.getRowMap(TABLE_NAME, rows); - System.out.println(StrUtil.format("批量查询行记录: {}", JSONUtil.toJsonStr(rowMap))); - Assertions.assertThat(rowMap).isNotEmpty(); - Assertions.assertThat(rowMap.size()).isEqualTo(rows.length); + @DisplayName("put get 实现 BaseHbaseEntity 的复杂 Java 实体数据") + public void test07() throws IOException { + + Date now = new Date(); + Product product1 = new Product("1", "product1", new BigDecimal(4000.0)); + Product product2 = new Product("2", "product2", new BigDecimal(5000.0)); + List products = CollectionUtil.newArrayList(product1, product2); + User user1 = new User(1, "user1"); + Map tags = new LinkedHashMap<>(); + tags.put("type", "tool"); + tags.put("color", "red"); + + Order originOrder = Order.builder() + .id("1") + .user(user1) + .products(products) + .desc("测试订单") + .date(now) + .tags(tags) + .build(); + + HBASE_TEMPLATE.put(TABLE_NAME, "f1", originOrder); + + Order order = HBASE_TEMPLATE.getEntity(TABLE_NAME, originOrder.getRowKey(), "f1", Order.class); + Assertions.assertThat(order).isNotNull(); + Assertions.assertThat(order.getDate()).isNotNull().isEqualTo(now); + Assertions.assertThat(order.getTags()).isNotNull().isEqualTo(tags); + Assertions.assertThat(order.getUser()).isNotNull().isEqualTo(user1); + Assertions.assertThat(order.getProducts()).isNotEmpty(); + + System.out.println("order: " + JsonUtil.toString(order)); + + HBASE_TEMPLATE.delete(TABLE_NAME, originOrder.getRowKey()); + order = HBASE_TEMPLATE.getEntity(TABLE_NAME, order.getRowKey(), "f1", Order.class); + Assertions.assertThat(order).isNull(); } } diff --git a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseTemplatePutTest.java b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseTemplatePutTest.java deleted file mode 100644 index f7593036..00000000 --- a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseTemplatePutTest.java +++ /dev/null @@ -1,105 +0,0 @@ -package io.github.dunwu.javadb.hbase; - -import cn.hutool.core.bean.BeanUtil; -import cn.hutool.core.collection.CollectionUtil; -import org.apache.hadoop.hbase.client.Put; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.math.BigDecimal; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Hbase Put 测试 - * - * @author Zhang Peng - * @date 2023-11-13 - */ -public class HbaseTemplatePutTest { - - public static final String TABLE_NAME = "test:test"; - - private static final HbaseTemplate HBASE_TEMPLATE; - - static { - try { - HBASE_TEMPLATE = HbaseFactory.newHbaseTemplate(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Test - @DisplayName("put 测试 01") - public void put01() throws IOException { - long timestamp = System.currentTimeMillis(); - HBASE_TEMPLATE.put(TABLE_NAME, "test-key-0", "f1", "name", "user0"); - HBASE_TEMPLATE.put(TABLE_NAME, "test-key-1", timestamp, "f1", "name", "user1"); - } - - @Test - @DisplayName("put 测试 02") - public void put02() throws IOException { - long timestamp = System.currentTimeMillis(); - User user2 = new User(2, "user2"); - HBASE_TEMPLATE.put(TABLE_NAME, "test-key-2", "f1", user2); - User user3 = new User(3, "user3"); - HBASE_TEMPLATE.put(TABLE_NAME, "test-key-3", timestamp, "f1", user3); - } - - @Test - @DisplayName("put 测试 03") - public void put03() throws IOException { - long timestamp = System.currentTimeMillis(); - User user4 = new User(4, "user4"); - Map map = BeanUtil.beanToMap(user4); - HBASE_TEMPLATE.put(TABLE_NAME, "test-key-4", timestamp, "f1", map); - } - - @Test - @DisplayName("put 测试 04") - public void put04() throws IOException { - long timestamp = System.currentTimeMillis(); - User user5 = new User(5, "user5"); - Product product5 = new Product("test-key-5", "product5", new BigDecimal(4000.0)); - Map> familyMap = new HashMap<>(2); - Map userMap = BeanUtil.beanToMap(user5); - familyMap.put("f1", userMap); - Map productMap = BeanUtil.beanToMap(product5); - familyMap.put("f2", productMap); - HBASE_TEMPLATE.put(TABLE_NAME, "test-key-5", timestamp, familyMap); - } - - @Test - @DisplayName("put 测试 05") - public void put05() throws IOException { - Put put = HbaseTemplate.newPut("test-key-6", null, "f1", "name", "user6"); - HBASE_TEMPLATE.put(TABLE_NAME, put); - } - - @Test - @DisplayName("put 测试 06") - public void put06() throws IOException, InterruptedException { - long timestamp = System.currentTimeMillis(); - User user7 = new User(5, "user7"); - Product product7 = new Product("test-key-7", "product5", new BigDecimal(4000.0)); - Put put1 = HbaseTemplate.newPut("test-key-7", timestamp, "f1", user7); - Put put2 = HbaseTemplate.newPut("test-key-7", timestamp, "f2", product7); - List list = Arrays.asList(put1, put2); - HBASE_TEMPLATE.batchPut(TABLE_NAME, list); - } - - @Test - @DisplayName("batchPut 测试2") - public void batchPut2() throws IOException, InterruptedException { - Product product1 = new Product("test-key-8", "product8", new BigDecimal(4000.0)); - Product product2 = new Product("test-key-9", "product9", new BigDecimal(5000.0)); - List products = CollectionUtil.newArrayList(product1, product2); - HBASE_TEMPLATE.batchPut(TABLE_NAME, "f2", products); - } - -} diff --git a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseTemplateScanTest.java b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseTemplateScanTest.java index e916b150..4496fc08 100644 --- a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseTemplateScanTest.java +++ b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HbaseTemplateScanTest.java @@ -1,32 +1,31 @@ package io.github.dunwu.javadb.hbase; import cn.hutool.core.collection.CollectionUtil; -import cn.hutool.core.util.ReflectUtil; +import cn.hutool.core.util.RandomUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; -import io.github.dunwu.javadb.hbase.entity.PageData; -import io.github.dunwu.javadb.hbase.entity.RowDo; -import io.github.dunwu.javadb.hbase.entity.ScrollData; +import io.github.dunwu.javadb.hbase.entity.common.PageData; +import io.github.dunwu.javadb.hbase.entity.common.RowDo; +import io.github.dunwu.javadb.hbase.entity.common.ScrollData; import io.github.dunwu.javadb.hbase.entity.scan.MultiFamilyScan; import io.github.dunwu.javadb.hbase.entity.scan.SingleFamilyScan; +import org.apache.hadoop.hbase.client.Put; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.lang.reflect.Field; +import java.math.BigDecimal; +import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Get 测试集 *

- * 测试前，先完整执行 {@link HbaseTemplatePutTest} + * 测试前，先完整执行 {@link HbaseTemplateGetTest} * * @author Zhang Peng * @date 2023-11-13 @@ -45,103 +44,136 @@ public class HbaseTemplateScanTest { } } + @Test + @DisplayName("批量初始化") + public void init() throws IOException, InterruptedException { + List products = new ArrayList<>(); + List userPuts = new ArrayList<>(); + for (int i = 1; i <= 100; i++) { + Product product = new Product(String.valueOf(i), "product" + i, + new BigDecimal(RandomUtil.randomDouble(9999.0))); + products.add(product); + + User user = new User(i, "user" + i); + Put put = HbaseTemplate.newPut(product.getRowKey(), null, "f2", user); + userPuts.add(put); + } + HBASE_TEMPLATE.batchPut(TABLE_NAME, "f1", products); + HBASE_TEMPLATE.batchPut(TABLE_NAME, userPuts); + } + @Test @DisplayName("单列族分页查询") - public void page() throws IOException { - for (int page = 1; page <= 2; page++) { - SingleFamilyScan scan = new SingleFamilyScan(); - scan.setFamily("f1") - .setTableName(TABLE_NAME) - .setPage(page) - .setSize(2) - .setReversed(true); - PageData rowDoMap = HBASE_TEMPLATE.page(scan); - System.out.println(StrUtil.format("查询实体: {}", JSONUtil.toJsonStr(rowDoMap))); - Assertions.assertThat(rowDoMap).isNotNull(); + public void test01() throws IOException { + SingleFamilyScan scan = new SingleFamilyScan(); + scan.setFamily("f1") + .setTableName(TABLE_NAME) + .setPage(1) + .setSize(10) + .setReversed(true); + PageData firstPage = HBASE_TEMPLATE.page(scan); + System.out.println(StrUtil.format("第 {} 页数据: {}", 1, JSONUtil.toJsonStr(firstPage))); + + int totalPages = firstPage.getTotalPages(); + for (int page = 2; page <= totalPages; page++) { + scan.setPage(page); + PageData nextPage = HBASE_TEMPLATE.page(scan); + System.out.println(StrUtil.format("第 {} 页数据: {}", page, JSONUtil.toJsonStr(nextPage))); + Assertions.assertThat(nextPage).isNotNull(); } } @Test @DisplayName("多列族分页查询") - public void page2() throws IOException { + public void test02() throws IOException { Map> familyColumnMap = new HashMap<>(); - familyColumnMap.put("f1", Collections.singleton("id")); - familyColumnMap.put("f2", Collections.singleton("name")); - for (int page = 1; page <= 2; page++) { - MultiFamilyScan scan = new MultiFamilyScan(); - scan.setFamilyColumnMap(familyColumnMap) - .setTableName(TABLE_NAME) - .setPage(page) - .setSize(2) - .setReversed(true); - PageData rowDoMap = HBASE_TEMPLATE.page(scan); - System.out.println(StrUtil.format("查询实体: {}", JSONUtil.toJsonStr(rowDoMap))); - Assertions.assertThat(rowDoMap).isNotNull(); + familyColumnMap.put("f1", CollectionUtil.newArrayList("id", "name", "price")); + familyColumnMap.put("f2", CollectionUtil.newArrayList("id", "name")); + + MultiFamilyScan scan = new MultiFamilyScan(); + scan.setFamilyColumnMap(familyColumnMap) + .setTableName(TABLE_NAME) + .setPage(1) + .setSize(10) + .setReversed(true); + PageData firstPage = HBASE_TEMPLATE.page(scan); + System.out.println(StrUtil.format("第 {} 页数据: {}", 1, JSONUtil.toJsonStr(firstPage))); + + int totalPages = firstPage.getTotalPages(); + for (int page = 1; page <= totalPages; page++) { + scan.setPage(page); + PageData nextPage = HBASE_TEMPLATE.page(scan); + System.out.println(StrUtil.format("查询实体: {}", JSONUtil.toJsonStr(nextPage))); + Assertions.assertThat(nextPage).isNotNull(); } } @Test - @DisplayName("查询实体列表") - public void getEntityPage() throws IOException { + @DisplayName("实体分页查询") + public void test03() throws IOException { + SingleFamilyScan scan = new SingleFamilyScan(); - scan.setFamily("f1") + scan.setFamily("f2") .setTableName(TABLE_NAME) .setPage(1) - .setSize(2) + .setSize(10) .setReversed(true); - PageData entityPage = HBASE_TEMPLATE.getEntityPage(scan, User.class); - System.out.println(StrUtil.format("查询实体列表: {}", JSONUtil.toJsonStr(entityPage))); - Assertions.assertThat(entityPage).isNotNull(); + PageData firstPage = HBASE_TEMPLATE.getEntityPage(scan, User.class); + System.out.println(StrUtil.format("第 {} 页数据: {}", 1, JSONUtil.toJsonStr(firstPage))); + + int totalPages = firstPage.getTotalPages(); + for (int page = 2; page <= totalPages; page++) { + scan.setPage(page); + PageData nextPage = HBASE_TEMPLATE.getEntityPage(scan, User.class); + System.out.println(StrUtil.format("第 {} 页数据: {}", page, JSONUtil.toJsonStr(nextPage))); + Assertions.assertThat(nextPage).isNotNull(); + } } @Test @DisplayName("单列族滚动查询") - public void scroll() throws IOException { + public void test04() throws IOException { SingleFamilyScan scan = new SingleFamilyScan(); scan.setFamily("f1") .setTableName(TABLE_NAME) - .setSize(1) - .setStartRow("test-key-1") - .setStopRow("test-key-9") + .setSize(10) .setReversed(false); - ScrollData data = HBASE_TEMPLATE.scroll(scan); - System.out.println(StrUtil.format("查询实体: {}", JSONUtil.toJsonPrettyStr(data))); - Assertions.assertThat(data).isNotNull(); - scan.setScrollRow(data.getScrollRow()); + + int page = 1; + ScrollData first = HBASE_TEMPLATE.scroll(scan); + System.out.println(StrUtil.format("第 {} 页数据: {}", page, JSONUtil.toJsonPrettyStr(first))); + Assertions.assertThat(first).isNotNull(); + scan.setScrollRow(first.getScrollRow()); while (true) { + page++; ScrollData next = HBASE_TEMPLATE.scroll(scan); if (next == null || CollectionUtil.isEmpty(next.getContent())) { break; } - System.out.println(StrUtil.format("查询实体: {}", JSONUtil.toJsonPrettyStr(next))); + System.out.println(StrUtil.format("第 {} 页数据: {}", page, JSONUtil.toJsonPrettyStr(first))); scan.setScrollRow(next.getScrollRow()); } } @Test @DisplayName("多列族滚动查询") - public void scroll2() throws IOException { - List userFields = Stream.of(ReflectUtil.getFields(User.class)) - .map(Field::getName).collect(Collectors.toList()); - List productFields = Stream.of(ReflectUtil.getFields(Product.class)) - .map(Field::getName).collect(Collectors.toList()); + public void test05() throws IOException { Map> familyColumnMap = new HashMap<>(); - familyColumnMap.put("f1", userFields); - familyColumnMap.put("f2", productFields); + familyColumnMap.put("f1", CollectionUtil.newArrayList("id", "name", "price")); + familyColumnMap.put("f2", CollectionUtil.newArrayList("id", "name")); MultiFamilyScan scan = new MultiFamilyScan(); scan.setFamilyColumnMap(familyColumnMap) .setTableName(TABLE_NAME) - .setSize(1) - .setStartRow("test-key-1") - .setStopRow("test-key-9") + .setSize(10) .setReversed(true); - ScrollData data = HBASE_TEMPLATE.scroll(scan); - System.out.println(StrUtil.format("查询实体: {}", JSONUtil.toJsonPrettyStr(data))); - Assertions.assertThat(data).isNotNull(); - scan.setScrollRow(data.getScrollRow()); + + ScrollData first = HBASE_TEMPLATE.scroll(scan); + System.out.println(StrUtil.format("查询实体: {}", JSONUtil.toJsonPrettyStr(first))); + Assertions.assertThat(first).isNotNull(); + scan.setScrollRow(first.getScrollRow()); while (true) { ScrollData next = HBASE_TEMPLATE.scroll(scan); @@ -155,28 +187,55 @@ public void scroll2() throws IOException { @Test @DisplayName("滚动查询实体") - public void getEntityScroll() throws IOException { + public void test06() throws IOException { SingleFamilyScan scan = new SingleFamilyScan(); scan.setFamily("f1") .setTableName(TABLE_NAME) - .setSize(1) - .setStartRow("test-key-1") - .setStopRow("test-key-9") + .setSize(10) .setReversed(false); - ScrollData data = HBASE_TEMPLATE.getEntityScroll(scan, User.class); - System.out.println(StrUtil.format("查询实体: {}", JSONUtil.toJsonPrettyStr(data))); - Assertions.assertThat(data).isNotNull(); - scan.setScrollRow(data.getScrollRow()); + ScrollData first = HBASE_TEMPLATE.getEntityScroll(scan, Product.class); + System.out.println(StrUtil.format("查询实体: {}", JSONUtil.toJsonPrettyStr(first))); + Assertions.assertThat(first).isNotNull(); + scan.setScrollRow(first.getScrollRow()); while (true) { - ScrollData next = HBASE_TEMPLATE.getEntityScroll(scan, User.class); + ScrollData next = HBASE_TEMPLATE.getEntityScroll(scan, Product.class); + if (next == null || CollectionUtil.isEmpty(next.getContent())) { + break; + } + System.out.println(StrUtil.format("查询实体: {}", JSONUtil.toJsonPrettyStr(next))); + scan.setScrollRow(next.getScrollRow()); + } + } + + @Test + @DisplayName("滚动删除全部记录") + public void clear() throws IOException, InterruptedException { + + SingleFamilyScan scan = new SingleFamilyScan(); + scan.setFamily("f1") + .setTableName(TABLE_NAME) + .setSize(100) + .setReversed(false); + + ScrollData first = HBASE_TEMPLATE.scroll(scan); + System.out.println(StrUtil.format("查询实体: {}", JSONUtil.toJsonPrettyStr(first))); + Assertions.assertThat(first).isNotNull(); + scan.setScrollRow(first.getScrollRow()); + HBASE_TEMPLATE.batchDelete(TABLE_NAME, + first.getContent().stream().map(RowDo::getRow).distinct().toArray(String[]::new)); + + while (true) { + ScrollData next = HBASE_TEMPLATE.scroll(scan); if (next == null || CollectionUtil.isEmpty(next.getContent())) { break; } System.out.println(StrUtil.format("查询实体: {}", JSONUtil.toJsonPrettyStr(next))); scan.setScrollRow(next.getScrollRow()); + HBASE_TEMPLATE.batchDelete(TABLE_NAME, + next.getContent().stream().map(RowDo::getRow).distinct().toArray(String[]::new)); } } diff --git a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/Order.java b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/Order.java new file mode 100644 index 00000000..53ab7787 --- /dev/null +++ b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/Order.java @@ -0,0 +1,34 @@ +package io.github.dunwu.javadb.hbase; + +import io.github.dunwu.javadb.hbase.annotation.RowKeyRule; +import io.github.dunwu.javadb.hbase.entity.BaseHbaseEntity; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +/** + * 较为复杂的 Java 实体 + * + * @author Zhang Peng + * @date 2023-11-20 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@RowKeyRule(pk = "id", length = 20) +public class Order implements BaseHbaseEntity { + + private String id; + private User user; + private List products; + private String desc; + private Date date; + private Map tags; + +} diff --git a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/ProductMapper.java b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/OrderMapper.java similarity index 53% rename from codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/ProductMapper.java rename to codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/OrderMapper.java index b95c1eca..f6639380 100644 --- a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/ProductMapper.java +++ b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/OrderMapper.java @@ -1,13 +1,15 @@ package io.github.dunwu.javadb.hbase; +import io.github.dunwu.javadb.hbase.mapper.BaseHbaseMapper; + /** * @author Zhang Peng * @date 2023-11-15 */ -public class ProductMapper extends BaseHbaseMapper { +public class OrderMapper extends BaseHbaseMapper { - public ProductMapper(HbaseTemplate hbaseTemplate, HbaseAdmin hbaseAdmin) { - super(hbaseTemplate, hbaseAdmin); + public OrderMapper(HbaseTemplate hbaseTemplate) { + super(hbaseTemplate); } @Override @@ -21,8 +23,8 @@ public String getFamily() { } @Override - public Class getEntityClass() { - return Product.class; + public Class getEntityClass() { + return Order.class; } } diff --git a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/Product.java b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/Product.java index ec15a1dd..f26ae56b 100644 --- a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/Product.java +++ b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/Product.java @@ -4,25 +4,22 @@ import io.github.dunwu.javadb.hbase.entity.BaseHbaseEntity; import lombok.AllArgsConstructor; import lombok.Data; -import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import java.math.BigDecimal; - /** * 产品实体 * * @author Zhang Peng * @date 2023-11-15 */ -@EqualsAndHashCode(callSuper = true) @Data @NoArgsConstructor @AllArgsConstructor -public class Product extends BaseHbaseEntity { +@RowKeyRule(pk = "id", length = 10) +public class Product implements BaseHbaseEntity { - @RowKeyRule(length = 20) private String id; private String name; private BigDecimal price;