Skip to content

Commit

Permalink
fix(tianmu): bugfix expression-column base on user value update by ro…
Browse files Browse the repository at this point in the history
…w order #1903

Cause:
  Materialize in temptable fill buffer by column order.The user value
base on previous column cannot get the true value.
Solution:
  Column base on user value fill buffer by row order.
  • Loading branch information
Double0101 committed Jul 5, 2023
1 parent d9faa83 commit 20e13dc
Show file tree
Hide file tree
Showing 9 changed files with 251 additions and 5 deletions.
95 changes: 95 additions & 0 deletions mysql-test/suite/tianmu/r/issue1903.result
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
DROP DATABASE IF EXISTS issue1903_test_db;
CREATE DATABASE issue1903_test_db;
USE issue1903_test_db;
CREATE TABLE `c1am_acct_day` (
`ACCOUNT_ID` decimal(18,0) NOT NULL DEFAULT '-1' COMMENT '账户ID',
`FISCAL_DATE` date DEFAULT NULL COMMENT '记账日期',
`BALANCE` decimal(16,2) NOT NULL DEFAULT '0.00' COMMENT '余额',
`DELETED_FLAG` char(1) NOT NULL DEFAULT '0' COMMENT '记录删除标志 [0]-未删除;[1]-逻辑删除'
) ENGINE=TIANMU;
CREATE TABLE `c1md_bank_acct` (
`ROW_ID` decimal(18,0) NOT NULL DEFAULT '-1' COMMENT 'ROW_ID',
`CURRENCY_ID` decimal(18,0) NOT NULL DEFAULT '-1' COMMENT '币种ID',
`DELETED_FLAG` char(1) NOT NULL DEFAULT '0' COMMENT '记录删除标志 [0]-未删除;[1]-逻辑删除'
) ENGINE=TIANMU;
INSERT INTO `c1am_acct_day`
VALUES
(3000000000028804, '2023-04-16', 7628617.08, '0'),
(3000000000028804, '2023-04-17', 7626656.73, '0'),
(3000000000028804, '2023-04-18', 7626471.23, '0'),
(3000000000028806, '2023-04-15', 605253889.19, '0'),
(3000000000028806, '2023-04-16', 611274357.27, '0'),
(3000000000028806, '2023-04-17', 605257716.01, '0'),
(3000000000028808, '2023-04-18', 79322521.29, '0'),
(3000000000028808, '2023-04-19', 79322521.29, '0'),
(3000000000028808, '2023-04-20', 79322521.29, '0'),
(3000000000028809, '2023-04-18', 79322521.29, '0'),
(3000000000028809, '2023-04-19', 79322521.29, '0'),
(3000000000028809, '2023-04-20', 79322521.29, '0');
INSERT INTO `c1md_bank_acct`
VALUES
(3000000000028804, 1, '0'),
(3000000000028806, 3, '0'),
(3000000000028808, 15, '0'),
(3000000000028809, 6, '0');
SELECT
result.*
FROM (
SELECT
a.*,
@rownum1 := @rownum1 + 1 inde,
IF(@pxydm1 = a.account_id,@rankno1 := @rankno1 + 1,@rankno1 := 1) AS rankno,
@pxydm1 := a.account_id
FROM (
SELECT
b.CURRENCY_ID,
a.account_id,
a.fiscal_date,
a.balance
FROM
c1am_acct_day a, c1md_bank_acct b
WHERE a.deleted_flag = '0'
AND b.deleted_flag = '0'
AND a.account_id = b.ROW_ID
ORDER BY a.account_id, a.fiscal_date) a) result
WHERE result.rankno = 1;
CURRENCY_ID account_id fiscal_date balance inde rankno @pxydm1 := a.account_id
1 3000000000028804 2023-04-16 7628617.08 NULL 1 3000000000028804
1 3000000000028804 2023-04-17 7626656.73 NULL 1 3000000000028804
1 3000000000028804 2023-04-18 7626471.23 NULL 1 3000000000028804
3 3000000000028806 2023-04-15 605253889.19 NULL 1 3000000000028806
3 3000000000028806 2023-04-16 611274357.27 NULL 1 3000000000028806
3 3000000000028806 2023-04-17 605257716.01 NULL 1 3000000000028806
15 3000000000028808 2023-04-18 79322521.29 NULL 1 3000000000028808
15 3000000000028808 2023-04-19 79322521.29 NULL 1 3000000000028808
15 3000000000028808 2023-04-20 79322521.29 NULL 1 3000000000028808
6 3000000000028809 2023-04-18 79322521.29 NULL 1 3000000000028809
6 3000000000028809 2023-04-19 79322521.29 NULL 1 3000000000028809
6 3000000000028809 2023-04-20 79322521.29 NULL 1 3000000000028809
SELECT
result.*
FROM (
SELECT
a.*,
@rownum1 := @rownum1 + 1 inde,
IF(@pxydm1 = a.account_id,@rankno1 := @rankno1 + 1,@rankno1 := 1) AS rankno,
@pxydm1 := a.account_id
FROM (
SELECT
b.CURRENCY_ID,
a.account_id,
a.fiscal_date,
a.balance
FROM
c1am_acct_day a, c1md_bank_acct b
WHERE a.deleted_flag = '0'
AND b.deleted_flag = '0'
AND a.account_id = b.ROW_ID
ORDER BY a.account_id, a.fiscal_date) a) result
WHERE result.rankno = 1;
CURRENCY_ID account_id fiscal_date balance inde rankno @pxydm1 := a.account_id
1 3000000000028804 2023-04-16 7628617.08 NULL 1 3000000000028804
3 3000000000028806 2023-04-15 605253889.19 NULL 1 3000000000028806
15 3000000000028808 2023-04-18 79322521.29 NULL 1 3000000000028808
6 3000000000028809 2023-04-18 79322521.29 NULL 1 3000000000028809
DROP DATABASE issue1903_test_db;
88 changes: 88 additions & 0 deletions mysql-test/suite/tianmu/t/issue1903.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
--source include/have_tianmu.inc

--disable_warnings
DROP DATABASE IF EXISTS issue1903_test_db;
--enable_warnings
CREATE DATABASE issue1903_test_db;
USE issue1903_test_db;

CREATE TABLE `c1am_acct_day` (
`ACCOUNT_ID` decimal(18,0) NOT NULL DEFAULT '-1' COMMENT '账户ID',
`FISCAL_DATE` date DEFAULT NULL COMMENT '记账日期',
`BALANCE` decimal(16,2) NOT NULL DEFAULT '0.00' COMMENT '余额',
`DELETED_FLAG` char(1) NOT NULL DEFAULT '0' COMMENT '记录删除标志 [0]-未删除;[1]-逻辑删除'
) ENGINE=TIANMU;

CREATE TABLE `c1md_bank_acct` (
`ROW_ID` decimal(18,0) NOT NULL DEFAULT '-1' COMMENT 'ROW_ID',
`CURRENCY_ID` decimal(18,0) NOT NULL DEFAULT '-1' COMMENT '币种ID',
`DELETED_FLAG` char(1) NOT NULL DEFAULT '0' COMMENT '记录删除标志 [0]-未删除;[1]-逻辑删除'
) ENGINE=TIANMU;

INSERT INTO `c1am_acct_day`
VALUES
(3000000000028804, '2023-04-16', 7628617.08, '0'),
(3000000000028804, '2023-04-17', 7626656.73, '0'),
(3000000000028804, '2023-04-18', 7626471.23, '0'),
(3000000000028806, '2023-04-15', 605253889.19, '0'),
(3000000000028806, '2023-04-16', 611274357.27, '0'),
(3000000000028806, '2023-04-17', 605257716.01, '0'),
(3000000000028808, '2023-04-18', 79322521.29, '0'),
(3000000000028808, '2023-04-19', 79322521.29, '0'),
(3000000000028808, '2023-04-20', 79322521.29, '0'),
(3000000000028809, '2023-04-18', 79322521.29, '0'),
(3000000000028809, '2023-04-19', 79322521.29, '0'),
(3000000000028809, '2023-04-20', 79322521.29, '0');

INSERT INTO `c1md_bank_acct`
VALUES
(3000000000028804, 1, '0'),
(3000000000028806, 3, '0'),
(3000000000028808, 15, '0'),
(3000000000028809, 6, '0');

SELECT
result.*
FROM (
SELECT
a.*,
@rownum1 := @rownum1 + 1 inde,
IF(@pxydm1 = a.account_id,@rankno1 := @rankno1 + 1,@rankno1 := 1) AS rankno,
@pxydm1 := a.account_id
FROM (
SELECT
b.CURRENCY_ID,
a.account_id,
a.fiscal_date,
a.balance
FROM
c1am_acct_day a, c1md_bank_acct b
WHERE a.deleted_flag = '0'
AND b.deleted_flag = '0'
AND a.account_id = b.ROW_ID
ORDER BY a.account_id, a.fiscal_date) a) result
WHERE result.rankno = 1;

SELECT
result.*
FROM (
SELECT
a.*,
@rownum1 := @rownum1 + 1 inde,
IF(@pxydm1 = a.account_id,@rankno1 := @rankno1 + 1,@rankno1 := 1) AS rankno,
@pxydm1 := a.account_id
FROM (
SELECT
b.CURRENCY_ID,
a.account_id,
a.fiscal_date,
a.balance
FROM
c1am_acct_day a, c1md_bank_acct b
WHERE a.deleted_flag = '0'
AND b.deleted_flag = '0'
AND a.account_id = b.ROW_ID
ORDER BY a.account_id, a.fiscal_date) a) result
WHERE result.rankno = 1;

DROP DATABASE issue1903_test_db;
26 changes: 26 additions & 0 deletions storage/tianmu/core/mysql_expression.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,32 @@ Item *MysqlExpression::TransformTree(Item *root, TransformDirection dir) {

MysqlExpression::SetOfVars &MysqlExpression::GetVars() { return vars; }

bool item_func_suser_traverse(Item_func *it_f) {
if (it_f->functype() == Item_func::Functype::SUSERVAR_FUNC)
return true;
bool res = false;
Item** args = nullptr;
uint cnt = it_f->arg_count;
if (it_f->arg_count > 0)
args = it_f->arguments();
for (uint i = 0; i < cnt && !res; ++i) {

if (args[i]->type() == Item::Type::FUNC_ITEM)
res = item_func_suser_traverse(reinterpret_cast<Item_func*>(args[i]));
}
return res;
}

/* Only return true when item set user value.
* It is equal to one of node in item tree is Item_func_set_user_var.
*/
bool MysqlExpression::BaseOnUserValue() {
if (item->type() != Item::Type::FUNC_ITEM)
return false;
Item_func *item_f = reinterpret_cast<Item_func*>(item);
return item_func_suser_traverse(item_f);
}

void MysqlExpression::SetBufsOrParams(var_buf_t *bufs) {
DEBUG_ASSERT(bufs);
for (auto &it : tianmu_fields_cache) {
Expand Down
1 change: 1 addition & 0 deletions storage/tianmu/core/mysql_expression.h
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ class MysqlExpression {
Item *GetItem() { return item; }
const tianmu_fields_cache_t &GetTIANMUItems() { return tianmu_fields_cache; }
bool IsDeterministic() { return deterministic; }
bool BaseOnUserValue();
/*! \brief Tests if other MysqlExpression is same as this one.
*
* \param other - equality to this MysqlExpression is being questioned.
Expand Down
2 changes: 2 additions & 0 deletions storage/tianmu/core/temp_table.h
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ class TempTable : public JustATable {
!term.vc->IsConst(); // constant value, the buffer is already
// filled in
}
bool BaseUserVar() { return term.vc->BaseUserVar(); }

enum phys_col_t ColType() const override { return phys_col_t::ATTR; }
//! Use in cases where actual string length is less than declared, before
Expand Down Expand Up @@ -387,6 +388,7 @@ class TempTable : public JustATable {
void MoveVC(vcolumn::VirtualColumn *vc, std::vector<vcolumn::VirtualColumn *> &from,
std::vector<vcolumn::VirtualColumn *> &to);
void FillbufferTask(Attr *attr, Transaction *txn, MIIterator *page_start, int64_t start_row, int64_t page_end);
void FillBufferByRow(std::vector<Attr*> attrs, MIIterator *page_start, int64_t start_row, int64_t page_end);
size_t TaskPutValueInST(MIIterator *it, Transaction *ci, SorterWrapper *st);
bool HasTempTable() const { return has_temp_table; }

Expand Down
33 changes: 31 additions & 2 deletions storage/tianmu/core/temp_table_low.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,7 @@ void TempTable::FillMaterializedBuffers(int64_t local_limit, int64_t local_offse
int64_t row = local_offset;
std::vector<char> skip_parafilloutput;
std::set<int> set_vcid;
std::vector<Attr*> attrs_fillbyrow;

for (auto &attr : attrs) {
// check if there is duplicated columns, mark skip flag for yes
Expand All @@ -367,6 +368,8 @@ void TempTable::FillMaterializedBuffers(int64_t local_limit, int64_t local_offse
skip = 1;
} else {
set_vcid.insert(attr->term.vc_id);
if (attr->BaseUserVar())
skip = 1;
}
}
skip_parafilloutput.push_back(skip);
Expand Down Expand Up @@ -418,9 +421,19 @@ void TempTable::FillMaterializedBuffers(int64_t local_limit, int64_t local_offse
}
res.get_all_with_except();

for (uint i = 1; i < attrs.size(); i++)
if (skip_parafilloutput[i])
for (uint i = 1; i < attrs.size(); i++) {
if (attrs[i]->BaseUserVar()) {
if (attrs[i]->NeedFill()) {
attrs_fillbyrow.push_back(attrs[i]);
}
} else if (skip_parafilloutput[i]) {
FillbufferTask(attrs[i], current_txn_, &page_start, start_row, page_end);
}
}
// if column base on user value, fill buffer by row order
if (!attrs_fillbyrow.empty()) {
FillBufferByRow(attrs_fillbyrow, &page_start, start_row, page_end);
}

if (lazy)
break;
Expand Down Expand Up @@ -573,6 +586,22 @@ void TempTable::FillbufferTask(Attr *attr, Transaction *txn, MIIterator *page_st
}
}

void TempTable::FillBufferByRow(std::vector<Attr*> attrs, MIIterator *page_start, int64_t start_row, int64_t page_end) {
MIIterator itr(*page_start);
int64_t n, cnt = page_end - start_row;
for (size_t n = 0; n < cnt; ++n) {
for (auto attr : attrs) {
if (itr.PackrowStarted() || n == 0) {
attr->term.vc->LockSourcePacks(itr);
}
attr->FillValue(itr, start_row + n);
}
++itr;
}
for (auto attr : attrs)
attr->term.vc->UnlockSourcePacks();
}

size_t TempTable::TaskPutValueInST(MIIterator *it, Transaction *ci, SorterWrapper *st) {
size_t local_row = 0;
bool continue_now = true;
Expand Down
6 changes: 5 additions & 1 deletion storage/tianmu/vc/expr_column.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ ExpressionColumn::ExpressionColumn(core::MysqlExpression *expr, core::TempTable
deterministic_(expr ? expr->IsDeterministic() : true) {
const std::vector<core::JustATable *> *tables = &temp_table->GetTables();
const std::vector<int> *aliases = &temp_table->GetAliases();

use_usr_var_ = false;
if (expr_) {
vars_ = expr_->GetVars(); // get all variables from complex term
first_eval_ = true;
Expand Down Expand Up @@ -77,6 +77,9 @@ ExpressionColumn::ExpressionColumn(core::MysqlExpression *expr, core::TempTable

// if (status == VC_EXPR && var_map_.size() == 0 )
// status = VC_CONST;

// if expr calculate base on user value set the value use_usr_val_ true
use_usr_var_ = expr_->BaseOnUserValue();
} else {
DEBUG_ASSERT(!"unexpected!!");
}
Expand All @@ -90,6 +93,7 @@ ExpressionColumn::ExpressionColumn(const ExpressionColumn &ec)
var_buf_(ec.var_buf_),
deterministic_(ec.deterministic_) {
var_map_ = ec.var_map_;
use_usr_var_ = ec.use_usr_var_;
}

void ExpressionColumn::SetParamTypes(core::MysqlExpression::TypOfVars *types) { expr_->EvalType(types); }
Expand Down
3 changes: 2 additions & 1 deletion storage/tianmu/vc/expr_column.h
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ class ExpressionColumn : public VirtualColumn {
}
core::MysqlExpression *expr_; //!= nullptr if ExpressionColumn encapsulates an expression. Note - a
//! constant is an expression

bool BaseUserVar() { return use_usr_var_; }
private:
/*! \brief Set variable buffers with values for a given iterator.
*
Expand All @@ -137,6 +137,7 @@ class ExpressionColumn : public VirtualColumn {
//! value for a given row is always the same or not? e.g. currenttime() is not
//! deterministic
bool deterministic_;
bool use_usr_var_;
};
} // namespace vcolumn
} // namespace Tianmu
Expand Down
2 changes: 1 addition & 1 deletion storage/tianmu/vc/virtual_column.h
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ class VirtualColumn : public VirtualColumnBase {

void LockSourcePacks(const core::MIIterator &mit) override { vc_pack_guard_.LockPackrow(mit); }
void UnlockSourcePacks() override { vc_pack_guard_.UnlockAll(); }

virtual bool BaseUserVar() { return false; }
private:
core::VCPackGuardian vc_pack_guard_;
};
Expand Down

0 comments on commit 20e13dc

Please sign in to comment.