Skip to content

Commit bf37f05

Browse files
kitaisrealkitaisreal
committed
Analyzer log used row policies
1 parent c391527 commit bf37f05

8 files changed

Lines changed: 38 additions & 10 deletions

File tree

‎src/Analyzer/QueryTreePassManager.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,6 @@ class ValidationChecker : public InDepthQueryTreeVisitor<ValidationChecker>
166166
* TODO: Support GROUP BY injective function elimination.
167167
* TODO: Support setting optimize_move_functions_out_of_any.
168168
* TODO: Support setting optimize_aggregators_of_group_by_keys.
169-
* TODO: Support setting optimize_duplicate_order_by_and_distinct.
170169
* TODO: Support setting optimize_monotonous_functions_in_order_by.
171170
* TODO: Add optimizations based on function semantics. Example: SELECT * FROM test_table WHERE id != id. (id is not nullable column).
172171
*/

‎src/Interpreters/InterpreterSelectQueryAnalyzer.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,4 +257,10 @@ void InterpreterSelectQueryAnalyzer::addStorageLimits(const StorageLimitsList &
257257
planner.addStorageLimits(storage_limits);
258258
}
259259

260+
void InterpreterSelectQueryAnalyzer::extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr & /*ast*/, ContextPtr /*context*/) const
261+
{
262+
for (const auto & used_row_policy : planner.getUsedRowPolicies())
263+
elem.used_row_policies.emplace(used_row_policy);
264+
}
265+
260266
}

‎src/Interpreters/InterpreterSelectQueryAnalyzer.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,14 +57,16 @@ class InterpreterSelectQueryAnalyzer : public IInterpreter
5757

5858
void addStorageLimits(const StorageLimitsList & storage_limits);
5959

60+
void extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr & /*ast*/, ContextPtr /*context*/) const override;
61+
6062
bool supportsTransactions() const override { return true; }
6163

6264
bool ignoreLimits() const override { return select_query_options.ignore_limits; }
6365

6466
bool ignoreQuota() const override { return select_query_options.ignore_quota; }
6567

66-
6768
const Planner & getPlanner() const { return planner; }
69+
6870
Planner & getPlanner() { return planner; }
6971

7072
const QueryTreeNodePtr & getQueryTree() const { return query_tree; }

‎src/Planner/Planner.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1201,6 +1201,8 @@ void Planner::buildPlanForUnionNode()
12011201
{
12021202
Planner query_planner(query_node, select_query_options);
12031203
query_planner.buildQueryPlanIfNeeded();
1204+
for (const auto & row_policy : query_planner.getUsedRowPolicies())
1205+
used_row_policies.insert(row_policy);
12041206
auto query_node_plan = std::make_unique<QueryPlan>(std::move(query_planner).extractQueryPlan());
12051207
query_plans_headers.push_back(query_node_plan->getCurrentDataStream().header);
12061208
query_plans.push_back(std::move(query_node_plan));
@@ -1348,8 +1350,10 @@ void Planner::buildPlanForQueryNode()
13481350
select_query_options,
13491351
top_level_identifiers,
13501352
planner_context);
1353+
13511354
auto from_stage = join_tree_query_plan.from_stage;
13521355
query_plan = std::move(join_tree_query_plan.query_plan);
1356+
used_row_policies = std::move(join_tree_query_plan.used_row_policies);
13531357

13541358
LOG_TRACE(&Poco::Logger::get("Planner"), "Query {} from stage {} to stage {}{}",
13551359
query_tree->formatConvertedASTForErrorMessage(),

‎src/Planner/Planner.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ class Planner
4444
return query_plan;
4545
}
4646

47+
const std::set<std::string> & getUsedRowPolicies() const
48+
{
49+
return used_row_policies;
50+
}
51+
4752
void buildQueryPlanIfNeeded();
4853

4954
QueryPlan && extractQueryPlan() &&
@@ -70,6 +75,7 @@ class Planner
7075
PlannerContextPtr planner_context;
7176
QueryPlan query_plan;
7277
StorageLimitsList storage_limits;
78+
std::set<std::string> used_row_policies;
7379
};
7480

7581
}

‎src/Planner/PlannerJoinTree.cpp‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,8 @@ bool shouldIgnoreQuotaAndLimits(const TableNode & table_node)
126126
return false;
127127
if (storage_id.database_name == DatabaseCatalog::SYSTEM_DATABASE)
128128
{
129-
static const boost::container::flat_set<String> tables_ignoring_quota{"quotas", "quota_limits", "quota_usage", "quotas_usage", "one"};
130-
if (tables_ignoring_quota.count(storage_id.table_name))
129+
static const boost::container::flat_set<std::string_view> tables_ignoring_quota{"quotas", "quota_limits", "quota_usage", "quotas_usage", "one"};
130+
if (tables_ignoring_quota.contains(storage_id.table_name))
131131
return true;
132132
}
133133
return false;
@@ -441,7 +441,8 @@ void updatePrewhereOutputsIfNeeded(SelectQueryInfo & table_expression_query_info
441441

442442
FilterDAGInfo buildRowPolicyFilterIfNeeded(const StoragePtr & storage,
443443
SelectQueryInfo & table_expression_query_info,
444-
PlannerContextPtr & planner_context)
444+
PlannerContextPtr & planner_context,
445+
std::set<std::string> & used_row_policies)
445446
{
446447
auto storage_id = storage->getStorageID();
447448
const auto & query_context = planner_context->getQueryContext();
@@ -450,6 +451,12 @@ FilterDAGInfo buildRowPolicyFilterIfNeeded(const StoragePtr & storage,
450451
if (!row_policy_filter || row_policy_filter->empty())
451452
return {};
452453

454+
for (const auto & row_policy : row_policy_filter->policies)
455+
{
456+
auto name = row_policy->getFullName().toString();
457+
used_row_policies.emplace(std::move(name));
458+
}
459+
453460
return buildFilterInfo(row_policy_filter->expression, table_expression_query_info.table_expression, planner_context);
454461
}
455462

@@ -586,6 +593,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
586593
auto * union_node = table_expression->as<UnionNode>();
587594

588595
QueryPlan query_plan;
596+
std::set<std::string> used_row_policies;
589597

590598
if (table_node || table_function_node)
591599
{
@@ -781,7 +789,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
781789
}
782790
};
783791

784-
auto row_policy_filter_info = buildRowPolicyFilterIfNeeded(storage, table_expression_query_info, planner_context);
792+
auto row_policy_filter_info = buildRowPolicyFilterIfNeeded(storage, table_expression_query_info, planner_context, used_row_policies);
785793
add_filter(row_policy_filter_info, "Row-level security filter");
786794
if (row_policy_filter_info.actions)
787795
table_expression_data.setRowLevelFilterActions(row_policy_filter_info.actions);
@@ -940,7 +948,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
940948
}
941949
}
942950

943-
return {std::move(query_plan), from_stage};
951+
return {std::move(query_plan), from_stage, std::move(used_row_policies)};
944952
}
945953

946954
JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_expression,
@@ -1399,7 +1407,10 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
13991407
drop_unused_columns_after_join_transform_step->setStepDescription("DROP unused columns after JOIN");
14001408
result_plan.addStep(std::move(drop_unused_columns_after_join_transform_step));
14011409

1402-
return {std::move(result_plan), QueryProcessingStage::FetchColumns};
1410+
for (const auto & right_join_tree_query_plan_row_policy : right_join_tree_query_plan.used_row_policies)
1411+
left_join_tree_query_plan.used_row_policies.insert(right_join_tree_query_plan_row_policy);
1412+
1413+
return {std::move(result_plan), QueryProcessingStage::FetchColumns, std::move(left_join_tree_query_plan.used_row_policies)};
14031414
}
14041415

14051416
JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_join_table_expression,
@@ -1477,7 +1488,7 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_
14771488
array_join_step->setStepDescription("ARRAY JOIN");
14781489
plan.addStep(std::move(array_join_step));
14791490

1480-
return {std::move(plan), QueryProcessingStage::FetchColumns};
1491+
return {std::move(plan), QueryProcessingStage::FetchColumns, std::move(join_tree_query_plan.used_row_policies)};
14811492
}
14821493

14831494
}

‎src/Planner/PlannerJoinTree.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ struct JoinTreeQueryPlan
1515
{
1616
QueryPlan query_plan;
1717
QueryProcessingStage::Enum from_stage;
18+
std::set<std::string> used_row_policies;
1819
};
1920

2021
/// Build JOIN TREE query plan for query node

‎tests/analyzer_tech_debt.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
01925_test_storage_merge_aliases
3030
01947_mv_subquery
3131
01952_optimize_distributed_group_by_sharding_key
32-
02131_used_row_policies_in_query_log
3332
02139_MV_with_scalar_subquery
3433
02174_cte_scalar_cache_mv
3534
02302_s3_file_pruning

0 commit comments

Comments
 (0)