From 26920b2b89a2a7b9a7ad219ba2a9d7542637962a Mon Sep 17 00:00:00 2001 From: Sage Griffin Date: Fri, 12 Jun 2026 10:44:08 -0600 Subject: [PATCH] Stop using `Vec::remove(0)` When using `pg_query::parse` on a very large query (for example `SELECT COUNT(*) FROM users WHERE users.id IN ({1.3M entries})`), pg_query would effectively hang indefinitely (5-6 minutes in production, never waited long enough on my local machine to get a measurement). Profiling the code showed 99.99% of the time was spent in `__memmove_avx_unaligned_erms`, called from `flatten` within `ParseResult::new`. Digging into `nodes`, it was clear why. These functions are operating by maintaining two lists, the remaining elements to iterate over, and the results to return. When an element is removed from the list to iterate over, it uses `Vec::remove(0)`, which must copy the entire remaining vector over each time. This results in a computational complexity of somewhere between `O(N^2)` and `O(N!)`. This commit changes the code to use `VecDeque` instead, which is built for exactly this purpose, and can pop from the front in constant time. Ideally these functions wouldn't allocate at all. Rather than returning `Vec`, they'd be much better off returning `impl Iterator` and yielding without allocation (especially since every caller of these internally just immediately iterates over the result). But that's a bigger rewrite, and would be a backwards incompatible change to the API. I am willing to implement that if such a change would be accepted. In the short term though, using `VecDeque` removes the degenerate performance case, and changes the performance from so slow I don't have a measurement, to 1.2s on my local machine which is somewhat in line with the Ruby implementation 975ms. --- src/node_enum.rs | 297 ++++++++++++++++++++++++----------------------- src/truncate.rs | 36 +++--- 2 files changed, 167 insertions(+), 166 deletions(-) diff --git a/src/node_enum.rs b/src/node_enum.rs index 96efe32..2e53b94 100644 --- a/src/node_enum.rs +++ b/src/node_enum.rs @@ -1,6 +1,7 @@ use crate::*; pub use protobuf::node::Node as NodeEnum; +use std::collections::VecDeque; #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, serde::Serialize)] pub enum Context { @@ -27,10 +28,10 @@ impl NodeEnum { } pub fn nodes(&self) -> Vec<(NodeRef<'_>, i32, Context, bool)> { - let mut iter = vec![(self.to_ref(), 0, Context::None, false)]; + let mut iter = VecDeque::new(); + iter.push_front((self.to_ref(), 0, Context::None, false)); let mut nodes = Vec::new(); - while !iter.is_empty() { - let (node, depth, context, has_filter_columns) = iter.remove(0); + while let Some((node, depth, context, has_filter_columns)) = iter.pop_front() { let depth = depth + 1; match node { // @@ -39,33 +40,33 @@ impl NodeEnum { NodeRef::SelectStmt(s) => { s.target_list.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select, false)); + iter.push_back((n.to_ref(), depth, Context::Select, false)); } }); if let Some(n) = &s.where_clause { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select, true)); + iter.push_back((n.to_ref(), depth, Context::Select, true)); } } s.sort_clause.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select, false)); + iter.push_back((n.to_ref(), depth, Context::Select, false)); } }); s.group_clause.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select, false)); + iter.push_back((n.to_ref(), depth, Context::Select, false)); } }); if let Some(n) = &s.having_clause { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select, false)); + iter.push_back((n.to_ref(), depth, Context::Select, false)); } } if let Some(clause) = &s.with_clause { clause.ctes.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select, false)); + iter.push_back((n.to_ref(), depth, Context::Select, false)); } }); } @@ -73,32 +74,32 @@ impl NodeEnum { Ok(protobuf::SetOperation::SetopNone) => { s.from_clause.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select, false)); + iter.push_back((n.to_ref(), depth, Context::Select, false)); } }); } Ok(protobuf::SetOperation::SetopUnion) => { if let Some(left) = s.larg.as_ref() { - iter.push((left.to_ref(), depth, Context::Select, false)); + iter.push_back((left.to_ref(), depth, Context::Select, false)); } if let Some(right) = s.rarg.as_ref() { - iter.push((right.to_ref(), depth, Context::Select, false)); + iter.push_back((right.to_ref(), depth, Context::Select, false)); } } Ok(protobuf::SetOperation::SetopExcept) => { if let Some(left) = s.larg.as_ref() { - iter.push((left.to_ref(), depth, Context::Select, false)); + iter.push_back((left.to_ref(), depth, Context::Select, false)); } if let Some(right) = s.rarg.as_ref() { - iter.push((right.to_ref(), depth, Context::Select, false)); + iter.push_back((right.to_ref(), depth, Context::Select, false)); } } Ok(protobuf::SetOperation::SetopIntersect) => { if let Some(left) = s.larg.as_ref() { - iter.push((left.to_ref(), depth, Context::Select, false)); + iter.push_back((left.to_ref(), depth, Context::Select, false)); } if let Some(right) = s.rarg.as_ref() { - iter.push((right.to_ref(), depth, Context::Select, false)); + iter.push_back((right.to_ref(), depth, Context::Select, false)); } } Ok(protobuf::SetOperation::Undefined) | Err(_) => (), @@ -107,46 +108,46 @@ impl NodeEnum { NodeRef::InsertStmt(s) => { if let Some(n) = &s.select_stmt { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML, false)); + iter.push_back((n.to_ref(), depth, Context::DML, false)); } } if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DML, false)); + iter.push_back((rel.to_ref(), depth, Context::DML, false)); } if let Some(clause) = &s.with_clause { clause.ctes.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML, false)); + iter.push_back((n.to_ref(), depth, Context::DML, false)); } }); } if let Some(n) = &s.on_conflict_clause { - iter.push((n.to_ref(), depth, Context::DML, false)); + iter.push_back((n.to_ref(), depth, Context::DML, false)); } } NodeRef::UpdateStmt(s) => { s.target_list.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML, false)); + iter.push_back((n.to_ref(), depth, Context::DML, false)); } }); s.where_clause.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML, true)); + iter.push_back((n.to_ref(), depth, Context::DML, true)); } }); s.from_clause.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select, false)); + iter.push_back((n.to_ref(), depth, Context::Select, false)); } }); if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DML, false)); + iter.push_back((rel.to_ref(), depth, Context::DML, false)); } if let Some(clause) = &s.with_clause { clause.ctes.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML, false)); + iter.push_back((n.to_ref(), depth, Context::DML, false)); } }); } @@ -154,74 +155,74 @@ impl NodeEnum { NodeRef::DeleteStmt(s) => { if let Some(n) = &s.where_clause { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML, true)); + iter.push_back((n.to_ref(), depth, Context::DML, true)); } } if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DML, false)); + iter.push_back((rel.to_ref(), depth, Context::DML, false)); } if let Some(clause) = &s.with_clause { clause.ctes.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML, false)); + iter.push_back((n.to_ref(), depth, Context::DML, false)); } }); } s.using_clause.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select, false)); + iter.push_back((n.to_ref(), depth, Context::Select, false)); } }); } NodeRef::MergeStmt(m) => { if let Some(t) = m.relation.as_ref() { - iter.push((t.to_ref(), depth, Context::DML, false)); + iter.push_back((t.to_ref(), depth, Context::DML, false)); } if let Some(clause) = &m.with_clause { clause.ctes.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML, false)); + iter.push_back((n.to_ref(), depth, Context::DML, false)); } }); } m.source_relation.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select, false)); + iter.push_back((n.to_ref(), depth, Context::Select, false)); } }); m.merge_when_clauses.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML, true)); + iter.push_back((n.to_ref(), depth, Context::DML, true)); } }); m.join_condition.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select, false)); + iter.push_back((n.to_ref(), depth, Context::Select, false)); } }); } NodeRef::CommonTableExpr(s) => { if let Some(n) = &s.ctequery { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, false)); + iter.push_back((n.to_ref(), depth, context, false)); } } } NodeRef::CopyStmt(s) => { if let Some(n) = &s.query { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML, false)); + iter.push_back((n.to_ref(), depth, Context::DML, false)); } } if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DML, false)); + iter.push_back((rel.to_ref(), depth, Context::DML, false)); } } NodeRef::CallStmt(s) => { if let Some(n) = s.funccall.as_ref() { - iter.push((n.to_ref(), depth, Context::Call, false)); + iter.push_back((n.to_ref(), depth, Context::Call, false)); } } // @@ -229,91 +230,91 @@ impl NodeEnum { // NodeRef::AlterTableStmt(s) => { if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL, false)); + iter.push_back((rel.to_ref(), depth, Context::DDL, false)); } } NodeRef::CreateStmt(s) => { if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL, false)); + iter.push_back((rel.to_ref(), depth, Context::DDL, false)); } } NodeRef::CreateTableAsStmt(s) => { if let Some(n) = &s.query { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DDL, false)); + iter.push_back((n.to_ref(), depth, Context::DDL, false)); } } if let Some(n) = &s.into { if let Some(rel) = n.rel.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL, false)); + iter.push_back((rel.to_ref(), depth, Context::DDL, false)); } } } NodeRef::TruncateStmt(s) => { s.relations.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DDL, false)); + iter.push_back((n.to_ref(), depth, Context::DDL, false)); } }); } NodeRef::ViewStmt(s) => { if let Some(n) = &s.query { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DDL, false)); + iter.push_back((n.to_ref(), depth, Context::DDL, false)); } } if let Some(rel) = s.view.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL, false)); + iter.push_back((rel.to_ref(), depth, Context::DDL, false)); } } NodeRef::IndexStmt(s) => { if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL, false)); + iter.push_back((rel.to_ref(), depth, Context::DDL, false)); } s.index_params.iter().for_each(|n| { if let Some(NodeEnum::IndexElem(n)) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DDL, false)); + iter.push_back((n.to_ref(), depth, Context::DDL, false)); if let Some(n) = n.expr.as_ref().and_then(|n| n.node.as_ref()) { - iter.push((n.to_ref(), depth, Context::DDL, false)); + iter.push_back((n.to_ref(), depth, Context::DDL, false)); } } }); if let Some(n) = s.where_clause.as_ref() { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DDL, true)); + iter.push_back((n.to_ref(), depth, Context::DDL, true)); } } } NodeRef::CreateTrigStmt(s) => { if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL, false)); + iter.push_back((rel.to_ref(), depth, Context::DDL, false)); } } NodeRef::RuleStmt(s) => { if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL, false)); + iter.push_back((rel.to_ref(), depth, Context::DDL, false)); } } NodeRef::VacuumStmt(s) => { for node in &s.rels { if let Some(NodeEnum::VacuumRelation(r)) = &node.node { if let Some(rel) = r.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL, false)); + iter.push_back((rel.to_ref(), depth, Context::DDL, false)); } } } } NodeRef::RefreshMatViewStmt(s) => { if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL, false)); + iter.push_back((rel.to_ref(), depth, Context::DDL, false)); } } NodeRef::GrantStmt(s) => { if let Ok(protobuf::ObjectType::ObjectTable) = protobuf::ObjectType::try_from(s.objtype) { s.objects.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DDL, false)); + iter.push_back((n.to_ref(), depth, Context::DDL, false)); } }); } @@ -321,14 +322,14 @@ impl NodeEnum { NodeRef::LockStmt(s) => { s.relations.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DDL, false)); + iter.push_back((n.to_ref(), depth, Context::DDL, false)); } }); } NodeRef::ExplainStmt(s) => { if let Some(n) = &s.query { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, false)); + iter.push_back((n.to_ref(), depth, context, false)); } } } @@ -338,106 +339,106 @@ impl NodeEnum { NodeRef::AExpr(e) => { if let Some(n) = &e.lexpr { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } } if let Some(n) = &e.rexpr { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::BoolExpr(e) => { e.args.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } }); } NodeRef::BooleanTest(e) => { if let Some(n) = &e.arg { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::CoalesceExpr(e) => { e.args.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } }); } NodeRef::MinMaxExpr(e) => { e.args.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } }); } NodeRef::NullTest(e) => { if let Some(n) = &e.arg { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::ResTarget(t) => { if let Some(n) = &t.val { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::SubLink(l) => { if let Some(n) = &l.subselect { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::FuncCall(c) => { c.args.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } }); } NodeRef::CaseExpr(c) => { c.args.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } }); if let Some(n) = &c.defresult { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::CaseWhen(w) => { if let Some(n) = &w.expr { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } } if let Some(n) = &w.result { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::SortBy(n) => { if let Some(n) = &n.node { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::TypeCast(n) => { if let Some(n) = &n.arg { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } } } @@ -447,7 +448,7 @@ impl NodeEnum { NodeRef::List(l) => { l.items.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } }); } @@ -455,7 +456,7 @@ impl NodeEnum { [&e.larg, &e.rarg, &e.quals].iter().for_each(|n| { if let Some(n) = n { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } } }); @@ -463,21 +464,21 @@ impl NodeEnum { NodeRef::RowExpr(e) => { e.args.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } }); } NodeRef::RangeSubselect(s) => { if let Some(n) = &s.subquery { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::RangeFunction(f) => { f.functions.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context, has_filter_columns)); + iter.push_back((n.to_ref(), depth, context, has_filter_columns)); } }); } @@ -495,10 +496,10 @@ impl NodeEnum { /// The caller may have to deal with dangling pointers, and passing an /// invalid tree back to libpg_query may cause it to panic. pub unsafe fn nodes_mut(&mut self) -> Vec<(NodeMut, i32, Context)> { - let mut iter = vec![(self.to_mut(), 0, Context::None)]; + let mut iter = VecDeque::new(); + iter.push_front((self.to_mut(), 0, Context::None)); let mut nodes = Vec::new(); - while !iter.is_empty() { - let (node, depth, context) = iter.remove(0); + while let Some((node, depth, context)) = iter.pop_front() { let depth = depth + 1; match node { // @@ -508,33 +509,33 @@ impl NodeEnum { let s = s.as_mut().unwrap(); s.target_list.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::Select)); + iter.push_back((n.to_mut(), depth, Context::Select)); } }); if let Some(n) = s.where_clause.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::Select)); + iter.push_back((n.to_mut(), depth, Context::Select)); } } s.sort_clause.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::Select)); + iter.push_back((n.to_mut(), depth, Context::Select)); } }); s.group_clause.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::Select)); + iter.push_back((n.to_mut(), depth, Context::Select)); } }); if let Some(n) = s.having_clause.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::Select)); + iter.push_back((n.to_mut(), depth, Context::Select)); } } if let Some(clause) = s.with_clause.as_mut() { clause.ctes.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::Select)); + iter.push_back((n.to_mut(), depth, Context::Select)); } }); } @@ -542,32 +543,32 @@ impl NodeEnum { Ok(protobuf::SetOperation::SetopNone) => { s.from_clause.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::Select)); + iter.push_back((n.to_mut(), depth, Context::Select)); } }); } Ok(protobuf::SetOperation::SetopUnion) => { if let Some(left) = s.larg.as_mut() { - iter.push((left.to_mut(), depth, Context::Select)); + iter.push_back((left.to_mut(), depth, Context::Select)); } if let Some(right) = s.rarg.as_mut() { - iter.push((right.to_mut(), depth, Context::Select)); + iter.push_back((right.to_mut(), depth, Context::Select)); } } Ok(protobuf::SetOperation::SetopExcept) => { if let Some(left) = s.larg.as_mut() { - iter.push((left.to_mut(), depth, Context::Select)); + iter.push_back((left.to_mut(), depth, Context::Select)); } if let Some(right) = s.rarg.as_mut() { - iter.push((right.to_mut(), depth, Context::Select)); + iter.push_back((right.to_mut(), depth, Context::Select)); } } Ok(protobuf::SetOperation::SetopIntersect) => { if let Some(left) = s.larg.as_mut() { - iter.push((left.to_mut(), depth, Context::Select)); + iter.push_back((left.to_mut(), depth, Context::Select)); } if let Some(right) = s.rarg.as_mut() { - iter.push((right.to_mut(), depth, Context::Select)); + iter.push_back((right.to_mut(), depth, Context::Select)); } } Ok(protobuf::SetOperation::Undefined) | Err(_) => (), @@ -577,47 +578,47 @@ impl NodeEnum { let s = s.as_mut().unwrap(); if let Some(n) = s.select_stmt.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::DML)); + iter.push_back((n.to_mut(), depth, Context::DML)); } } if let Some(rel) = s.relation.as_mut() { - iter.push((rel.to_mut(), depth, Context::DML)); + iter.push_back((rel.to_mut(), depth, Context::DML)); } if let Some(clause) = s.with_clause.as_mut() { clause.ctes.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::DML)); + iter.push_back((n.to_mut(), depth, Context::DML)); } }); } if let Some(n) = s.on_conflict_clause.as_mut() { - iter.push((n.to_mut(), depth, Context::DML)); + iter.push_back((n.to_mut(), depth, Context::DML)); } } NodeMut::UpdateStmt(s) => { let s = s.as_mut().unwrap(); s.target_list.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::DML)); + iter.push_back((n.to_mut(), depth, Context::DML)); } }); s.where_clause.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::DML)); + iter.push_back((n.to_mut(), depth, Context::DML)); } }); s.from_clause.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::Select)); + iter.push_back((n.to_mut(), depth, Context::Select)); } }); if let Some(rel) = s.relation.as_mut() { - iter.push((rel.to_mut(), depth, Context::DML)); + iter.push_back((rel.to_mut(), depth, Context::DML)); } if let Some(clause) = s.with_clause.as_mut() { clause.ctes.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::DML)); + iter.push_back((n.to_mut(), depth, Context::DML)); } }); } @@ -626,22 +627,22 @@ impl NodeEnum { let s = s.as_mut().unwrap(); if let Some(n) = s.where_clause.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::DML)); + iter.push_back((n.to_mut(), depth, Context::DML)); } } if let Some(rel) = s.relation.as_mut() { - iter.push((rel.to_mut(), depth, Context::DML)); + iter.push_back((rel.to_mut(), depth, Context::DML)); } if let Some(clause) = s.with_clause.as_mut() { clause.ctes.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::DML)); + iter.push_back((n.to_mut(), depth, Context::DML)); } }); } s.using_clause.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::Select)); + iter.push_back((n.to_mut(), depth, Context::Select)); } }); } @@ -649,7 +650,7 @@ impl NodeEnum { let s = s.as_mut().unwrap(); if let Some(n) = s.ctequery.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } } @@ -657,17 +658,17 @@ impl NodeEnum { let s = s.as_mut().unwrap(); if let Some(n) = s.query.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::DML)); + iter.push_back((n.to_mut(), depth, Context::DML)); } } if let Some(rel) = s.relation.as_mut() { - iter.push((rel.to_mut(), depth, Context::DML)); + iter.push_back((rel.to_mut(), depth, Context::DML)); } } NodeMut::CallStmt(s) => { let s = s.as_mut().unwrap(); if let Some(n) = s.funccall.as_mut() { - iter.push((n.to_mut(), depth, Context::Call)); + iter.push_back((n.to_mut(), depth, Context::Call)); } } // @@ -676,25 +677,25 @@ impl NodeEnum { NodeMut::AlterTableStmt(s) => { let s = s.as_mut().unwrap(); if let Some(rel) = s.relation.as_mut() { - iter.push((rel.to_mut(), depth, Context::DDL)); + iter.push_back((rel.to_mut(), depth, Context::DDL)); } } NodeMut::CreateStmt(s) => { let s = s.as_mut().unwrap(); if let Some(rel) = s.relation.as_mut() { - iter.push((rel.to_mut(), depth, Context::DDL)); + iter.push_back((rel.to_mut(), depth, Context::DDL)); } } NodeMut::CreateTableAsStmt(s) => { let s = s.as_mut().unwrap(); if let Some(n) = s.query.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::DDL)); + iter.push_back((n.to_mut(), depth, Context::DDL)); } } if let Some(n) = s.into.as_mut() { if let Some(rel) = n.rel.as_mut() { - iter.push((rel.to_mut(), depth, Context::DDL)); + iter.push_back((rel.to_mut(), depth, Context::DDL)); } } } @@ -702,7 +703,7 @@ impl NodeEnum { let s = s.as_mut().unwrap(); s.relations.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::DDL)); + iter.push_back((n.to_mut(), depth, Context::DDL)); } }); } @@ -710,22 +711,22 @@ impl NodeEnum { let s = s.as_mut().unwrap(); if let Some(n) = s.query.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::DDL)); + iter.push_back((n.to_mut(), depth, Context::DDL)); } } if let Some(rel) = s.view.as_mut() { - iter.push((rel.to_mut(), depth, Context::DDL)); + iter.push_back((rel.to_mut(), depth, Context::DDL)); } } NodeMut::IndexStmt(s) => { let s = s.as_mut().unwrap(); if let Some(rel) = s.relation.as_mut() { - iter.push((rel.to_mut(), depth, Context::DDL)); + iter.push_back((rel.to_mut(), depth, Context::DDL)); } s.index_params.iter_mut().for_each(|n| { if let Some(NodeEnum::IndexElem(n)) = n.node.as_mut() { if let Some(n) = n.expr.as_mut().and_then(|n| n.node.as_mut()) { - iter.push((n.to_mut(), depth, Context::DDL)); + iter.push_back((n.to_mut(), depth, Context::DDL)); } } }); @@ -733,13 +734,13 @@ impl NodeEnum { NodeMut::CreateTrigStmt(s) => { let s = s.as_mut().unwrap(); if let Some(rel) = s.relation.as_mut() { - iter.push((rel.to_mut(), depth, Context::DDL)); + iter.push_back((rel.to_mut(), depth, Context::DDL)); } } NodeMut::RuleStmt(s) => { let s = s.as_mut().unwrap(); if let Some(rel) = s.relation.as_mut() { - iter.push((rel.to_mut(), depth, Context::DDL)); + iter.push_back((rel.to_mut(), depth, Context::DDL)); } } NodeMut::VacuumStmt(s) => { @@ -747,7 +748,7 @@ impl NodeEnum { for node in s.rels.iter_mut() { if let Some(NodeEnum::VacuumRelation(r)) = node.node.as_mut() { if let Some(rel) = r.relation.as_mut() { - iter.push((rel.to_mut(), depth, Context::DDL)); + iter.push_back((rel.to_mut(), depth, Context::DDL)); } } } @@ -755,7 +756,7 @@ impl NodeEnum { NodeMut::RefreshMatViewStmt(s) => { let s = s.as_mut().unwrap(); if let Some(rel) = s.relation.as_mut() { - iter.push((rel.to_mut(), depth, Context::DDL)); + iter.push_back((rel.to_mut(), depth, Context::DDL)); } } NodeMut::GrantStmt(s) => { @@ -763,7 +764,7 @@ impl NodeEnum { if let Ok(protobuf::ObjectType::ObjectTable) = protobuf::ObjectType::try_from(s.objtype) { s.objects.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::DDL)); + iter.push_back((n.to_mut(), depth, Context::DDL)); } }); } @@ -772,7 +773,7 @@ impl NodeEnum { let s = s.as_mut().unwrap(); s.relations.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, Context::DDL)); + iter.push_back((n.to_mut(), depth, Context::DDL)); } }); } @@ -780,7 +781,7 @@ impl NodeEnum { let s = s.as_mut().unwrap(); if let Some(n) = s.query.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } } @@ -791,12 +792,12 @@ impl NodeEnum { let e = e.as_mut().unwrap(); if let Some(n) = e.lexpr.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } if let Some(n) = e.rexpr.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } } @@ -804,7 +805,7 @@ impl NodeEnum { let e = e.as_mut().unwrap(); e.args.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } }); } @@ -812,7 +813,7 @@ impl NodeEnum { let e = e.as_mut().unwrap(); e.args.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } }); } @@ -820,7 +821,7 @@ impl NodeEnum { let e = e.as_mut().unwrap(); e.args.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } }); } @@ -828,7 +829,7 @@ impl NodeEnum { let e = e.as_mut().unwrap(); if let Some(n) = e.arg.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } } @@ -836,7 +837,7 @@ impl NodeEnum { let t = t.as_mut().unwrap(); if let Some(n) = t.val.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } } @@ -844,7 +845,7 @@ impl NodeEnum { let l = l.as_mut().unwrap(); if let Some(n) = l.subselect.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } } @@ -852,7 +853,7 @@ impl NodeEnum { let c = c.as_mut().unwrap(); c.args.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } }); } @@ -860,12 +861,12 @@ impl NodeEnum { let c = c.as_mut().unwrap(); c.args.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } }); if let Some(n) = c.defresult.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } } @@ -873,12 +874,12 @@ impl NodeEnum { let w = w.as_mut().unwrap(); if let Some(n) = w.expr.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } if let Some(n) = w.result.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } } @@ -886,7 +887,7 @@ impl NodeEnum { let n = n.as_mut().unwrap(); if let Some(n) = n.node.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } } @@ -894,7 +895,7 @@ impl NodeEnum { let t = t.as_mut().unwrap(); if let Some(n) = t.arg.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } } @@ -905,7 +906,7 @@ impl NodeEnum { let l = l.as_mut().unwrap(); l.items.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } }); } @@ -913,17 +914,17 @@ impl NodeEnum { let e = e.as_mut().unwrap(); if let Some(n) = e.larg.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } if let Some(n) = e.rarg.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } if let Some(n) = e.quals.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } } @@ -931,7 +932,7 @@ impl NodeEnum { let e = e.as_mut().unwrap(); e.args.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } }); } @@ -939,7 +940,7 @@ impl NodeEnum { let s = s.as_mut().unwrap(); if let Some(n) = s.subquery.as_mut() { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } } } @@ -947,7 +948,7 @@ impl NodeEnum { let f = f.as_mut().unwrap(); f.functions.iter_mut().for_each(|n| { if let Some(n) = n.node.as_mut() { - iter.push((n.to_mut(), depth, context)); + iter.push_back((n.to_mut(), depth, context)); } }); } diff --git a/src/truncate.rs b/src/truncate.rs index dde6ce4..7df93da 100644 --- a/src/truncate.rs +++ b/src/truncate.rs @@ -1,4 +1,5 @@ use std::cmp::Ordering; +use std::collections::VecDeque; use crate::*; @@ -35,13 +36,13 @@ pub fn truncate(protobuf: &protobuf::ParseResult, max_length: usize) -> Result = Vec::new(); + let mut truncations: VecDeque = VecDeque::new(); for (node, depth, _context) in protobuf.nodes_mut().into_iter() { match node { NodeMut::SelectStmt(s) => { let s = s.as_mut().ok_or(Error::InvalidPointer)?; if !s.target_list.is_empty() { - truncations.push(PossibleTruncation { + truncations.push_back(PossibleTruncation { attr: TruncationAttr::TargetList, node, depth, @@ -49,7 +50,7 @@ pub fn truncate(protobuf: &protobuf::ParseResult, max_length: usize) -> Result Result Result { let s = s.as_mut().ok_or(Error::InvalidPointer)?; if !s.target_list.is_empty() { - truncations.push(PossibleTruncation { + truncations.push_back(PossibleTruncation { attr: TruncationAttr::TargetList, node, depth, @@ -76,7 +77,7 @@ pub fn truncate(protobuf: &protobuf::ParseResult, max_length: usize) -> Result Result { let s = s.as_mut().ok_or(Error::InvalidPointer)?; if let Some(clause) = s.where_clause.as_ref() { - truncations.push(PossibleTruncation { + truncations.push_back(PossibleTruncation { attr: TruncationAttr::WhereClause, node, depth, @@ -98,7 +99,7 @@ pub fn truncate(protobuf: &protobuf::ParseResult, max_length: usize) -> Result { let s = s.as_mut().ok_or(Error::InvalidPointer)?; if let Some(clause) = s.where_clause.as_ref() { - truncations.push(PossibleTruncation { + truncations.push_back(PossibleTruncation { attr: TruncationAttr::WhereClause, node, depth, @@ -109,13 +110,13 @@ pub fn truncate(protobuf: &protobuf::ParseResult, max_length: usize) -> Result { let s = s.as_mut().ok_or(Error::InvalidPointer)?; if !s.cols.is_empty() { - truncations.push(PossibleTruncation { attr: TruncationAttr::Cols, node, depth, length: cols_len(s.cols.clone())? }); + truncations.push_back(PossibleTruncation { attr: TruncationAttr::Cols, node, depth, length: cols_len(s.cols.clone())? }); } } NodeMut::IndexStmt(s) => { let s = s.as_mut().ok_or(Error::InvalidPointer)?; if let Some(clause) = s.where_clause.as_ref() { - truncations.push(PossibleTruncation { + truncations.push_back(PossibleTruncation { attr: TruncationAttr::WhereClause, node, depth, @@ -126,7 +127,7 @@ pub fn truncate(protobuf: &protobuf::ParseResult, max_length: usize) -> Result { let s = s.as_mut().ok_or(Error::InvalidPointer)?; if let Some(clause) = s.where_clause.as_ref() { - truncations.push(PossibleTruncation { + truncations.push_back(PossibleTruncation { attr: TruncationAttr::WhereClause, node, depth, @@ -137,7 +138,7 @@ pub fn truncate(protobuf: &protobuf::ParseResult, max_length: usize) -> Result { let s = s.as_mut().ok_or(Error::InvalidPointer)?; if let Some(cte) = s.ctequery.as_ref() { - truncations.push(PossibleTruncation { + truncations.push_back(PossibleTruncation { attr: TruncationAttr::CTEQuery, node, depth: depth + 1, @@ -148,7 +149,7 @@ pub fn truncate(protobuf: &protobuf::ParseResult, max_length: usize) -> Result { let s = s.as_mut().ok_or(Error::InvalidPointer)?; if let Some(clause) = s.where_clause.as_ref() { - truncations.push(PossibleTruncation { + truncations.push_back(PossibleTruncation { attr: TruncationAttr::WhereClause, node, depth, @@ -159,7 +160,7 @@ pub fn truncate(protobuf: &protobuf::ParseResult, max_length: usize) -> Result { let s = s.as_mut().ok_or(Error::InvalidPointer)?; if !s.target_list.is_empty() { - truncations.push(PossibleTruncation { + truncations.push_back(PossibleTruncation { attr: TruncationAttr::TargetList, node, depth, @@ -167,7 +168,7 @@ pub fn truncate(protobuf: &protobuf::ParseResult, max_length: usize) -> Result Result a.length.cmp(&b.length).reverse(), other => other, }); - while !truncations.is_empty() { - let truncation = truncations.remove(0); + while let Some(truncation) = truncations.pop_front() { match (truncation.node, truncation.attr) { (NodeMut::SelectStmt(s), TruncationAttr::TargetList) => { let s = s.as_mut().ok_or(Error::InvalidPointer)?;