diff --git a/interactive/examples/ddir.rs b/interactive/examples/ddir.rs index a706bd804..576c176be 100644 --- a/interactive/examples/ddir.rs +++ b/interactive/examples/ddir.rs @@ -14,6 +14,9 @@ //! - `--diag`: serve timely/DD diagnostics on port 51371. //! - `--backend=vec|corgi`: rendering substrate (default `vec`). The corgi //! backend is single-worker (its arrange does not exchange). +//! - `--sync=K`: await completion only every K rounds (default 1), letting K +//! timestamps retire with whatever inter-timestamp concurrency the system +//! finds — the open(er)-loop regime DD adapts into under load. use mimalloc::MiMalloc; @@ -37,6 +40,7 @@ struct Flags { debug_demand: bool, diag: bool, corgi: bool, + sync: u64, } fn run( @@ -184,7 +188,10 @@ fn run( cursor += 1; } for i in inputs.iter_mut() { i.advance_to(time); i.flush(); } - while probe.less_than(&time) { worker.step(); } + let sync = flags.sync.max(1); + if (round + 1) % sync == 0 || round + 1 == limit { + while probe.less_than(&time) { worker.step(); } + } round += 1; if round % 100 == 0 { @@ -209,6 +216,7 @@ fn main() { else if let Some(q) = a.strip_prefix("--query=") { flags.query = Some(q.to_string()); } else if a == "--debug-demand" { flags.debug_demand = true; } else if a == "--diag" { flags.diag = true; } + else if let Some(k) = a.strip_prefix("--sync=") { flags.sync = k.parse().expect("--sync=K"); } else if let Some(b) = a.strip_prefix("--backend=") { flags.corgi = match b { "corgi" => true, "vec" => false, other => panic!("unknown backend {other:?} (vec|corgi)") }; } diff --git a/interactive/examples/ddir_col.rs.disabled b/interactive/examples/ddir_col.rs.disabled deleted file mode 100644 index de4162508..000000000 --- a/interactive/examples/ddir_col.rs.disabled +++ /dev/null @@ -1,138 +0,0 @@ -//! DD IR columnar driver: parse, lower, render (via `interactive::backend::col`), execute. - -use mimalloc::MiMalloc; - -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - -use differential_dataflow::dynamic::pointstamp::PointStamp; - -use interactive::parse; -use interactive::lower; -use interactive::backend::col::{render_tree, Row, Diff}; - -type DdirOuterUpdate = (Row, Row, u64, Diff); - -fn run(name: &str, stmts: Vec, n_inputs: usize, nodes: u64, edges: u64, arity: usize, batch: u64, rounds: Option) { - let mut tree = lower::lower_tree(stmts); - let ops_before = tree.op_count(); - tree.optimize(); - let tree_export_idx = tree.root.exports.iter().position(|e| e.name == "result").unwrap_or(0); - println!("{}: {} ops before optimize, {} after; driving export {:?}", - name, ops_before, tree.op_count(), tree.root.exports[tree_export_idx].name); - let name = name.to_string(); - - timely::execute_from_args(std::env::args().skip(4), move |worker| { - use timely::dataflow::InputHandle; - use timely::container::PushInto; - use differential_dataflow::columnar::ValColBuilder; - - type OuterBuilder = ValColBuilder; - - let (mut inputs, probe) = worker.dataflow::(|scope| { - let mut handles = Vec::new(); - let mut collections = Vec::new(); - for _ in 0..n_inputs { - let mut h = >::new_with_builder(); - let stream = h.to_stream(scope); - handles.push(h); - collections.push(differential_dataflow::Collection::new(stream)); - } - let mut probe = timely::dataflow::ProbeHandle::new(); - let output = scope.iterative::, _, _>(|inner| { - let entered: Vec<_> = collections.iter().map(|c| c.clone().enter(inner)).collect(); - let root_imports: Vec<_> = tree.root.imports.iter().map(|imp| match &imp.from { - interactive::scope_ir::Source::Input(n) => entered[*n].clone(), - interactive::scope_ir::Source::Trace(name) => panic!("ddir_col: Import {:?} not supported in this harness (no trace registry).", name), - interactive::scope_ir::Source::Parent(_) => unreachable!("root scope cannot import from a parent"), - }).collect(); - let exports = render_tree(&tree.root, inner, 0, root_imports); - exports[tree_export_idx].clone().leave(scope) - }); - output.probe_with(&mut probe); - (handles, probe) - }); - - let index = worker.index(); - let peers = worker.peers(); - - let mut builders: Vec = (0..n_inputs).map(|_| OuterBuilder::default()).collect(); - - let timer = std::time::Instant::now(); - let timer_load = std::time::Instant::now(); - for e in 0..edges { - if (e as usize) % peers == index { - let input_idx = (e as usize) % inputs.len(); - let (key, val) = interactive::gen_row::(e, nodes, arity); - let time = *inputs[input_idx].time(); - builders[input_idx].push_into((key, val, time, 1i64)); - } - } - for (i, h) in inputs.iter_mut().enumerate() { - use timely::container::ContainerBuilder; - while let Some(container) = builders[i].finish() { h.send_batch(container); } - h.advance_to(1); - h.flush(); - } - while probe.less_than(&1u64) { worker.step(); } - println!("worker {}: {} loaded ({} edges, total {:.2?}, load {:.2?})", index, name, edges, timer.elapsed(), timer_load.elapsed()); - - let mut cursor = 0u64; - let mut round = 0u64; - let limit = rounds.unwrap_or(u64::MAX); - while round < limit { - let timer_round = std::time::Instant::now(); - let time = (round + 2) as u64; - for _ in 0..batch { - let remove_idx = cursor; - let add_idx = edges + cursor; - if (remove_idx as usize) % peers == index { - let input_idx = (remove_idx as usize) % inputs.len(); - let (key, val) = interactive::gen_row::(remove_idx, nodes, arity); - builders[input_idx].push_into((key, val, time, -1i64)); - } - if (add_idx as usize) % peers == index { - let input_idx = (add_idx as usize) % inputs.len(); - let (key, val) = interactive::gen_row::(add_idx, nodes, arity); - builders[input_idx].push_into((key, val, time, 1i64)); - } - cursor += 1; - } - for (i, h) in inputs.iter_mut().enumerate() { - use timely::container::ContainerBuilder; - while let Some(container) = builders[i].finish() { h.send_batch(container); } - h.advance_to(time); - h.flush(); - } - while probe.less_than(&time) { worker.step(); } - - round += 1; - if round % 100 == 0 { - println!("worker {}: {} round {} (total {:.2?}, round {:.2?})", index, name, round, timer.elapsed(), timer_round.elapsed()); - } - } - println!("worker {}: {} done ({} rounds, batch {}, total {:.2?})", index, name, round, batch, timer.elapsed()); - }).unwrap(); -} - -fn main() { - let program = std::env::args().nth(1).unwrap_or_else(|| { std::process::exit(0); }); - let arity: usize = std::env::args().nth(2).unwrap_or("2".into()).parse().unwrap(); - let nodes: u64 = std::env::args().nth(3).unwrap_or("10".into()).parse().unwrap(); - let edges: u64 = std::env::args().nth(4).unwrap_or_else(|| (2 * nodes).to_string()).parse().unwrap(); - let batch: u64 = std::env::args().nth(5).unwrap_or("1".into()).parse().unwrap(); - let rounds: Option = std::env::args().nth(6).map(|s| s.parse().unwrap()); - - let source = interactive::load_program(&program); - let stmts = if program.ends_with(".ddp") { - parse::pipe::parse(&source) - } else { - parse::applicative::parse(&source) - }; - let (n_inputs, imports) = interactive::survey_sources(&stmts); - if !imports.is_empty() { - panic!("ddir_col: program references imports {:?} but this harness has no trace registry.", imports); - } - let name = std::path::Path::new(&program).file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or(program.clone()); - run(&name, stmts, n_inputs, nodes, edges, arity, batch, rounds); -} diff --git a/interactive/src/backend/col.rs b/interactive/src/backend/col.rs deleted file mode 100644 index 1948c55d1..000000000 --- a/interactive/src/backend/col.rs +++ /dev/null @@ -1,259 +0,0 @@ -//! Columnar rendering substrate. -//! -//! Rows are a stride-encoded `Row(Vec)`; the differential container is -//! columnar `RecordedUpdates`. Supplies the substrate leaf operators (join, -//! reduce, arrange over columnar builders/batchers/spines); the scope-tree walk -//! lives in [`crate::backend::render_tree`]. - -mod types { - /// A row type backed by Vec but using Strides for columnar bounds. - /// This ensures uniform-length rows (common in the IR) get compact - /// stride-based offset encoding rather than per-element u64 bounds. - #[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] - pub struct Row(pub Vec); - - impl Row { - pub fn new() -> Self { Row(Vec::new()) } - pub fn push(&mut self, v: i64) { self.0.push(v); } - } - - impl std::ops::Deref for Row { - type Target = [i64]; - fn deref(&self) -> &[i64] { &self.0 } - } - - impl std::iter::FromIterator for Row { - fn from_iter>(iter: I) -> Self { - Row(iter.into_iter().collect()) - } - } - - impl<'a> IntoIterator for &'a Row { - type Item = &'a i64; - type IntoIter = std::slice::Iter<'a, i64>; - fn into_iter(self) -> Self::IntoIter { self.0.iter() } - } - - impl IntoIterator for Row { - type Item = i64; - type IntoIter = std::vec::IntoIter; - fn into_iter(self) -> Self::IntoIter { self.0.into_iter() } - } - - impl columnar::Columnar for Row { - type Container = columnar::Vecs, columnar::primitive::offsets::Strides>; - - fn into_owned<'a>(other: columnar::Ref<'a, Self>) -> Self { - Row(other.into_iter().copied().collect()) - } - - fn copy_from<'a>(&mut self, other: columnar::Ref<'a, Self>) { - self.0.clear(); - self.0.extend(other.into_iter().copied()); - } - } - - impl crate::ir::RowLike for Row { - fn new() -> Self { Row::new() } - fn push(&mut self, v: i64) { Row::push(self, v); } - fn as_slice(&self) -> &[i64] { &self.0 } - fn extend_from_slice(&mut self, other: &[i64]) { self.0.extend_from_slice(other); } - } - - pub type Diff = i64; - pub type Time = timely::order::Product>; -} - -use differential_dataflow::columnar as columnar_support; - -mod columnar { - use super::types::*; - - pub use super::columnar_support::*; - pub use super::columnar_support::{ValSpine, ValBatcher, ValBuilder}; - - pub type DdirUpdate = (Row, Row, Time, Diff); - pub type DdirRecordedUpdates = RecordedUpdates; - - pub type ColValSpine = ValSpine; - pub type ColValBatcher = ValBatcher; - pub type ColValBuilder = ValBuilder; - pub type ColValChunker = ValChunker; -} - -mod render { - use std::sync::Arc; - use timely::order::Product; - use differential_dataflow::Collection; - use differential_dataflow::dynamic::pointstamp::PointStamp; - use differential_dataflow::operators::arrange::{Arranged, TraceAgent}; - use columnar::Columnar; - use super::types::*; - use crate::ir::{LinearOp, RowLike, eval_fields, eval_field_into, eval_condition}; - use crate::parse::{Projection, Reducer}; - use crate::backend::Backend; - - use super::columnar::{DdirUpdate, DdirRecordedUpdates}; - use super::columnar::{ColValSpine, ColValBuilder}; - - type ConcreteTime = Product>; - - pub type Col<'scope> = Collection<'scope, ConcreteTime, DdirRecordedUpdates>; - type Arr<'scope> = Arranged<'scope, TraceAgent>>; - - /// Render a Linear chain: one pass applying the ops in sequence. `level` is - /// the op's scope depth — it locates the iteration coord for LiftIter and - /// the coordinate position EnterAt's delay lands in. - fn render_linear<'scope>(c: Col<'scope>, ops: Vec, level: usize) -> Col<'scope> { - super::columnar::join_function(c, move |k, v, t_in, _d| { - use timely::progress::Timestamp; - let k: Row = Columnar::into_owned(k); - let v: Row = Columnar::into_owned(v); - // Materialize input time once so LiftIter can read - // the iter coord at the operator's scope depth. - let t_owned: Time = Columnar::into_owned(t_in); - let iter_at_level: i64 = level - .checked_sub(1) - .and_then(|idx| t_owned.inner.get(idx).copied()) - .unwrap_or(0) as i64; - let mut results: Vec<(Row, Row, Time, Diff)> = vec![(k, v, Time::minimum(), 1i64)]; - for op in &ops { - let mut next = Vec::new(); - for (k, v, t, d) in results { - match op { - LinearOp::Project(proj) => { - let i = [k.as_slice(), v.as_slice()]; - next.push((eval_fields(&proj.key, &i), eval_fields(&proj.val, &i), t, d)); - }, - LinearOp::Filter(cond) => { - let i = [k.as_slice(), v.as_slice()]; - if eval_condition(cond, &i) { next.push((k, v, t, d)); } - }, - LinearOp::Negate => { - next.push((k, v, t, -d)); - }, - LinearOp::EnterAt(field) => { - let delay = { - let mut r = Row::new(); - eval_field_into(field, &[k.as_slice(), v.as_slice()], &mut r); - 256 * (64 - (r.as_slice().first().copied().unwrap_or(0) as u64).leading_zeros() as u64) - }; - let mut coords = smallvec::SmallVec::<[u64; 1]>::new(); - for _ in 0..level.saturating_sub(1) { coords.push(0); } - coords.push(delay); - next.push((k, v, Product::new(0u64, PointStamp::new(coords)), d)); - }, - LinearOp::LiftIter => { - let mut new_v = v.clone(); - new_v.push(iter_at_level); - next.push((k, new_v, t, d)); - }, - } - } - results = next; - } - results.into_iter() - }) - } - - fn render_join<'scope>(l: Arr<'scope>, r: Arr<'scope>, projection: &Projection) -> Col<'scope> { - let proj = projection.clone(); - use differential_dataflow::operators::join::join_traces; - use differential_dataflow::collection::AsCollection; - use super::columnar::ValColBuilder; - let stream = join_traces::<_, _, _, _, ValColBuilder>(l, r, move |k, v1, v2, t, d1, d2, c| { - use differential_dataflow::difference::Multiply; - let d = d1.clone().multiply(d2); - let i = [k.as_slice(), v1.as_slice(), v2.as_slice()]; - let (k2, v2): (Row, Row) = (eval_fields(&proj.key, &i), eval_fields(&proj.val, &i)); - c.give((k2, v2, t, d)); - }); - stream.as_collection() - } - - fn render_reduce<'scope>(a: Arr<'scope>, reducer: &Reducer) -> Arr<'scope> { - let reducer = reducer.clone(); - type ReduceFn = dyn for<'a> Fn(columnar::Ref<'a, Row>, &[(columnar::Ref<'a, Row>, Diff)], &mut Vec<(Row, Diff)>) + Send + Sync; - let f: Arc = match reducer { - Reducer::Min => Arc::new(|_key, vals, output| { - if let Some(min) = vals.iter().map(|(v, _)| v.as_slice()).min() { - output.push((Row(min.to_vec()), 1)); - } - }), - Reducer::Distinct => Arc::new(|_key, _vals, output| { output.push((Row::new(), 1)); }), - Reducer::Count => Arc::new(|_key, vals, output| { - let count: Diff = vals.iter().map(|(_, d)| *d).sum(); - if count != 0 { let mut r = Row::new(); r.push(count); output.push((r, 1)); } - }), - }; - a.reduce_abelian::<_, ColValBuilder<_,_,_,_>, ColValSpine<_,_,_,_>, _, _>( - "Reduce", - move |k, vals, output| { f(k, vals, output); }, - |col, key, upds| { - use columnar::{Clear, Push}; - col.keys.clear(); - col.vals.clear(); - col.times.clear(); - col.diffs.clear(); - for (val, time, diff) in upds.drain(..) { col.push((key, &val, &time, &diff)); } - // NOTE: required because push above doesn't group by key, val. - *col = std::mem::take(col).consolidate(); - }, - ) - } - - fn render_inspect<'scope>(col: Col<'scope>, label: String) -> Col<'scope> { - col.inspect_container(move |event| { - if let Ok((_time, container)) = event { - for (k, v, t, d) in container.updates.view().iter() { - eprintln!(" [{}] ({:?}, {:?}, {:?}, {:?})", label, ::into_owned(k), ::into_owned(v),