Alex Moore-Niemi mooreniemi

mooreniemi / Cargo.toml

Created September 11, 2024 04:00

	[package]
	name = "dagger"
	version = "0.1.0"
	edition = "2021"

	# Add dependencies as needed
	[dependencies]
	petgraph = "0.6" # for DAG structure and traversal
	tokio = { version = "1", features = ["full"] } # for async orchestration
	serde = { version = "1.0", features = ["derive"] }

mooreniemi / Cargo.toml

Created September 10, 2024 02:54

	[package]
	name = "gen_conc"
	version = "0.1.0"
	edition = "2021"

	# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

	[dependencies]
	tokio = { version = "1", features = ["full"] }
	sha2 = "0.10" # For hashing document content (SHA-256)

mooreniemi / Cargo.toml

Created May 20, 2024 03:50

	[package]
	name = "cete_node"
	version = "0.1.0"
	edition = "2021"
	# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

	[dependencies]
	actix-web = "4"
	aws-config = "*"
	aws-sdk-codecommit = "*"

mooreniemi / Cargo.toml

Created May 20, 2024 03:17

	[package]
	name = "delta_fake"
	version = "0.1.0"
	edition = "2021"

	# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

	[dependencies]
	anyhow = "1.0.86"
	dashmap = "5.5.3"

mooreniemi / Cargo.toml

Created May 19, 2024 19:31

`SharedFile` using `unsafe`

	[package]
	name = "delta_fake"
	version = "0.1.0"
	edition = "2021"

	# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

	[dependencies]
	anyhow = "1.0.86"
	fd-lock = "4.0.2"

mooreniemi / Cargo.toml

Last active May 10, 2024 02:54

mooreniemi / Cargo.toml

Created May 7, 2024 14:18

This is a minimal example of reading a delta table from S3 with Rust. From [this discussion](https://delta-users.slack.com/archives/C013LCAEB98/p1714949120391979).

	[package]
	name = "read-table-example"
	version = "0.1.0"
	edition = "2021"

	# note: I did not double-check this and your dependencies might slightly vary (I cut some of mine off)
	[dependencies]
	deltalake = { version = "*", features = ["s3"] }
	tokio = { version = "1", features = ["full"] }
	quick-xml = { version = "0.23.0-alpha3" }

mooreniemi / Cargo.toml

Last active May 6, 2024 04:46

streaming parquet read from s3

	[package]
	name = "stream-pq"
	version = "0.1.0"
	edition = "2021"

	[dependencies]
	deltalake = { version = "*", features = ["s3"] }
	tokio = { version = "1", features = ["full"] }
	quick-xml = { version = "0.23.0-alpha3" }
	aws-config = { version = "*", features = ["rustls"] }

mooreniemi / basic_process.rs

Created May 5, 2024 20:29

	use arrow::array::ArrayRef;
	use arrow::record_batch::RecordBatch;
	use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
	use parquet::arrow::ArrowWriter;
	use std::env;
	use std::fs::File;
	use std::path::Path;
	use std::time::Instant;

	fn main() -> Result<(), Box<dyn std::error::Error>> {

mooreniemi / Cargo.toml

Created May 5, 2024 02:32