Skip to content

Instantly share code, notes, and snippets.

@eevmanu
Created February 17, 2023 16:07
Show Gist options
  • Save eevmanu/45b1161cc40217fec6d296a07ae787b7 to your computer and use it in GitHub Desktop.
Save eevmanu/45b1161cc40217fec6d296a07ae787b7 to your computer and use it in GitHub Desktop.
results of tracking memory usage for alternatives to flatten a list of list in python (https://stackoverflow.com/a/59913424/3889948)
# --------------------------------------------------------------------------------
# option 1: using numpy.array(< list >, dtype=object).flat)
# --------------------------------------------------------------------------------
import tracemalloc
import functools
import operator
import itertools
import numpy
l1 = [[1,2,3],[4,5,6], [7], [8,9]] * 10_000
tracemalloc.start()
l2 = list(numpy.array(l1, dtype=object).flat)
size, peak = tracemalloc.get_traced_memory()
tracemalloc.reset_peak()
tracemalloc.stop()
print(f"{size=}, {peak=}")
# size=325_246, peak=647_522
# size=325_246, peak=647_522
# --------------------------------------------------------------------------------
# option 2: functools.reduce + operator.iconcat
# --------------------------------------------------------------------------------
import tracemalloc
import functools
import operator
import itertools
l1 = [[1,2,3],[4,5,6], [7], [8,9]] * 10_000
tracemalloc.start()
l2 = functools.reduce(operator.iconcat, l1, [])
size, peak = tracemalloc.get_traced_memory()
tracemalloc.reset_peak()
tracemalloc.stop()
print(f"{size=}, {peak=}")
# size=796_432, peak=806_712
# size=796_824, peak=807_128
# --------------------------------------------------------------------------------
# option 3: itertools.chain.from_iterable
# --------------------------------------------------------------------------------
import tracemalloc
import functools
import operator
import itertools
l1 = [[1,2,3],[4,5,6], [7], [8,9]] * 10_000
tracemalloc.start()
l2 = list(itertools.chain.from_iterable(l1))
size, peak = tracemalloc.get_traced_memory()
tracemalloc.reset_peak()
tracemalloc.stop()
print(f"{size=}, {peak=}")
# size=802_272, peak=812_448
# size=802_112, peak=812_408
# --------------------------------------------------------------------------------
# option 4: map + extend
# --------------------------------------------------------------------------------
import tracemalloc
import functools
import operator
import itertools
l1 = [[1,2,3],[4,5,6], [7], [8,9]] * 10_000
tracemalloc.start()
l2 = []
list(map(l2.extend, l1))
size, peak = tracemalloc.get_traced_memory()
tracemalloc.reset_peak()
tracemalloc.stop()
print(f"{size=}, {peak=}")
# size=796_488, peak=1_147_668
# size=796_880, peak=1_148_076
# --------------------------------------------------------------------------------
@eevmanu
Copy link
Author

eevmanu commented Feb 17, 2023

check tracemalloc - Trace memory allocations (from official docs 📄) to understand more about it

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment