Skip to content

Instantly share code, notes, and snippets.

@jammycakes
Last active December 21, 2017 16:39
Show Gist options
  • Save jammycakes/ed697dcf226721a873108e97040c70d2 to your computer and use it in GitHub Desktop.
Save jammycakes/ed697dcf226721a873108e97040c70d2 to your computer and use it in GitHub Desktop.
Getting
import boto3
def get_paginated_results(func, key, **kwargs):
"""
Many boto3 methods return only a limited number of results at once,
with pagination information. This function handles the pagination to
retrieve the entire result set.
@param func
The function to call to get the data.
@param key
The key in the result set returned from the function containing the data
that we want
@param kwargs
The arguments to pass to each call to func.
@returns
An iterable containing the results of the consecutive calls to func.
"""
next_token = ''
more = True
while more:
result = func(**kwargs, nextToken=next_token)
things = result[key]
next_token = result.get('nextToken', None)
more = next_token != None
for thing in things:
yield thing
def bundle(ids, block_size):
"""
Given an iterable, bundles it into lists, each block_size long.
@param ids
The things to bundle
@param block_size
The maximum size of each block
@returns
An iterable of blocks, each containing up to block_size things.
"""
block = []
for id in ids:
block.append(id)
if len(block) == block_size:
yield block
block = []
if len(block):
yield block
def bundled_map(ids, block_size, func):
"""
A map function where the mapping function is applied to blocks of items
rather than to individual things.
@param ids
The things to be mapped.
@param block_size
The number of things to be mapped at a time
@param func
A mapping function that maps a block of block_size things at a time.
"""
blocks = bundle(ids, block_size)
for block in blocks:
things = func(block)
for thing in things:
yield thing
# ====== Example usage ====== #
def get_services(cluster):
ecs = boto3.client('ecs')
service_ids = get_paginated_results(ecs.list_services, 'serviceArns', cluster=cluster)
services = bundled_map(
service_ids, 10,
lambda x: ecs.describe_services(cluster=cluster, services=x)['services']
)
return services
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment