Last active
November 3, 2023 14:21
-
-
Save secretorange/887fac16d2739155ca883ffc45dc68c9 to your computer and use it in GitHub Desktop.
How to manually paginate whilst scanning using AWS DynamoDB and C# SDK
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
public async Task<IPaged<Item>> Pagination(string serialisedPaginationInfo, int size) | |
{ | |
// We will fill our 'page' until it's met the pageSize or we complete our search (i.e. scan ALL records) | |
var page = new List<DM.Document>(); | |
// Get a reference to the table. Only the lower level API seems to suppot manual pagination | |
var table = Dynamo.GetTargetTable<Item>(); | |
var pagination = String.IsNullOrWhiteSpace(serialisedPaginationInfo) | |
? new PaginationInfo() | |
: JsonConvert.DeserializeObject<PaginationInfo>(serialisedPaginationInfo); | |
var token = pagination.Token; | |
var exit = false; | |
// Build your query... | |
var filter = new DM.ScanFilter(); | |
filter.AddCondition("Type", DM.ScanOperator.Equal, 2); | |
var counter = 0; | |
do | |
{ | |
var search = table.Scan(new DM.ScanOperationConfig() | |
{ | |
Filter = filter, | |
// The token will tell Dynamo where to start | |
PaginationToken = token, | |
// The limit is the max number of scans Dynamo is going to make | |
Limit = Math.Max(100, size) | |
}); | |
// Hit the DB and get the next batch. | |
// Note: This will return between 0 and {Limit} results. Remember, it can return 0 results even if the search isn't complete yet. | |
var batch = await search.GetNextSetAsync(); | |
// If it's the first batch and there's a lastId present, let's remove items we've already seen | |
if (counter == 0 && pagination.LastId != null) | |
{ | |
int index = batch.FindIndex(x => x["Id"] == pagination.LastId); | |
if (index > -1) | |
batch.RemoveRange(0, index + 1); | |
} | |
// Try and top up our list as much we can (until we hit the page size) | |
var take = batch.Take(size - page.Count); | |
page.AddRange(take); | |
// We'll exit the loop if either we have filled our page or we've scanned all records in the db | |
exit = (page.Count == size) || search.IsDone; | |
// Check to see if there are any documents left on the current token | |
if (search.IsDone && take.Count() == batch.Count()) | |
{ | |
// If there are no documents left, null the token so we don't send it back to the client | |
token = null; | |
} | |
else if(!exit) | |
{ | |
// Let's keep going. Get the next pagination token ready to hit Dynamo again... | |
token = search.PaginationToken; | |
} | |
counter++; | |
} | |
while (!exit); | |
var items = page.Select(doc => Convert(doc)); | |
var paged = new PagedItems<Item>(items, 0, items.Count(), items.Count()); | |
if (token != null) | |
{ | |
var lastId = paged.Items.Select(i => i.Id).LastOrDefault(); | |
paged.Position = JsonConvert.SerializeObject(new PaginationInfo(token, lastId)); | |
} | |
return paged; | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
it would be nice to have a few more details, e.g. where do you get the IPaged interface from? Where do you get the Dynamo object. None of this is present in the AWS SDK, as far as I can see