shentonfreude · December 21, 2018 16:43
diff --git a/s3videopager.py b/s3videopager.py
 #!/usr/bin/env python3
 # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/paginators.html
 """Loop over S3 videos looking for ones that need new thumbs.

 Finds all the video/assetid/ prefixes and loops over them, then for each
 with enough keys, looks to see if they have thumbs (so skip this one) or a
 ~orig. file we can use to create thumbs.
 """

 import os

 import boto3

 S3_NAME = os.environ['S3_NAME']

 client = boto3.client('s3')
 paginator = client.get_paginator('list_objects')
 page_iter = paginator.paginate(Bucket=S3_NAME, Prefix='video/', Delimiter='/')
 for page in page_iter:  # returns 1000 page['Contents'] or page['CommonPrefixes'])
    for prefixobj in page['CommonPrefixes']:  # video/hst2005_640x480/
        video_prefix = prefixobj['Prefix']
        objs = client.list_objects_v2(Bucket=S3_NAME, Prefix=video_prefix)
        keys = [obj['Key'] for obj in objs['Contents']]
        if len(keys) < 3:  # need at least metadata.json, collection.json, ...~orig.*
            continue
        got_thumb = False
        orig_video = False
        for key in keys:
            if key.endswith('~small.jpg'):
                got_thumb = key
                break           # so we don't need to thumb this one
            if '~orig.' in key:
                orig_video = key
        if got_thumb:
            print('gotthumb %s' % got_thumb)
            # we've got thumbs, no need to do anything, ignore
            continue            # we've got a thumb, so skip to next video
        if orig_video:
            print('CREATE   %s' % orig_video)
            # Call code to create at video_prefix with orig_video
        else:
            print('wtf?     %s' % keys)
            # this is probably bogus from testing or broken code, ignore
	#!/usr/bin/env python3
	# https://boto3.amazonaws.com/v1/documentation/api/latest/guide/paginators.html
	"""Loop over S3 videos looking for ones that need new thumbs.

	Finds all the video/assetid/ prefixes and loops over them, then for each
	with enough keys, looks to see if they have thumbs (so skip this one) or a
	~orig. file we can use to create thumbs.
	"""

	import os

	import boto3

	S3_NAME = os.environ['S3_NAME']

	client = boto3.client('s3')
	paginator = client.get_paginator('list_objects')
	page_iter = paginator.paginate(Bucket=S3_NAME, Prefix='video/', Delimiter='/')
	for page in page_iter: # returns 1000 page['Contents'] or page['CommonPrefixes'])
	for prefixobj in page['CommonPrefixes']: # video/hst2005_640x480/
	video_prefix = prefixobj['Prefix']
	objs = client.list_objects_v2(Bucket=S3_NAME, Prefix=video_prefix)
	keys = [obj['Key'] for obj in objs['Contents']]
	if len(keys) < 3: # need at least metadata.json, collection.json, ...~orig.*
	continue
	got_thumb = False
	orig_video = False
	for key in keys:
	if key.endswith('~small.jpg'):
	got_thumb = key
	break # so we don't need to thumb this one
	if '~orig.' in key:
	orig_video = key
	if got_thumb:
	print('gotthumb %s' % got_thumb)
	# we've got thumbs, no need to do anything, ignore
	continue # we've got a thumb, so skip to next video
	if orig_video:
	print('CREATE %s' % orig_video)
	# Call code to create at video_prefix with orig_video
	else:
	print('wtf? %s' % keys)
	# this is probably bogus from testing or broken code, ignore