This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import logging | |
import boto3 | |
def _log_botocore_errors(*_args, **kwargs): | |
"""Handler for botocore needs-retry events.""" | |
# Arg parsing as per | |
# https://github.com/boto/botocore/blob/6451ae1fad57f4453af97649e7ed9192b0f623be/botocore/retries/standard.py#L106 | |
attempt_number = kwargs["attempts"] | |
operation_model = kwargs["operation"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2019-05-04 10:20:07,160 INFO node_provider.py:34 -- ClusterState: Loaded cluster state: {'YOUR_HEAD_NODE_HOSTNAME': {'tags': {'ray-node-type': 'head'}, 'state': 'terminated'}} | |
2019-05-04 10:20:07,161 INFO node_provider.py:59 -- ClusterState: Writing cluster state: {'YOUR_HEAD_NODE_HOSTNAME': {'tags': {'ray-node-type': 'head'}, 'state': 'terminated'}} | |
2019-05-04 10:20:07,162 INFO autoscaler.py:380 -- StandardAutoscaler: {'cluster_name': 'default', 'min_workers': 0, 'max_workers': 1, 'initial_workers': 0, 'target_utilization_fraction': 0.8, 'idle_timeout_minutes': 5, 'provider': {'type': 'local', 'head_ip': 'YOUR_HEAD_NODE_HOSTNAME', 'worker_ips': []}, 'auth': {'ssh_user': 'YOUR_USERNAME', 'ssh_private_key': '~/.ssh/id_rsa'}, 'head_node': {}, 'worker_nodes': {}, 'file_mounts': {}, 'setup_commands': [], 'head_setup_commands': [], 'worker_setup_commands': [], 'head_start_ray_commands': [], 'worker_start_ray_commands': []} | |
2019-05-04 10:20:07,172 INFO autoscaler.py:630 -- StandardAutoscaler: 0/0 target nodes (0 pe |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python3 | |
"""List S3 bucket objects in parallel. | |
This module contains a parallel implementation of S3 list_objects_v2() | |
API call. The implementation lists objects under every distinct prefix | |
in parallel. Hence, a speedup is achieved if objects are spread under | |
multiple distinct prefixes. | |
""" | |
import argparse |