Skip to content

Instantly share code, notes, and snippets.

@mariusvniekerk

mariusvniekerk/adapt_example_localcluster.py

Created January 22, 2021 13:58
Show Gist options
  • Save mariusvniekerk/edeefd93b17a7ae01472e1818512ac55 to your computer and use it in GitHub Desktop.
Save mariusvniekerk/edeefd93b17a7ae01472e1818512ac55 to your computer and use it in GitHub Desktop.
Download ZIP
Example localcluster adaptive
Raw
adapt_example_localcluster.py
import distributed
import logging
import time
import os
import psutil
logfmt = '%(asctime)s %(process)d:%(levelname)s:%(name)s:%(message)s'
logging.basicConfig(
level=logging.INFO,
format=logfmt)
for h in logging.root.handlers:
h.setFormatter(logging.Formatter(logfmt))
n_workers = 8
print(distributed.__version__)
def work(i):
print(os.getpid(), i)
time.sleep(i / 100)
return i
def run():
with distributed.LocalCluster(n_workers=1, threads_per_worker=1) as cluster:
adapt = cluster.adapt(minimum=1, maximum=n_workers, interval='20ms')
self_process = psutil.Process()
try:
with distributed.Client(cluster) as client:
futs = [client.submit(work, x) for x in range(100)]
for res in distributed.as_completed(futs):
print("child_pids:", [p.pid for p in self_process.children(recursive=True)])
print(res.result())
del futs
finally:
adapt.stop()
if __name__ == '__main__':
run()
Raw
output.txt
/Users/mvanniekerk/miniconda3/envs/blockscorepy38/bin/python "/Users/mvanniekerk/Library/Application Support/JetBrains/PyCharm2020.3/scratches/dask-sched.py"
2020.12.0
2020.12.0
1945 0
1945 41
child_pids: [1942, 1943, 1945]
0
1945 99
child_pids: [1942, 1943, 1945]
41
2020.12.0
2020.12.0
2020.12.0
2020.12.0
1946 89
1948 51
1947 81
1949 53
1948 34
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949]
51
1949 8
2020.12.0
2020.12.0
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951]
53
1949 32
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951]
1945 95
8
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951]
99
1947 26
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951]
81
1948 2
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951]
34
1948 12
1946 42
1950 21
1951 7
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951]
2
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951]
2020.12.0
89
1951 28
1948 93
1949 61
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
7
1947 91
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
12
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
32
1950 3
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
26
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
1950 67
21
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
1952 68
3
1951 75
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
28
1946 5
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
42
1946 17
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
5
1946 52
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
17
1949 64
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
61
1945 92
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
95
1950 76
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
67
1952 77
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
68
1948 94
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
93
1947 39
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
91
1951 65
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
75
1946 44
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
52
1949 60
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
64
1947 13
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
39
1946 79
1947 36
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
44
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
13
1950 59
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
76
1945 82
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
92
1952 38
1951 47
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
77
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
65
1948 33
1947 62
1949 57
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
94
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
36
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
60
1952 20
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
38
1951 46
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
47
1950 90
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
59
1948 85
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
33
1952 78
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
20
1946 49
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
79
1945 72
1949 63
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
82
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
57
1947 69
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
62
1951 30
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
46
1946 18
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
49
1951 45
1946 25
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
30
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
18
1952 58
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
1948 56
78
1950 31
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
1949 71
85
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
90
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
63
1945 66
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
72
1947 22
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
69
1946 1
1946 83
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
25
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
1
1951 23
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
45
1950 84
1947 74
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
31
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
22
1951 86
1952 43
1948 37
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
23
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
58
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
56
1949 40
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
71
1945 27
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
66
1948 73
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
37
1952 16
1946 24
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
43
1945 11
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
83
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
27
1947 48
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
74
1949 50
1945 6
1952 97
1950 15
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
40
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
1945 88
11
1946 80
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
16
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
84
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
6
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
24
1950 9
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
15
1951 96
1950 4
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
86
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
9
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
4
1950 35
1947 10
child_pids:1949 98
[1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
1948 87
48
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
50
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
1947 19
73
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
10
1950 55
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
35
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
19
1946 70
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
1952 54
1947 14
1945 14
80
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
88
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
97
1951 29
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
14
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
96
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
55
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
87
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
98
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
29
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
54
child_pids: [1942, 1943, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952]
70
Process finished with exit code 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment