Created
April 26, 2022 21:19
-
-
Save pomponchik/ed33d6093597489143e4e4ed6c265082 to your computer and use it in GitHub Desktop.
Пробуем ускорить вывод с помощью собственного постраничного кэширования
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from collections import deque | |
| from threading import Lock, Thread | |
| from sys import argv | |
| from time import time | |
| from concurrent.futures import ThreadPoolExecutor | |
| class Outer: | |
| def __init__(self, filename): | |
| self.file = open(filename, 'ab') | |
| self.buffer = deque() | |
| self.buffer_size = 0 | |
| self.lock = Lock() | |
| self.block_size = self.get_block_size() | |
| self.executor = ThreadPoolExecutor(max_workers=1) | |
| def get_block_size(self): | |
| return 262144 | |
| def __call__(self, string): | |
| converted = bytes(string, 'utf-8') | |
| self.maybe_flush(converted) | |
| #self.executor.submit(self.maybe_flush, converted) | |
| def maybe_flush(self, converted): | |
| with self.lock: | |
| self.buffer.append(converted) | |
| self.buffer_size += len(converted) | |
| if self.buffer_size > self.block_size: | |
| cycles_number = self.buffer_size // self.block_size | |
| bytes_limit = cycles_number * self.block_size | |
| buffer_2 = [] | |
| number_of_cutting = 0 | |
| cutted = 0 | |
| tail = None | |
| for data in self.buffer: | |
| if cutted + len(data) <= bytes_limit: | |
| cutted += len(data) | |
| buffer_2.append(data) | |
| number_of_cutting += 1 | |
| elif cutted + len(data) > bytes_limit: | |
| diff_size = bytes_limit - cutted | |
| data_half = data[:diff_size] | |
| tail = data[diff_size:] | |
| number_of_cutting += 1 | |
| cutted += len(data_half) | |
| buffer_2.append(data_half) | |
| if cutted == bytes_limit: | |
| break | |
| for _ in range(number_of_cutting): | |
| self.buffer.popleft() | |
| if tail is not None: | |
| self.buffer.appendleft(tail) | |
| output_data = b''.join(buffer_2) | |
| self.file.write(output_data) | |
| self.file.flush() | |
| if len(argv) <= 2: | |
| print('no') | |
| exit(1) | |
| mode = argv[1] | |
| file_path = argv[2] | |
| if mode == 'my': | |
| timer = time() | |
| outer = Outer(file_path) | |
| for x in range(1000000): | |
| outer('kek\n') | |
| print('result:', time() - timer) | |
| elif mode == 'system': | |
| timer = time() | |
| outer = open(file_path, 'a') | |
| for x in range(1000000): | |
| outer.write('kek\n') | |
| outer.flush() | |
| print('result:', time() - timer) | |
| elif mode == 'noflush': | |
| timer = time() | |
| outer = open(file_path, 'a') | |
| for x in range(1000000): | |
| outer.write('kek\n') | |
| print('result:', time() - timer) | |
| else: | |
| print('no') | |
| exit(1) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment