Created
March 21, 2024 20:39
-
-
Save PowerX-NOT/7b9ed0014f2a71780bc51f2a684adf63 to your computer and use it in GitHub Desktop.
FIX duplicates in proprietarys
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| def remove_duplicates(): | |
| file_paths = input("Enter file paths separated by spaces: ").split() | |
| for file_path in file_paths: | |
| try: | |
| with open(file_path, 'r') as file: | |
| lines = file.readlines() | |
| except FileNotFoundError: | |
| print(f"File {file_path} not found. Skipping...") | |
| continue | |
| line_count = {} | |
| for line in lines: | |
| line = line.strip() | |
| if line in line_count: | |
| line_count[line] += 1 | |
| else: | |
| line_count[line] = 1 | |
| duplicate_lines = {line: count for line, count in line_count.items() if count > 1} | |
| print(f"Duplicate lines in {file_path}:") | |
| for line, count in duplicate_lines.items(): | |
| print(f"{' ' * 6}{count} {line}") | |
| delete = input("Do you want to delete the duplicate lines? (y/n): ") | |
| if delete.lower() == 'y': | |
| unique_lines = [] | |
| seen_lines = set() | |
| for line in lines: | |
| line = line.strip() | |
| if line not in seen_lines or line_count[line] == 1: | |
| seen_lines.add(line) | |
| unique_lines.append(line) | |
| with open(file_path, 'w') as file: | |
| prev_line_empty = False | |
| for line in unique_lines: | |
| if line.startswith('#') and not prev_line_empty: | |
| file.write('\n') # Add a blank line above lines starting with # | |
| file.write(line + '\n') | |
| prev_line_empty = (line == '') | |
| print(f"Duplicate lines in {file_path} deleted.") | |
| else: | |
| print(f"Duplicate lines in {file_path} not deleted.") | |
| # Example usage | |
| remove_duplicates() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment