Last active
April 2, 2024 22:52
-
-
Save RohanAwhad/038d243808098e5eee8f62deb914e09f to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // Updated extractLinkedInPosts function with continuous scrolling | |
| async function extractLinkedInPosts() { | |
| const postsData = []; | |
| const seenTexts = new Set(); | |
| let lastLength = 0; | |
| while (postsData.length < 20) { | |
| window.scrollTo(0, document.body.scrollHeight); | |
| await new Promise(resolve => setTimeout(resolve, 2000)); // Wait for more posts to load | |
| const postsList = document.getElementsByTagName('ul')[1].getElementsByTagName('li'); | |
| if (postsList.length === lastLength) { | |
| break; // Break if no new posts are loaded | |
| } | |
| lastLength = postsList.length; | |
| for (let i = 0; i < postsList.length && postsData.length < 20; i++) { | |
| const post = postsList[i]; | |
| const seeMoreButton = post.querySelector('button.feed-shared-inline-show-more-text__see-more-less-toggle'); | |
| if (seeMoreButton) seeMoreButton.click(); | |
| await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for the text to expand | |
| const textContainer = post.querySelector('div.feed-shared-update-v2__description-wrapper > div'); | |
| const text = textContainer ? textContainer.textContent.trim() : ''; | |
| const linkAnchor = post.querySelector('a.app-aware-link'); | |
| const link = linkAnchor ? linkAnchor.href : ''; | |
| if (!seenTexts.has(text) && text) { | |
| seenTexts.add(text); | |
| postsData.push({ text, link }); | |
| } | |
| } | |
| } | |
| return postsData; | |
| } | |
| // Function to download posts data as JSON | |
| function downloadPostsData(postsData) { | |
| const dataStr = "data:text/json;charset=utf-8," + encodeURIComponent(JSON.stringify(postsData)); | |
| const downloadAnchorNode = document.createElement('a'); | |
| downloadAnchorNode.setAttribute("href", dataStr); | |
| downloadAnchorNode.setAttribute("download", "LinkedInPostsData.json"); | |
| document.body.appendChild(downloadAnchorNode); | |
| downloadAnchorNode.click(); | |
| downloadAnchorNode.remove(); | |
| } | |
| // Main runner function | |
| async function main() { | |
| const postsData = await extractLinkedInPosts(); | |
| console.log(postsData); // Log the extracted data | |
| downloadPostsData(postsData); // Download the data | |
| } | |
| main(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment