How to Scrape LinkedIn Posts Safely

1from selenium import webdriver
2from selenium.webdriver.common.by import By
3from selenium.webdriver.chrome.options import Options
4from selenium.webdriver.support.ui import WebDriverWait
5from selenium.webdriver.support import expected_conditions as EC
6import time
7import json
8
9class LinkedInPostScraper:
10    def __init__(self, email, password):
11        self.email = email
12        self.password = password
13        self.driver = None
14        
15    def setup_driver(self):
16        """Initialize Chrome with stealth settings"""
17        chrome_options = Options()
18        chrome_options.add_argument('--disable-blink-features=AutomationControlled')
19        chrome_options.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64)')
20        chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
21        
22        self.driver = webdriver.Chrome(options=chrome_options)
23        
24    def login(self):
25        """Login to LinkedIn (HIGH BAN RISK!)"""
26        self.driver.get('https://www.linkedin.com/login')
27        time.sleep(2)
28        
29        # Enter credentials
30        self.driver.find_element(By.ID, 'username').send_keys(self.email)
31        self.driver.find_element(By.ID, 'password').send_keys(self.password)
32        self.driver.find_element(By.CSS_SELECTOR, 'button[type="submit"]').click()
33        
34        time.sleep(5)
35        
36        # Check for CAPTCHA
37        if 'checkpoint' in self.driver.current_url:
38            print("⚠️ CAPTCHA detected! Manual intervention required.")
39            input("Complete CAPTCHA and press Enter...")
40    
41    def scrape_profile_posts(self, profile_url):
42        """
43        Scrape posts from a profile
44        WARNING: Extremely fragile! Breaks constantly!
45        """
46        self.driver.get(f"{profile_url}/recent-activity/all/")
47        time.sleep(3)
48        
49        posts = []
50        
51        # Scroll to load more posts (lazy loading)
52        scroll_pause = 2
53        last_height = self.driver.execute_script("return document.body.scrollHeight")
54        
55        for _ in range(5):  # Scroll 5 times
56            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
57            time.sleep(scroll_pause)
58            
59            new_height = self.driver.execute_script("return document.body.scrollHeight")
60            if new_height == last_height:
61                break
62            last_height = new_height
63        
64        # Try to extract post data (CSS selectors change constantly!)
65        try:
66            post_elements = self.driver.find_elements(
67                By.CSS_SELECTOR, 
68                'div.feed-shared-update-v2'
69            )
70            
71            for post_elem in post_elements[:10]:  # Limit to first 10
72                try:
73                    # Extract post text (selectors are fragile!)
74                    post_text = post_elem.find_element(
75                        By.CSS_SELECTOR, 
76                        'span.break-words'
77                    ).text
78                    
79                    # Try to get engagement metrics (often fails)
80                    try:
81                        reactions = post_elem.find_element(
82                            By.CSS_SELECTOR,
83                            'span.social-details-social-counts__reactions-count'
84                        ).text
85                    except:
86                        reactions = "0"
87                    
88                    posts.append({
89                        'text': post_text[:200],  # Truncate
90                        'reactions': reactions
91                    })
92                    
93                except Exception as e:
94                    continue
95            
96            return posts
97            
98        except Exception as e:
99            print(f"Error scraping posts: {e}")
100            return []
101    
102    def close(self):
103        if self.driver:
104            self.driver.quit()
105
106# Usage (DON'T DO THIS - WILL GET YOUR ACCOUNT BANNED!)
107"""
108scraper = LinkedInPostScraper('[email protected]', 'your_password')
109scraper.setup_driver()
110scraper.login()
111
112posts = scraper.scrape_profile_posts('https://www.linkedin.com/in/ryanroslansky')
113print(f"Scraped {len(posts)} posts")
114
115scraper.close()
116"""

1import requests
2from bs4 import BeautifulSoup
3
4class LinkedInPostRequestsScraper:
5    def __init__(self, li_at_cookie):
6        """Initialize with LinkedIn session cookie"""
7        self.session = requests.Session()
8        self.session.cookies.set('li_at', li_at_cookie)
9        self.session.headers.update({
10            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
11            'Accept': 'text/html,application/xhtml+xml'
12        })
13    
14    def get_posts(self, profile_username):
15        """Attempt to fetch posts (usually fails)"""
16        url = f'https://www.linkedin.com/in/{profile_username}/recent-activity/all/'
17        
18        try:
19            response = self.session.get(url, timeout=10)
20            
21            if response.status_code == 429:
22                print("⚠️ Rate limited!")
23                return None
24            elif response.status_code == 999:
25                print("⚠️ LinkedIn detected automation!")
26                return None
27            
28            # Try to parse HTML (good luck with this!)
29            soup = BeautifulSoup(response.text, 'html.parser')
30            
31            # LinkedIn's post HTML structure is incredibly complex
32            # and changes constantly. This approach rarely works.
33            
34            return []
35            
36        except Exception as e:
37            print(f"Request failed: {e}")
38            return None
39
40# Usage (also risky and unreliable)
41"""
42scraper = LinkedInPostRequestsScraper('your_li_at_cookie')
43posts = scraper.get_posts('ryanroslansky')
44"""

1{
2  "success": true,
3  "statusCode": 200,
4  "message": "Data retrieved successfully",
5  "errors": null,
6  "data": {
7    "posts": [
8      {
9        "text": "Excited to announce our new product launch...",
10        "url": "https://www.linkedin.com/feed/update/urn:li:activity:7392580420552273920",
11        "urn": "urn:li:activity:7392580420552273920",
12        "author": {
13          "name": "Ryan Roslansky",
14          "headline": "CEO at LinkedIn",
15          "urn": "ACoAAAHT6VYBz_yVpo-jE-Or8WNKjArXMd-S6KM",
16          "id": "urn:li:member:30665046",
17          "url": "https://www.linkedin.com/in/ryanroslansky",
18          "profilePictureURL": "https://media.licdn.com/dms/image/..."
19        },
20        "postedAt": {
21          "timestamp": 1762528519762,
22          "fullDate": "2025-11-07 15:15:19.00 +0000 UTC",
23          "relativeDay": "1 day"
24        },
25        "edited": false,
26        "engagements": {
27          "totalReactions": 1543,
28          "commentsCount": 87,
29          "repostsCount": 234,
30          "reactions": [
31            {
32              "reactionType": "LIKE",
33              "reactionCount": 890
34            },
35            {
36              "reactionType": "EMPATHY",
37              "reactionCount": 234
38            },
39            {
40              "reactionType": "PRAISE",
41              "reactionCount": 156
42            }
43          ]
44        },
45        "mediaContent": null,
46        "resharedPostContent": null
47      }
48    ]
49  }
50}

1pip install linkdapi

1import asyncio
2from linkdapi import AsyncLinkdAPI
3
4async def extract_featured_posts():
5    """
6    Extract featured (pinned) posts from a profile
7    Docs: https://linkdapi.com/docs?endpoint=/api/v1/posts/featured&folder=Posts
8    """
9    client = AsyncLinkdAPI("your_api_key")
10    
11    try:
12        # Get featured posts - returns array directly in data
13        response = await client.get_featured_posts("ryanroslansky")
14        featured_posts = response.get('data', [])
15        
16        print(f"Found {len(featured_posts)} featured posts\n")
17        
18        for post in featured_posts:
19            print(f"Title: {post.get('title', 'No title')}")
20            print(f"Text: {post['text'][:100]}...")
21            print(f"Type: {post['type']}")
22            print(f"Likes: {post['likesCount']}")
23            print(f"Comments: {post['commentsCount']}")
24            if post.get('mediaContent'):
25                print(f"Media: {len(post['mediaContent'])} items")
26            print("-" * 80)
27        
28        return featured_posts
29        
30    except Exception as e:
31        print(f"Error: {e}")
32        return None
33
34# Run
35asyncio.run(extract_featured_posts())

1import asyncio
2from linkdapi import AsyncLinkdAPI
3
4async def extract_all_posts(profile_urn, max_posts=50):
5    """
6    Extract all posts from a profile with pagination
7    Docs: https://linkdapi.com/docs?endpoint=/api/v1/posts/all&folder=Posts
8    """
9    client = AsyncLinkdAPI("your_api_key")
10    
11    try:
12        all_posts = []
13        cursor = ''
14        start = 0
15        
16        while len(all_posts) < max_posts:
17            # Get batch of posts
18            response = await client.get_all_posts(
19                urn=profile_urn,
20                cursor=cursor,
21                start=start
22            )
23            
24            # Extract posts from response
25            data = response.get('data', {})
26            posts = data.get('posts', [])
27            
28            if not posts:
29                break
30            
31            all_posts.extend(posts)
32            print(f"Extracted {len(all_posts)} posts so far...")
33            
34            # Update cursor for pagination
35            cursor = data.get('cursor', '')
36            start += len(posts)
37            
38            # Check if we got a cursor for next page
39            if not cursor:
40                break
41            
42            # Rate limiting
43            await asyncio.sleep(0.5)
44        
45        print(f"\n✓ Total posts extracted: {len(all_posts)}")
46        
47        # Analyze engagement
48        if all_posts:
49            avg_reactions = sum(p['engagements']['totalReactions'] for p in all_posts) / len(all_posts)
50            avg_comments = sum(p['engagements']['commentsCount'] for p in all_posts) / len(all_posts)
51            
52            print(f"Average reactions per post: {avg_reactions:.1f}")
53            print(f"Average comments per post: {avg_comments:.1f}")
54        
55        return all_posts
56        
57    except Exception as e:
58        print(f"Error: {e}")
59        return []
60
61# Usage - first get profile URN, then extract posts
62async def main():
63    client = AsyncLinkdAPI("your_api_key")
64    
65    # Get profile to obtain URN
66    profile = await client.get_profile_overview("ryanroslansky")
67    profile_urn = profile['profileUrn']
68    
69    # Extract all posts
70    posts = await extract_all_posts(profile_urn, max_posts=100)
71
72asyncio.run(main())

1import asyncio
2from linkdapi import AsyncLinkdAPI
3
4async def extract_post_with_comments(post_urn):
5    """
6    Extract detailed post information including full comment threads
7    Docs: https://linkdapi.com/docs?endpoint=/api/v1/posts/info&folder=Posts
8    Docs: https://linkdapi.com/docs?endpoint=/api/v1/posts/comments&folder=Posts
9    """
10    client = AsyncLinkdAPI("your_api_key")
11    
12    try:
13        # Get post details
14        post_response = await client.get_post_info(post_urn)
15        post_info = post_response.get('data', {}).get('post', {})
16        
17        print("=== POST DETAILS ===")
18        print(f"Author: {post_info['author']['name']}")
19        print(f"Posted: {post_info['postedAt']['fullDate']}")
20        print(f"Text: {post_info['text'][:200]}...")
21        print(f"\nEngagement:")
22        print(f"  Reactions: {post_info['engagements']['totalReactions']}")
23        print(f"  Comments: {post_info['engagements']['commentsCount']}")
24        print(f"  Reposts: {post_info['engagements']['repostsCount']}")
25        
26        # Get detailed reactions breakdown
27        if post_info['engagements'].get('reactions'):
28            print(f"\nReaction Breakdown:")
29            for reaction in post_info['engagements']['reactions']:
30                print(f"  {reaction['reactionType']}: {reaction['reactionCount']}")
31        
32        # Get comments
33        comments_response = await client.get_post_comments(
34            urn=post_urn,
35            start=0,
36            count=20  # Get first 20 comments
37        )
38        
39        comments = comments_response.get('data', {}).get('comments', [])
40        
41        print(f"\n=== COMMENTS ({len(comments)}) ===")
42        
43        for i, comment in enumerate(comments, 1):
44            author = comment.get('author', {})
45            print(f"\n{i}. {author.get('name', 'Unknown')}")
46            print(f"   {comment.get('comment', '')[:100]}...")
47            print(f"   Likes: {comment.get('engagements', {}).get('totalReactions', 0)}")
48            print(f"   Posted: {comment.get('createdAt', 'Unknown')}")
49        
50        return {
51            'post': post_info,
52            'comments': comments
53        }
54        
55    except Exception as e:
56        print(f"Error: {e}")
57        return None
58
59# Usage
60asyncio.run(extract_post_with_comments("urn:li:activity:7259185939057438720"))

1import asyncio
2from linkdapi import AsyncLinkdAPI
3import json
4
5async def bulk_extract_posts(usernames, posts_per_profile=20):
6    """
7    Extract posts from multiple profiles concurrently
8    This is the RECOMMENDED high-performance approach
9    """
10    client = AsyncLinkdAPI("your_api_key")
11    
12    async def extract_profile_posts(username):
13        """Helper to extract posts from one profile"""
14        try:
15            # Get profile URN
16            profile = await client.get_profile_overview(username)
17            urn = profile['profileUrn']
18            
19            # Get posts
20            response = await client.get_all_posts(urn=urn, start=0)
21            posts = response.get('data', {}).get('posts', [])
22            
23            # Limit to specified number
24            posts = posts[:posts_per_profile]
25            
26            print(f"✓ Extracted {len(posts)} posts from {profile['firstName']} {profile['lastName']}")
27            
28            return {
29                'username': username,
30                'profile': {
31                    'name': f"{profile['firstName']} {profile['lastName']}",
32                    'headline': profile['headline']
33                },
34                'posts': posts
35            }
36            
37        except Exception as e:
38            print(f"✗ Failed {username}: {e}")
39            return None
40    
41    # Execute all extractions concurrently
42    tasks = [extract_profile_posts(username) for username in usernames]
43    results = await asyncio.gather(*tasks)
44    
45    # Filter out failures
46    successful = [r for r in results if r is not None]
47    
48    # Calculate statistics
49    total_posts = sum(len(r['posts']) for r in successful)
50    
51    print(f"\n=== BULK EXTRACTION COMPLETE ===")
52    print(f"Profiles processed: {len(successful)}/{len(usernames)}")
53    print(f"Total posts extracted: {total_posts}")
54    
55    return successful
56
57# Example: Extract posts from multiple thought leaders
58usernames = [
59    "ryanroslansky",
60    "jeffweiner08",
61    "satyanadella",
62    "sundarpichai",
63    "tim-cook",
64    # Add hundreds more...
65]
66
67results = asyncio.run(bulk_extract_posts(usernames, posts_per_profile=10))
68
69# Save to file
70with open('linkedin_posts_bulk.json', 'w') as f:
71    json.dump(results, f, indent=2)
72
73print("✓ Results saved to linkedin_posts_bulk.json")

1import asyncio
2from linkdapi import AsyncLinkdAPI
3
4async def extract_post_likers(post_urn):
5    """
6    Extract users who reacted to a post
7    Docs: https://linkdapi.com/docs?endpoint=/api/v1/posts/likes&folder=Posts
8    """
9    client = AsyncLinkdAPI("your_api_key")
10    
11    try:
12        # Get users who liked the post
13        response = await client.get_post_likes(urn=post_urn, start=0)
14        data = response.get('data', {})
15        likers = data.get('likes', [])
16        total_likes = data.get('totalLikes', 0)
17        
18        print(f"Found {total_likes} total reactions")
19        print(f"Showing {len(likers)} reactions\n")
20        
21        for liker in likers[:20]:  # Show first 20
22            actor = liker.get('actor', {})
23            print(f"- {actor.get('name', 'Unknown')}")
24            print(f"  {actor.get('headline', '')}")
25            print(f"  Reaction: {liker.get('reactionType', 'LIKE')}")
26            print()
27        
28        return likers
29        
30    except Exception as e:
31        print(f"Error: {e}")
32        return []
33
34# Usage
35asyncio.run(extract_post_likers("urn:li:activity:7259185939057438720"))