How to scrape linkedin data

1from selenium import webdriver
2from selenium.webdriver.common.by import By
3from selenium.webdriver.chrome.options import Options
4from selenium.webdriver.support.ui import WebDriverWait
5from selenium.webdriver.support import expected_conditions as EC
6import time
7import json
8
9class LinkedInScraper:
10    def __init__(self, email, password):
11        self.email = email
12        self.password = password
13        self.driver = None
14        
15    def setup_driver(self):
16        """Initialize Chrome with anti-detection settings"""
17        chrome_options = Options()
18        chrome_options.add_argument('--disable-blink-features=AutomationControlled')
19        chrome_options.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64)')
20        chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
21        chrome_options.add_experimental_option('useAutomationExtension', False)
22        
23        self.driver = webdriver.Chrome(options=chrome_options)
24        self.driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
25        
26    def login(self):
27        """Login to LinkedIn (account ban risk!)"""
28        try:
29            self.driver.get('https://www.linkedin.com/login')
30            time.sleep(2)
31            
32            # Enter credentials
33            email_input = self.driver.find_element(By.ID, 'username')
34            password_input = self.driver.find_element(By.ID, 'password')
35            
36            email_input.send_keys(self.email)
37            password_input.send_keys(self.password)
38            
39            # Submit
40            login_button = self.driver.find_element(By.CSS_SELECTOR, 'button[type="submit"]')
41            login_button.click()
42            
43            time.sleep(5)
44            
45            # Check for CAPTCHA or security checkpoint
46            if 'checkpoint' in self.driver.current_url or 'challenge' in self.driver.current_url:
47                print("⚠️ CAPTCHA or security checkpoint detected!")
48                print("Manual intervention required...")
49                input("Complete the challenge and press Enter to continue...")
50                
51        except Exception as e:
52            print(f"Login failed: {e}")
53            raise
54    
55    def scrape_profile(self, profile_url):
56        """
57        Extract basic profile data
58        WARNING: This is extremely fragile and breaks frequently!
59        """
60        try:
61            self.driver.get(profile_url)
62            time.sleep(3)  # Wait for page load
63            
64            # Scroll to load lazy content
65            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight/2);")
66            time.sleep(2)
67            
68            profile_data = {}
69            
70            # Extract name (CSS selectors change frequently!)
71            try:
72                name = WebDriverWait(self.driver, 10).until(
73                    EC.presence_of_element_located((By.CSS_SELECTOR, 'h1.text-heading-xlarge'))
74                ).text
75                profile_data['name'] = name
76            except:
77                profile_data['name'] = None
78            
79            # Extract headline
80            try:
81                headline = self.driver.find_element(By.CSS_SELECTOR, 'div.text-body-medium').text
82                profile_data['headline'] = headline
83            except:
84                profile_data['headline'] = None
85            
86            # Extract location
87            try:
88                location = self.driver.find_element(By.CSS_SELECTOR, 'span.text-body-small').text
89                profile_data['location'] = location
90            except:
91                profile_data['location'] = None
92            
93            return profile_data
94            
95        except Exception as e:
96            print(f"Error scraping {profile_url}: {e}")
97            return None
98    
99    def close(self):
100        """Cleanup"""
101        if self.driver:
102            self.driver.quit()
103
104# Usage example (DON'T DO THIS - WILL GET YOU BANNED!)
105"""
106scraper = LinkedInScraper('[email protected]', 'your_password')
107scraper.setup_driver()
108scraper.login()
109
110results = scraper.scrape_profile('https://www.linkedin.com/in/ryanroslansky/')
111print(results)
112
113scraper.close()
114"""

1import requests
2from bs4 import BeautifulSoup
3import json
4
5class LinkedInRequestsScraper:
6    def __init__(self, li_at_cookie):
7        """Initialize with LinkedIn session cookie"""
8        self.session = requests.Session()
9        self.session.cookies.set('li_at', li_at_cookie)
10        
11        # Mimic real browser headers
12        self.session.headers.update({
13            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
14            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
15            'Accept-Language': 'en-US,en;q=0.5',
16            'DNT': '1',
17            'Connection': 'keep-alive',
18        })
19    
20    def get_profile(self, username):
21        """Fetch profile HTML"""
22        url = f'https://www.linkedin.com/in/{username}/'
23        
24        try:
25            response = self.session.get(url, timeout=10)
26            
27            if response.status_code == 200:
28                return self.parse_profile_html(response.text, username)
29            elif response.status_code == 429:
30                print("⚠️ Rate limited! LinkedIn is blocking requests.")
31                return None
32            elif response.status_code == 999:
33                print("⚠️ Request rejected. LinkedIn detected automation.")
34                return None
35            else:
36                print(f"Error: HTTP {response.status_code}")
37                return None
38                
39        except Exception as e:
40            print(f"Request failed: {e}")
41            return None
42    
43    def parse_profile_html(self, html, username):
44        """
45        Parse HTML to extract profile data
46        WARNING: This breaks constantly as LinkedIn changes HTML structure!
47        """
48        soup = BeautifulSoup(html, 'html.parser')
49        
50        profile = {
51            'username': username,
52            'url': f'https://www.linkedin.com/in/{username}/'
53        }
54        
55        # Try to extract name (selectors change frequently)
56        try:
57            name_tag = soup.find('h1', class_='text-heading-xlarge')
58            if name_tag:
59                profile['name'] = name_tag.text.strip()
60        except:
61            pass
62        
63        return profile
64
65# Usage (also risky - requires your LinkedIn cookie)
66"""
67scraper = LinkedInRequestsScraper('your_li_at_cookie_value')
68profile = scraper.get_profile('ryanroslansky')
69print(json.dumps(profile, indent=2))
70"""

1{
2  "success": true,
3  "statusCode": 200,
4  "message": "Data retrieved successfully",
5  "errors": null,
6  "data": {
7    "firstName": "Ryan",
8    "lastName": "Roslansky",
9    "fullName": "Ryan Roslansky",
10    "headline": "CEO at LinkedIn",
11    "publicIdentifier": "ryanroslansky",
12    "followerCount": 878919,
13    "connectionsCount": 8535,
14    "creator": true,
15    "qualityProfile": true,
16    "joined": 1086269234000,
17    "profileID": "678940",
18    "urn": "ACoAAAAKXBwBikfbNJww68eYvcu2dqDYJhHbp4g",
19    "maidenName": "",
20    "summary": "As CEO of LinkedIn, I am passionate about connecting the world’s professionals to make them more productive and successful. In my years with LinkedIn, I've been fortunate to work alongside talented and innovative colleagues, and together we have developed the world's leading professional networking platform. As we look to grow and evolve LinkedIn in the years to come, I'm excited to continue driving our vision of creating economic opportunity for every member of the global workforce.",
21    "industryName": "Computer Software",
22    "industryUrn": "urn:li:fs_industry:4",
23    "location": {
24      "countryCode": "us",
25      "countryName": "United States",
26      "city": "San Francisco Bay Area",
27      "region": "",
28      "fullLocation": "San Francisco Bay Area",
29      "geoCountryUrn": "urn:li:fs_geo:103644278"
30    },
31    "backgroundImageURL": "https://media.licdn.com/dms/image/v2/C4D16AQHXtyQ-bg4B2Q/profile-displaybackgroundimage-shrink_350_1400/profile-displaybackgroundimage-shrink_350_1400/0/1580864697728?e=1756944000&v=beta&t=20F0SeOxERPniuARJQFt8nFPujcwWt6Q7V1KwuGuKxU",
32    "profilePictureURL": "https://media.licdn.com/dms/image/v2/C4D03AQELbnIckyItlw/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1667929254389?e=1756944000&v=beta&t=Xj-5fIqsj-As3o8T0Ry9OTyhyDypkOdJI-0OfXe32hc",
33    "supportedLocales": [
34      {
35        "country": "US",
36        "language": "en"
37      }
38    ],
39    "showEducationOnProfile": true,
40    "isStudent": false
41  }
42}

1pip install linkdapi

1import asyncio
2from linkdapi import AsyncLinkdAPI
3
4async def extract_profile():
5    """Extract a LinkedIn profile using async API"""
6    # Initialize the async client
7    client = AsyncLinkdAPI("your_api_key")
8    
9    try:
10        # Get profile overview
11        # Docs: https://linkdapi.com/docs?endpoint=/api/v1/profile/overview
12        profile = await client.get_profile_overview("ryanroslansky")
13        
14        print(f"Name: {profile['firstName']} {profile['lastName']}")
15        print(f"Headline: {profile['headline']}")
16        print(f"Location: {profile['location']}")
17        print(f"Connections: {profile['connectionCount']}")
18        print(f"Followers: {profile['followerCount']}")
19        
20        return profile
21        
22    except Exception as e:
23        print(f"Error: {e}")
24        return None
25
26# Run the async function
27asyncio.run(extract_profile())

1import asyncio
2from linkdapi import AsyncLinkdAPI
3
4async def extract_detailed_profile(username):
5    """Extract comprehensive profile data"""
6    client = AsyncLinkdAPI("your_api_key")
7    
8    try:
9        # Get profile overview to obtain the URN
10        # Docs: https://linkdapi.com/docs?endpoint=/api/v1/profile/overview
11        overview = await client.get_profile_overview(username)
12        profile_urn = overview['profileUrn']
13        
14        print(f"\n=== {overview['firstName']} {overview['lastName']} ===\n")
15        
16        # Get detailed profile information
17        # Docs: https://linkdapi.com/docs?endpoint=/api/v1/profile/details
18        details = await client.get_profile_details(profile_urn)
19        
20        # Get contact information
21        # Docs: https://linkdapi.com/docs?endpoint=/api/v1/profile/contact
22        contact = await client.get_contact_info(username)
23        if contact.get('emailAddress'):
24            print(f"Email: {contact['emailAddress']}")
25        
26        # Get full experience history
27        # Docs: https://linkdapi.com/docs?endpoint=/api/v1/profile/experience
28        experience = await client.get_full_experience(profile_urn)
29        print(f"\nWork Experience ({len(experience)} positions):")
30        for exp in experience[:3]:
31            print(f"  - {exp['title']} at {exp.get('companyName', 'N/A')}")
32        
33        # Get education
34        # Docs: https://linkdapi.com/docs?endpoint=/api/v1/profile/education
35        education = await client.get_education(profile_urn)
36        print(f"\nEducation ({len(education)} degrees):")
37        for edu in education:
38            print(f"  - {edu.get('degreeName', 'N/A')} from {edu['schoolName']}")
39        
40        # Get skills
41        # Docs: https://linkdapi.com/docs?endpoint=/api/v1/profile/skills
42        skills = await client.get_skills(profile_urn)
43        print(f"\nTop Skills:")
44        for skill in skills[:5]:
45            print(f"  - {skill['name']} ({skill['endorsementCount']} endorsements)")
46        
47        return {
48            'overview': overview,
49            'details': details,
50            'contact': contact,
51            'experience': experience,
52            'education': education,
53            'skills': skills
54        }
55        
56    except Exception as e:
57        print(f"Error extracting profile: {e}")
58        return None
59
60# Usage
61asyncio.run(extract_detailed_profile("ryanroslansky"))

1import asyncio
2from linkdapi import AsyncLinkdAPI
3import json
4
5async def extract_multiple_profiles(usernames):
6    """
7    Extract multiple profiles concurrently for maximum performance
8    This is the RECOMMENDED approach - much faster than threading!
9    """
10    client = AsyncLinkdAPI("your_api_key")
11    
12    async def extract_single(username):
13        """Helper function to extract one profile"""
14        try:
15            profile = await client.get_profile_overview(username)
16            print(f"✓ Extracted: {profile['firstName']} {profile['lastName']}")
17            return profile
18        except Exception as e:
19            print(f"✗ Failed {username}: {e}")
20            return None
21    
22    # Execute all extractions concurrently
23    tasks = [extract_single(username) for username in usernames]
24    results = await asyncio.gather(*tasks)
25    
26    # Filter out failures
27    successful = [r for r in results if r is not None]
28    
29    return successful
30
31# Example: Extract 100 profiles in seconds!
32usernames = [
33    "ryanroslansky",
34    "jeffweiner08",
35    "williamhgates",
36    "satyanadella",
37    # Add thousands more...
38]
39
40# Run the extraction
41profiles = asyncio.run(extract_multiple_profiles(usernames))
42
43# Save results
44with open('linkedin_profiles_bulk.json', 'w') as f:
45    json.dump(profiles, f, indent=2)
46
47print(f"\n✓ Successfully extracted {len(profiles)} profiles")