Scraping of Fake Jobs Webpage in Python with Beautiful Soup

Scraping Fake Jobs Webpage

import requests
from bs4 import BeautifulSoup

# (1) Send Request
URL = "https://realpython.github.io/fake-jobs/" # page URL
page = requests.get(URL) # get the webpage

# (2) Parse the HTML
soup = BeautifulSoup(page.content, "html.parser") # parse page

# (3) Navigate the Parse Tree
results = soup.find(id="ResultsContainer") # jobs are listed in this div ResultsContainer
# print(results.prettify())

# card-content divs contain the job details 
job_elements = results.find_all("div", class_="card-content")

# (4) Extract and Print Jobs Data
# for job_element in job_elements:
#     print(job_element, end="\n"*2)

# use lambda function to extract jobs titles containing the string 'python'
python_jobs = results.find_all(
    "h2", string=lambda text: "python" in text.lower()
)

print(len(python_jobs)) # number of jobs listed on page

# fetch great-grandparent elements to get access to all the information you want
python_job_elements = [
    h2_element.parent.parent.parent for h2_element in python_jobs
]

# print out the job detail - title, company and location
for job_element in python_job_elements:
    title_element = job_element.find("h2", class_="title")
    company_element = job_element.find("h3", class_="company")
    location_element = job_element.find("p", class_="location")
    print(title_element.text.strip())
    print(company_element.text.strip())
    print(location_element.text.strip())
    
    '''
    Print all links
    links = job_element.find_all("a")
    for link in links:
         print(link.text.strip())
         link_url = link["href"]
         print(f"Apply here: {link_url}\n")
    '''
        
    # Print only the 2nd link
    link_url = job_element.find_all("a")[1]["href"]
    print(f"Apply here: {link_url}\n")

    print()