Notes

Python Crash Course (Basics needed for web scraping)

The basics you need to know in order to scrape data from websites and compile it in CSVs

Edit on GitHub

Python
2 minutes

scrape data

 1import requests
 2from bs4 import BeautifulSoup
 3
 4url = 'https://www.samplestore.pk/search/?keywords=amazfit&dataBi=k1'
 5
 6page = requests.get(url)
 7soup = BeautifulSoup(page.content, 'html.parser')
 8
 9titles = soup.select('h2.title > a')
10prices = soup.select('p.price > .shop_s')
11
12# iterate over an array
13# for title in titles:
14#   print(title.text)
15
16# # iterate over values from multiple arrays
17for title, price in zip(titles, prices):
18  print(title.text, price.text)

scrape: - find elements by classname

1prices = soup.select('p.price > .shop_s')
1# BeautifulSoup 4.1.2+
2soup.find_all("div", class_="stylelistrowone stylelistrowtwo")
1# BeautifulSoup 4+
2
3# single classname
4mydivs = soup.find_all('div', 'class_name')
5
6#  pass the list of class names as parameter like :
7mydivs = soup.find_all('div', ['class1', 'class2'])

iterate over arrays

 1# iterate over an array
 2for title in titles:
 3  print(title.text)
 4
 5for price in prices:
 6  print(price.text)
 7
 8# iterate over values from multiple arrays
 9for title, price in zip(titles, prices):
10  print(title.text, price.text)
1for f, b in zip(foo, bar):
2  print(f, b)

write a CSV from dictionary (array data)

 1# Write these to a CSV file
 2filename = 'products.csv'
 3
 4# products.csv file will be created in the current working
 5with open( filename, 'w', newline='') as file:
 6  # filednames = a list object which should contain the column headers specifying the order in which data should be written in the CSV file
 7  fieldnames = ['Title', 'Price']
 8  writer = csv.DictWriter(file, fieldnames = fieldnames)
 9
10  # first row that contains the column headings
11  writer.writeheader()
12
13  for title, price in zip(titles, prices):
14    writer.writerow({'Title': title.text, 'Price': price.text})
Title,Price
Amazfit Stratos Multisport GPS...,"Rs 28,000"
Amazfit GTR Smartwatch,"Rs 22,500"
Amazfit GTS Sports Smart Watch,"Rs 23,499"
Amazfit Bip Lite Smartwatch,"Rs 9,000"
Amazfit GTS/GTR Charging Cable,Rs 800
Amazfit T-Rex Military Smart W...,"Rs 23,500"
Amazfit AirRun Treadmill,"Rs 149,000"
Amazfit Neo Retro Design,"Rs 6,499"
Amazfit GTR 2,"Rs 31,499"
Amazfit GTS 2 Smart Watch,"Rs 30,599"
Amazfit Bip U,"Rs 11,000"