python - Crawling 의류 데이터 크롤링

0. 파이썬 설치 & 주피터 노트북 설치(cmd)

- 파이썬 설치

- 파이썬 환경 변수 설정

- 주피터 노트북 설치

 

 

 

 

 

[Crawling] 전국 스타벅스 매장 정보 .. : 네이버블로그 (naver.com)[Crawling] 전국 스타벅스 매장 정보 .. : 네이버블로그 (naver.com)

[Crawling] 전국 스타벅스 매장 정보 가져오기

 

1. 자동 클릭(selenium) & selector로 element 가져오기 (Beautifulsoup)

https://smothly.github.io/data%20science/nlp/2020/01/13/%EB%84%A4%EC%9D%B4%EB%B2%84-%EC%98%81%ED%99%94-%EB%A6%AC%EB%B7%B0-%ED%81%AC%EB%A1%A4%EB%A7%81(Crawling).html 

네이버 영화 리뷰 크롤링(Crawling) By Selenium | Seungho's Blog

2. xpath 문법

https://velog.io/@mjhuh263/TIL-23-HTML-XPATH-%EB%AC%B8%EB%B2%95%EA%B3%BC-selenium%EC%97%90-XPATH-%EC%9D%B4%EC%9A%A9%ED%95%98%EA%B8%B0

TIL #23 : [HTML] XPATH 문법과 selenium에 XPATH 이용하기

https://wkdtjsgur100.github.io/selenium-xpath/

(python) selenium에서 xpath를 이용해 크롤링 하기

 

3. 이미지 크롤링

https://youngq.tistory.com/7?category=764296 

파이썬 웹 크롤링 초급(with, Selenium) (2/2)

 

 

4. 그외

https://wayhome25.github.io/python/2017/04/25/cs-27-crawling/

 

 

# 크롤링 모듈 임포트
from selenium import webdriver
from bs4 import BeautifulSoup
import pandas as pd
import time
import csv
import urllib.request
import openpyxl

wb = openpyxl.Workbook()


# 한섬 홈페이지 접속
browser = webdriver.Chrome('chromedriver.exe')
url = 'http://www.thehandsome.com/ko/c/ou_we014/ou#1_0_0_0_0_641_0_0_0'
browser.get(url)

count=1

brand=[]
productname=[]
price=[]
productid=[]
longdetail=[]
shortdetail=[]
productcolor=[]
productsize=[]
productimage=[]
productlist=wb.active
columns = ['id','brand','sex','maincategoryId','subcategoryId','title','price','longdetail','shortdetail','color','size','imagePath']
productlist.append(columns)

//2페이지까지만
for k in range(2):
    print('page')
    print(k+1)
    time.sleep(4)
    
    //페이지 클릭
    locates=f'//*[@id="bodyWrap"]/div[2]/div[2]/span/a[{k+1}]'
    page = browser.find_element_by_xpath(locates)
    page.send_keys('\n')
    url = browser.current_url
    
    item_xpath = []
    
    //페이지마다 12개의 상품 클릭
    for i in range(12):
        locates=f'//*[@id="listBody"]/li[{i+1}]/div'
        item_xpath.append(locates)

    for xpath in item_xpath:
        time.sleep(6)
        browser.find_element_by_xpath(xpath).click()
        time.sleep(3)


        html = browser.page_source
        soup = BeautifulSoup(html, 'html.parser')


		//상품 상세 정보 크롤링
        locates='//*[@id="contentDiv"]/div[1]/div[1]/h4/div/a[1]'
        brand=browser.find_element_by_xpath(locates).text
        locates='//*[@id="contentDiv"]/div[1]/div[1]/h4/span'
        productname=browser.find_element_by_xpath(locates).text
        locates='//*[@id="contentDiv"]/div[1]/div[1]/p[1]/span[1]'
        price=browser.find_element_by_xpath(locates).text
        price=price[1:]
        locates='//*[@id="contentDiv"]/div[1]/div[1]/div[2]/strong/span'
        productid=browser.find_element_by_xpath(locates).text
        locates='//*[@id="contentDiv"]/div[1]/div[1]/div[2]/div/p'
        longdetail=browser.find_element_by_xpath(locates).text
        locates='//*[@id="contentDiv"]/div[1]/div[1]/div[2]/li'
        shortdetail=browser.find_element_by_xpath(locates).text
        locates='#contentDiv > div.info > div:nth-child(5) > ul > li:nth-child(1) > div > ul > li'
        colors=soup.select(locates)



		//색상 여러개인 경우 색상마다 색상코드와 이미지저장
        j=1;
        for color in colors:
            style = color.select('a')[0]['style']
            stylesplit = style.split()
            for word in stylesplit:
                if 'background' in word:
                    productcolor = word[12:18]

            print(productcolor)
            locates=f'//*[@id="contentDiv"]/div[1]/div[4]/ul/li[1]/div/ul/li[{j}]/a'
            colorimage=browser.find_element_by_xpath(locates)
            colorimage.send_keys('\n')
            time.sleep(1)

            locates='//*[@id="imageDiv"]/ul/li[1]/img'
            productimage=browser.find_element_by_xpath(locates)
            productimage=productimage.get_attribute('src')

            urllib.request.urlretrieve(productimage,'../img/woman/top/neat/'+str(count)+'.jpg')


			//size여러개인 경우 사이즈마다 데이터 저장
            locates = '#contentDiv > div.info > div:nth-child(5) > ul > li:nth-child(2) > span.txt > ul > li'
            sizes=soup.select(locates)
            for size in sizes:
                s = size.select('a')[0].text
                s = s.split()
                productsize = s[0]
                print(productsize)
                productlist.append([productid,brand,'w','2','10',productname,price,longdetail,shortdetail,productcolor,productsize,count])


            j=j+1

            
            count=count+1;

        time.sleep(2)
        
        browser.get(url)
        
        time.sleep(5)

        locates=f'//*[@id="bodyWrap"]/div[2]/div[2]/span/a[{k+1}]'
        page = browser.find_element_by_xpath(locates)
        page.send_keys('\n')

        
   
    
   
   
    
//엑셀 저장
wb.save('../product/woman/top/neat/woman_top_neat.xlsx')