파이썬 병렬화 질문 드립니다
글쓴이: cldnjs2085 / 작성시간: 목, 2017/11/09 - 5:17오후
파이썬으로 이미지를 다운로드하려고 합니다. 그런데 밑에 이미지를 다운하는 함수 download_album_art()가 너무 오래걸리는거같아서 시간을 재어보니 약 40초정도 소요되더군요. 병렬화를 사용할수 있을것같은데 어떻게 하는지를 모르겠네요. 병렬화의 방법과 혹시 안된다면 속도를 향상시킬만한 방법을 알려주세요ㅜㅜ
import os
import shutil
import requests
from bs4 import BeautifulSoup
from urllib import request
URL = 'https://music.bugs.co.kr/chart/track/day/total'
PATH = os.getcwd() + '/static/images/'
# Scrapping html code
def get_html(target_url):
_html = ""
response = requests.get(target_url)
if response.status_code == 200:
_html = response.text
return _html
# parse image url and save in list
def get_image_url():
html = get_html(URL)
soup = BeautifulSoup(html, 'html.parser')
img_url = []
for image in soup.select('a.thumbnail > img'):
if image.has_attr('src'):
img_url.append(image.get('src'))
else:
continue
return img_url
# download album art in static/images directory
def download_album_arts():
images = get_image_url()
for i in range(0, 100):
url = images[i]
file_name = PATH + str(i + 1) + '.png'
request.urlretrieve(url, file_name)
# delete all album art
def delete_album_art():
path = os.getcwd() + '/static/images'
if os.path.exists(path):
shutil.rmtree(path)
os.mkdir(path)
else:
os.mkdir(path)Forums:
직접병렬화 하지 않아도 됩니당
multiprocessing 모듈을 이용해서 해도 되지만, joblib이라는 스레드 기반 병렬화 라이브러리가 있습니다. 제가 고쳐서 실험했을때는 11초 -> 5초 정도로 실행시간이 줄더군요 ㅎㅎ..
import os import shutil import requests from bs4 import BeautifulSoup from urllib import request from joblib import Parallel, delayed URL = 'https://music.bugs.co.kr/chart/track/day/total' PATH = os.getcwd() + '/static/images/' # Scrapping html code def get_html(target_url): _html = "" response = requests.get(target_url) if response.status_code == 200: _html = response.text return _html # parse image url and save in list def get_image_url(): html = get_html(URL) soup = BeautifulSoup(html, 'html.parser') img_url = [] for image in soup.select('a.thumbnail > img'): if image.has_attr('src'): img_url.append(image.get('src')) else: continue return img_url # download album art in static/images directory def download_album_arts(index, image): url = image file_name = PATH + str(index) + '.png' request.urlretrieve(url, file_name) # delete all album art def delete_album_art(): path = os.getcwd() + '/static/images' if os.path.exists(path): shutil.rmtree(path) os.mkdir(path) else: os.mkdir(path) if __name__ == "__main__": Parallel(n_jobs=-1)(delayed(download_album_arts)(index, image) for index, image in enumerate(get_image_url()))댓글 달기