import json

from utils.spider import Walmart
from utils.proxy import start_session
from utils.base import Base, engine
from utils.util import log

Base.metadata.create_all(engine)
gateway, session = start_session('https://www.walmart.com/ip/Wonder-Nation-Girls-Peace-2-Backpack/731463013')

with open('skus.json', 'r') as f:
    urls = json.load(f)

skus = [ele['url'].split('/')[-1] for ele in urls]

for sku in skus:
    p = Walmart(sku, session)
    p.get_product_reviews(sku, 1)
    log.info(f'{sku} scrapped successfully')
