#!/usr/bin/env python3
#coding=utf8

# soffice --convert-to csv *.ods # only converts first worksheet

# Debian Package Dependencies
# python3-bs4

import sys
import pycurl
import io
import bs4
import re
import csv
import json
import os
#import copy
import optparse
import pymysql.cursors
import MySQLdb
import yaml
import traceback
#import lockfile
import psutil

tesco_offer_any_x_for_y = re.compile(u'Any ([0-9]+) for £([0-9.]+)')
tesco_offer_half_price = re.compile(u'Half Price')
tesco_offer_better_than_half_price = re.compile(u'Better Than Half Price')
tesco_offer_save = re.compile(u'Save')
tesco_clubcard_price = re.compile(u'(.*) Clubcard Price')
iceland_offer_x_for_y = re.compile(u'([0-9]+) for £([0-9.]+)')

def curl(url, headers = [], postdata = None):
  #useragent = 'Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0'
  useragent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0'
  #print(url)
  # v http://pycurl.io/docs/latest/quickstart.html
  buffer = io.BytesIO()
  c = pycurl.Curl()
  c.setopt(c.URL, url)
  c.setopt(c.WRITEDATA, buffer)
  c.setopt(c.USERAGENT, useragent) # required for Tesco/Savers/Asda
  c.setopt(c.HTTPHEADER, headers)
  c.setopt(c.ENCODING, 'gzip, deflate') # required for Savers
  if postdata:
    c.setopt(c.POSTFIELDS, postdata)
  #c.setopt(c.PROXY, '127.0.0.1')
  #c.setopt(c.PROXYPORT, 8080)
  c.setopt(c.SSL_VERIFYPEER, 0)   
  c.setopt(c.SSL_VERIFYHOST, 0)
  c.setopt(c.FOLLOWLOCATION, True)
  c.setopt(c.TIMEOUT, 10)
  c.perform()
  c.close()
  return buffer.getvalue()

def scrape_asda(url):
  #print(url)
  #https://groceries.asda.com/product/washing-machine-cleaners/dettol-washing-machine-cleaner-single-use/910002053945
  sku = url.split('/')[-1]
  #print(sku)
  #url = 'https://groceries.asda.com/p13nservice/recommendations?storeId=4565&placement=item_page.pdp1&skuId=%s' % sku

  #curl 'https://groceries.asda.com/api/bff/graphql' \
  #-H 'request-origin: gi' \
  #-H 'Content-Type: application/json' \
  #--data-raw '{"requestorigin":"gi","contract":"web/cms/product-details-page","variables":{"user_segments":[],"store_id":"4565","type":"content","payload":{"page_id":"1000105873612","page_type":"productDetailsPage"}}}'
  url = 'https://groceries.asda.com/api/bff/graphql'
  payload = {
          "contract":"web/cms/product-details-page",
          "variables":{
              "store_id":"4565",
              "user_segments":[],
              "payload":{
                  "page_id":sku,
                  "page_type":"productDetailsPage",
                  },
              },
          }
  #print(json.dumps(payload))
  body = curl(url, ['request-origin: gi','Content-Type: application/json'], json.dumps(payload))
  #body = curl(url, ['request-origin: gi','Content-Type: application/json'], '{"requestorigin":"gi","contract":"web/cms/product-details-page","variables":{"store_id":"4565","type":"content","payload":{"page_id":"%s","page_type":"productDetailsPage"}}}' % sku)
  j = json.loads(body)
  #print(json.dumps(j,indent=2))

  price = j['data']['tempo_cms_content']['zones'][0]['configs']['products']['items'][0]['price']['price_info']['price'][1:]

  promo_unit_price = None
  promo_quantity = None  
  if j['data']['tempo_cms_content']['zones'][0]['configs']['products']['items'][0]['promotion_info'][0]['linksave']:
    promo_offer_type = j['data']['tempo_cms_content']['zones'][0]['configs']['products']['items'][0]['promotion_info'][0]['linksave']['promo_offer_type']
    promo_detail = j['data']['tempo_cms_content']['zones'][0]['configs']['products']['items'][0]['promotion_info'][0]['linksave']['promo_detail']
    #print('scrape_asda %s - promo_detail: %s' % (url, promo_detail))
    promo_quantity = j['data']['tempo_cms_content']['zones'][0]['configs']['products']['items'][0]['promotion_info'][0]['linksave']['promo_quantity']
    promo_value = j['data']['tempo_cms_content']['zones'][0]['configs']['products']['items'][0]['promotion_info'][0]['linksave']['promo_value']
    if promo_offer_type == '8': # Buy x Save %
      promo_percent_off = promo_value[0:-1]
      promo_unit_price = (float(price) / 100) * (100 - float(promo_percent_off))
    elif promo_offer_type == "15": # x for y
      #print(promo_value[1:])
      promo_unit_price = float(promo_value[1:]) / float(promo_quantity)
    else:
      print('scrape_asda: Unknown promo_offer_type: %s, promo_detail: %s' % (promo_offer_type, promo_detail))

  return ('Asda', price, promo_unit_price, promo_quantity)

# eg
# - 40p -> 0.4
# - £3.50 -> 3.5
def price_to_pounds_sterling(price):
  if re.compile('p$').search(price):
    # convert pence sterling to pounds sterling
    price = '0.' + price[0:-1]
  else:
    price = price[1:]
  try:
    #print(price)
    float(price)
  except Exception as e:
    print(e)
  return price

def scrape_tesco(url):
  #https://www.tesco.com/groceries/en-GB/products/271246162
  print('scrape_tesco - url: %s' % url)
  body = curl(url, ['Accept-Language: en'])

  # Body is a string in some encoding.
  # In Python 2, we can print it without knowing what the encoding is.
  #print(body)
  # ^ http://pycurl.io/docs/latest/quickstart.html
  soup = bs4.BeautifulSoup(body, 'html.parser')

#  print 'scrape_tesco(soup)'
  #divs = soup.find_all('div', class_='price-per-sellable-unit')
  #if len(divs) < 1:
  #  print('len(divs) < 1')
  #  return (None, None, None, None)
  #spans = divs[0].find_all('span', class_="value")
  #if len(spans) < 1:
  #  return (None, None, None, None)
  #price = spans[0].get_text()
  #print('scrape_tesco - A price: %s' % price)
  divs = soup.find_all('div', class_='ddsweb-price__container')
  price = price_to_pounds_sterling(divs[0].find_all('p')[0].text)
  offer_unit_price = False
  offer_quantity = False
  find_all_offer_text = soup.find_all('span', class_='offer-text')
  if find_all_offer_text != []:
    offer = find_all_offer_text[0].get_text()
    print('scrape_tesco - url: %s, offer: %s' % (url, offer))
    offer_understood = False
    re_match_tesco_offer_any_x_for_y = tesco_offer_any_x_for_y.match(offer)
    re_match_tesco_clubcard_price = tesco_clubcard_price.match(offer)
    if re_match_tesco_offer_any_x_for_y:
      offer_understood = 'tesco_offer_any_x_for_y'
#      print re_match.groups()
      offer_quantity = float(re_match_tesco_offer_any_x_for_y.group(1))
      offer_price = float(re_match_tesco_offer_any_x_for_y.group(2))
      offer_unit_price = offer_price / offer_quantity
      #print 'offer_quantity: %s' % offer_quantity
      #print 'offer_price: %s' % offer_price
      #print 'offer_unit_price: %s' % offer_unit_price
    elif tesco_offer_half_price.match(offer):
      offer_understood = 'tesco_offer_half_price'
    elif tesco_offer_better_than_half_price.match(offer):
      offer_understood = 'tesco_offer_better_than_half_price'
    elif tesco_offer_save.search(offer):
      offer_understood = 'tesco_offer_save'
    elif re_match_tesco_clubcard_price:
      offer_understood = 'tesco_clubcard_price'
      #print('Tesco Clubcard Price: %s - %s - %s' % (url, price, re_match.group(1)))
      clubcard_price = re_match_tesco_clubcard_price.group(1)
      clubcard_price = price_to_pounds_sterling(clubcard_price)
      if clubcard_price:
        price = clubcard_price
      #if re.compile('p$').search(clubcard_price):
      #  # convert pence sterling to pounds sterling
      #  clubcard_price = '0.' + clubcard_price[0:-1]
      #else:
      #  clubcard_price = clubcard_price[1:]
      #try:
      #  float(clubcard_price)
      #  price = clubcard_price
      #except Exception as e:
      #  print(e)
    if offer_understood == False:
      print('scrape_tesco - Offer not understood: %s' % offer)
    else:
      print('scrape_tesco - Offer understood: %s - %s' % (offer_understood, offer))
      if offer_unit_price and offer_quantity:
        print('scrape_tesco - offer_unit_price: %s, offer_quantity: %.0f' % (offer_unit_price, offer_quantity))
    #print('scrape_tesco - B price: %s' % price)
  return ('Tesco', price, offer_unit_price, offer_quantity)

def scrape_morrisons_old(url):
  # https://groceries.morrisons.com/products/ktc-chopped-tomatoes-212082011
  # https://groceries.morrisons.com/webshop/api/v1/products/212082011
  sku = url.split('-')[-1]
  url = 'https://groceries.morrisons.com/webshop/api/v1/products/%s' % sku
  print('scrape_morrisons - url: %s' % url)
  body = curl(url)
  print(body)
  j = json.loads(body)
  #print(json.dumps(j,indent=2))
  price = j['price']['current']
  offer_quantity = None
  offer_unit_price = None
  if 'offer' in j:
    offer_text = j['offer']['text']
    groups = re.compile('Buy (.*) for (.*)').search(offer_text)
    if groups:
      offer_price = float(groups[2][1:])
      offer_quantity = float(groups[1])
      offer_unit_price = offer_price / offer_quantity
    else:
      print('scrape_morrisons - unknown offer: %s' % offer_text)
  return ('Morrisons', price, offer_unit_price, offer_quantity)

def scrape_morrisons(url):
  # https://groceries.morrisons.com/products/fruit-shoot-apple-blackcurrant-kids-juice-drink/100163721
  print('scrape_morrisons - url: %s' % url)
  body = curl(url)
  soup = bs4.BeautifulSoup(body, 'html.parser')
  price = price_to_pounds_sterling(soup.find('div', {'data-test':"price-container"}).find('span').text)
  return('Morrisons', price, None, None)

def scrape_lidl(url):
  # https://www.lidl.co.uk/p/w5/w5-lemon-all-purpose-cleaner/p859
  body = curl(url)
  soup = bs4.BeautifulSoup(body, 'html.parser')
  #print(soup)
  sku = url.split('/')[-1][2:]
  #divs = soup.findAll(attrs={"data-id": sku})
  div = soup.find('div', class_='attributebox')
  price = (div['data-price'][2:])
  return('Lidl', price, None, None)
  
def scrape_savers(url):
  # https://www.savers.co.uk/Health-%26-Wellbeing/Incontinence/Female-Incontinence/Tena-Lady-Maxi-Night-Sanitary-Towels-6-Pack/p/255755
  body = curl(url,['Accept-Language: en'])
  soup = bs4.BeautifulSoup(body, 'html.parser')
  scripts = soup.findAll('script', type="application/ld+json")
  j = json.loads(scripts[1].contents[0])
  #print(json.dumps(j, indent=2))
  price = j['offers']['price']
  return('Savers', price, None, None)

def scrape_iceland(url):
  body = curl(url)
  soup = bs4.BeautifulSoup(body, 'html.parser')
  scripts = soup.findAll('script', type="application/ld+json")
  #print(len(scripts))
  #print(scripts)
  if not scripts:
    return('Iceland', None, None, None)
  #y = yaml.load(scripts[2].contents[0])
  y = yaml.load(scripts[0].contents[0])
  price = y['offers']['price']
  offer = soup.findAll('div', class_='two-line-offer')
  offer_unit_price = None
  offer_quantity = None
  if offer:
    s = ' '.join([x.strip() for x in offer[0].contents[1].contents if isinstance(x, bs4.element.NavigableString)])
    print('scrape_iceland: - %s' % s)
    re_match_iceland_offer_x_for_y = iceland_offer_x_for_y.match(s)
    if re_match_iceland_offer_x_for_y:
      (offer_quantity, offer_price) = re_match_iceland_offer_x_for_y.groups()
      offer_unit_price = float(offer_price) / float(offer_quantity)
  return('Iceland', price, offer_unit_price, offer_quantity)

def scrape_sainsburys(url):
  product = url.split('/')[-1]
  log(2, product)
  product_url = 'https://www.sainsburys.co.uk/groceries-api/gol-services/product/v1/product?filter[product_seo_url]=gb%2Fgroceries%2F' + product
  body = curl(product_url)
  log(2, body)
  j = json.loads(body)
  log(2, json.dumps(j, indent=2))
  price = j['products'][0]['retail_price']['price']
  log(2, price)
  offer_unit_price = None
  offer_quantity = None
  return("Sainsbury's", price, offer_unit_price, offer_quantity)

def scrape_bm(url):
  body = curl(url)
  soup = bs4.BeautifulSoup(body, 'html.parser')
  price = float(soup.find('meta', property="og:price:amount")['content'])
  return('B&M', price, None, None)

def scrape(url, shop = None):
  try:
    result = (None, None, None, None)
    #print('v scrape - shop: %s)' % shop)
    if False:
      pass
    elif url.startswith('https://www.tesco.com') and (not shop or shop == 'tesco'):
      result = scrape_tesco(url)
    elif url.startswith('https://groceries.asda.com') and (not shop or shop == 'asda'):
      result = scrape_asda(url)
    elif url.startswith('https://groceries.morrisons.com') and (not shop or shop == 'morrisons'):
      result = scrape_morrisons(url)
    elif url.startswith('https://www.lidl.co.uk') and (not shop or shop == 'lidl'):
      result = scrape_lidl(url)
    elif url.startswith('https://www.savers.co.uk') and (not shop or shop == 'savers'):
      result = scrape_savers(url)
    elif url.startswith('https://www.iceland.co.uk') and (not shop or shop == 'iceland'):
      result = scrape_iceland(url)
    elif url.startswith('https://www.sainsburys.co.uk') and (not shop or shop == 'sainsburys'):
      result = scrape_sainsburys(url)
    elif url.startswith('https://www.bmstores.co.uk') and (not shop or shop == 'bm'):
      result = scrape_bm(url)
    else:
      print('No enabled scraper for URL: %s' % url)
      result = (None, None, None, None)
    #print('^ scrape - shop: %s)' % shop)
    return (result[0], result[1], result[2], result[3], None)
  except Exception as e:
    print('scrape exception: %s - %s' % (url, e))
    print(e.__traceback__)
    print(sys.exc_info())
    traceback.print_exc(file=sys.stdout)
    result = (None, None, None, None, e)

def d_sort_element(d):
  #print('d_sort_element')
  #print(d)
  normalized_price = d['unit_price']
  normalized_offer_unit_price = None
  if d['offer_unit_price'] and d['pack_quantity']:
    normalized_offer_unit_price = d['offer_unit_price'] / d['pack_quantity']
  retval = normalized_offer_unit_price if normalized_offer_unit_price else normalized_price
  #print(retval)
  #print('!d_sort_element')
  return retval

def process_csv(incsv, outcsv = None, shop_filter = None, outhtml = None):
  if outcsv == None:
    outcsvfh = sys.stdout
  else:
    outcsvfh = open(outcsv, 'w')
  if outhtml == None:
    outhtmlfh = sys.stdout
  else:
    outhtmlfh = open(outhtml, 'w')
  headers = ['Product', 'Rank', 'Shop', 'Price','Unit Price','Pack Quantity','Offer Unit Price','Offer Quantity','Offer Total Price','Offer Total Quantity', 'Page']
  csvwriter = csv.DictWriter(outcsvfh, headers)
  csvwriter.writeheader()
  print('<html><head><title>price_scraper_spreadsheet.py</title><style>tr {outline: solid thin}</style></head><body><table><tr>', file=outhtmlfh)
  for header in headers:
    print('<th>' + header + '</th>', file=outhtmlfh)
  print('</tr>', file=outhtmlfh)
  #print(incsv)
  with open(incsv) as incsvfh:
    csvreader = csv.DictReader(incsvfh)
    exceptions = []
    for row in csvreader:
      #cheapest = []
      #cheapest_offer = []
      d_list = []
      for combine in row['Pages'].split("\n"):
        page = combine.split('|')[0]
        pack_quantity = float(combine.split('|')[1]) if len(combine.split('|')) == 2 else 1
        retval = scrape(page, shop_filter)
        #print(retval)
        if not retval:
          continue
        if retval[4]:
          exceptions.append({page, retval[4]})
          continue
        #print(retval)
        # v check price is well-formed
        #print(type(price))
        if type(retval[1]) == str:
          if re.compile('[^0-9.]').search(retval[1]):
            assert('Malformed price: %s' % retval[1])
        # ^ check price is well-formed
        price = float(retval[1]) if retval[1] else None
        shop = retval[0]
        #print('%s - %s - %s' % (row['Product'], shop, page))
        offer_quantity = float(retval[3]) if retval[3] else None
        offer_unit_price = float(retval[2]) if retval[2] else None
        d = {
            'page': page,
            'shop': shop,
            'price': price,
            'pack_quantity': pack_quantity,
            'unit_price': price / pack_quantity if price else None,
            'offer_quantity': offer_quantity,
            'offer_unit_price': offer_unit_price,
            'offer_total_quantity': offer_quantity * pack_quantity if offer_quantity and pack_quantity else None,
            'offer_total_price': offer_unit_price * offer_quantity if offer_unit_price and offer_quantity else None,
        }
        #print(json.dumps(d,indent=2))
        if d['price'] != None and d['unit_price'] != None:
          d_list.append(d)
        #if (price != None):
        #  if cheapest == [] or d['unit_price'] < cheapest[0]['unit_price']:
        #    cheapest = [copy.deepcopy(d)]
        #  elif d['unit_price'] == cheapest[0]['unit_price']:
        #    cheapest.append(copy.deepcopy(d))
        #  if d['offer_unit_price']:
        #    if cheapest_offer == [] or d['offer_unit_price'] < cheapest_offer[0]['offer_unit_price']:
        #      cheapest_offer = [copy.deepcopy(d)]
        #    elif d['offer_unit_price'] == cheapest_offer[0]['offer_unit_price']:
        #      cheapest_offer.append(copy.deepcopy(d))
      #for cheapest_offer_item in cheapest_offer:
      #  found = False
      #  for cheapest_item in cheapest:
      #    if cheapest_offer_item['page'] == cheapest_item['page']:
      #      found = True
      #    elif cheapest_item['unit_price'] < cheapest_offer_item['offer_unit_price']:
      #      found = True
      #    elif cheapest_item['offer_unit_price'] and cheapest_offer_item['offer_unit_price'] < cheapest_item['offer_unit_price']:
      #      found = True
      #  if found == False:
      #    cheapest.append(cheapest_offer_item)
      #for item in cheapest:
      d_list.sort(key=d_sort_element)
      last_sort_element = None
      for item in enumerate(d_list):
        print(json.dumps(item[1], indent=2))
        if last_sort_element and d_sort_element(item[1]) == last_sort_element:
          rank = rank
        else:
          rank = item[0]
          last_sort_element = d_sort_element(item[1])
        row_map = {
          'Product': row['Product'],
          'Page': item[1]['page'],
          'Shop': item[1]['shop'],
          'Price': item[1]['price'],
          'Unit Price': item[1]['unit_price'],
          'Pack Quantity': item[1]['pack_quantity'],
          'Offer Unit Price': item[1]['offer_unit_price'] if item[1]['offer_unit_price'] else '',
          'Offer Quantity': item[1]['offer_quantity'] if item[1]['offer_quantity'] else '',
          'Offer Total Price': item[1]['offer_total_price'] if item[1]['offer_total_price'] else '',
          'Offer Total Quantity': item[1]['offer_total_quantity'] if item[1]['offer_total_quantity'] else '',
          'Rank': rank + 1,
        }
        csvwriter.writerow(row_map)
        format_string = '<tr>' + ''.join(['<td>{' + x + '}</td>' for x in headers]) + '</tr>'
        print(format_string.format(**row_map), file=outhtmlfh)
      print(outhtmlfh, '</table></body></html')
      outhtmlfh.flush()
      outcsvfh.flush()
    for item in exceptions:
      print(item[0])
      print(item[1])
  outcsvfh.close()
  outhtmlfh.close()
  return None

def load_csv_to_db(csvfile, connection, cursor):
  with open(csvfile) as csvfh:
    csvreader = csv.DictReader(csvfh)
    for csvrow in csvreader:
      print(csvrow)
      if csvrow['Product'] != '':
        sql = 'SELECT * FROM product WHERE name = %s'
        print(cursor.mogrify(sql, (csvrow['Product'])))
        cursor.execute(sql, (csvrow['Product']))
        print(cursor.rowcount)
        for dbrow in cursor.fetchall():
          print(dbrow)
        sql = 'INSERT INTO product (name) values (%s)'
        cursor.execute(sql, (csvrow['Product']))
        insert_id = connection.insert_id()
        print(insert_id)
        for combine in csvrow['Pages'].split("\n"):
          page = combine.split('|')[0]
          pack_quantity = float(combine.split('|')[1]) if len(combine.split('|')) == 2 else 1
  connection.commit()
  return None

def process_db(options, connection, csvfile, shop = None):
  with connection.cursor() as cursor:
    if options.db_drop_tables:
      sql = 'DROP TABLE IF EXISTS product'
      cursor.execute(sql)
      sql = 'DROP TABLE IF EXISTS page'
      cursor.execute(sql)
      connection.commit()
    sql = '''CREATE TABLE IF NOT EXISTS product (
      id int auto_increment primary key,
      name text
    )'''
    cursor.execute(sql)
    sql = '''CREATE TABLE IF NOT EXISTS page (
      id int auto_increment primary key,
      product_id int,
      url text,
      pack_quantity double
    )'''
    cursor.execute(sql)
    connection.commit()
    if csvfile:
      load_csv_to_db(csvfile, connection, cursor)
  return None  

def convert_ods_to_csv(filename):
  os.system('soffice --convert-to csv %s' % filename)
  csvfile = '.'.join(filename.split('.')[0:-1] + ['csv'])
  return csvfile

def log(messagelevel, message):
  #print('log(%d, "%s")' % (messagelevel, message))
  global logfh
  global options
  if options.loglevel >= messagelevel:
    print(message)
    #traceback.print_stack()
    if logfh:
      print(message, file=logfh)

parser = optparse.OptionParser()
parser.add_option('-s', '--ods', dest='ods')
parser.add_option('-c', '--csv', dest='csv')
parser.add_option('-u', '--url', dest='url')
parser.add_option('-o', '--outcsv', dest='outcsv')
parser.add_option('--outhtml', dest='outhtml')
parser.add_option('-p', '--pidfile', dest='pidfile')
parser.add_option('--db_host', dest='db_host', default='localhost')
parser.add_option('--db_port', dest='db_port', type='int', default=3306)
parser.add_option('--db_username', dest='db_username')
parser.add_option('--db_password', dest='db_password')
parser.add_option('--db_name', dest='db_name')
parser.add_option('--db_drop_tables', dest='db_drop_tables', action="store_true")
parser.add_option('--shop', dest='shop')
parser.add_option('-f', '--logfile', dest='logfile')
parser.add_option('-l', '--loglevel', dest='loglevel', type="int", default=0)

(options, args) = parser.parse_args()

logfh = None
if options.logfile:
  logfh = open(options.logfile, 'a', buffering=1) # buffering=1 means line buffered

log(1, options)
log(1, args)

# argv[0] --ods document.ods --db... # convert document.ods into CSV then load into DB then process DB
# argv[0] --csv document.csv --db... # load document.csv into DB then process DB
# argv[0] --ods document.ods # convert document.ods into CSV then process CSV
# argv[0] --csv document.csv # process document.csv
# argv[0] --url ... # scrape url

if options.pidfile:
  try:
    with open(options.pidfile, 'r') as pidfh:
      pid = int(pidfh.read())
      if psutil.pid_exists(pid):
        log(0, 'pid exists: %d' % pid)
        sys.exit(1)
  except FileNotFoundError as e:
    pass
  with open(options.pidfile, 'w') as pidfh:
    pidfh.write(str(os.getpid()))
if options.db_host and options.db_port and options.db_username and options.db_password and options.db_name:
  csvfile = None
  if options.ods:
    csvfile = convert_ods_to_csv(options.ods)
  elif options.csv:
    csvfile = options.csv
  connection = pymysql.connect(host=options.db_host,
                               port=options.db_port,
                               user=options.db_username,
                               password=options.db_password,
                               db=options.db_name,
                               charset='utf8mb4',
                               cursorclass=pymysql.cursors.DictCursor,
  )
  try:
    process_db(options, connection, csvfile, options.shop)
  finally:
    connection.close()
elif options.url:
  log(0, scrape(options.url, options.shop))
elif options.ods:
  process_csv(convert_ods_to_csv(options.ods), options.outcsv, options.shop, options.outhtml)
elif options.csv:
  process_csv(options.csv, options.outcsv, options.shop, options.outhtml)
else:
  log(0, 'No option. Please specify one of url, ods, csv.')

sys.exit(0)