#!/usr/bin/env python3.2
# -*- coding: utf-8 -*-

#For uploading files from names.voa.gov to commons

from bs4 import BeautifulSoup
import requests
from subprocess import call
import os.path
import traceback
from PyRWiki import Wiki #Requests based wrapper for api
from p import p

DEBUG=False

#http://stackoverflow.com/questions/1752662/beautifulsoup-easy-way-to-to-obtain-html-free-contents
#http://stackoverflow.com/questions/10993612/python-removing-xa0-from-string
#http://stackoverflow.com/questions/2077897/substitute-multiple-whitespace-with-single-whitespace-in-python
def textOf(soup):
    return ' '.join(''.join(soup.findAll(text=True)).replace('\xa0', ' ').strip().split())

#make first letter of word Upper if after space or "-", only single space/-
def fixname(oldname):
    oldname=oldname.strip()
    fixed=""
    lastwasspace=True#Make first letter upper
    for i in oldname:
        if lastwasspace:
            fixed += i.upper()
            lastwasspace=False
        else:
            if i == " " or i == "-":
                lastwasspace = True
            else:
                lastwasspace = False
            fixed += i.lower()
    return fixed

def log(stuff):
    print(stuff)

#Log in
commons = Wiki("https://commons.wikimedia.org/w/api.php","Smallbot")
commons.login('Smallbot',p.bP)
commons.setEditToken()

counter= 2 #starts at 2
log('Checking for last id.')
if os.path.isfile('last.txt'):
    with open('last.txt', 'r') as content_file:
        counter = int(content_file.read())
        log('Last id found: ' + str(counter))
else:
    log('No prior id found. Starting at 2.')

session=requests.session()
if DEBUG:
    session.proxies = {'http': 'http://localhost:8888'}
session.headers = {'Referer': 'https://commons.wikimedia.org/wiki/Commons:Batch_uploading/VOA_pronunciation_sound_files'}

lastsuccess=counter
reached404=0
try:
    while reached404 < 25: # up to 25 can be skipped
        r = session.get('http://names.voa.gov/modal.phrasedetail.php?id=' + str(counter))

        #if r.status_code == 404:
        if "Cannot find the requested name" in r.text:
            reached404 += 1
            log('404 reached for ' + str(counter))
        else:
            reached404=0#reset 404 counter
            lastsuccess=counter
            soup=BeautifulSoup(r.content)
            soupbody=soup.select('div.modal-body')[0]
            if textOf(soupbody) != "How do you say ?":
                name=textOf(soupbody.select("h2")[0])[15:-1] # remove "How do you say" and '?'
                name=fixname(name)
                pronounce= textOf(soupbody.select('p')[0])
                region=textOf(soup.select('h4')[0].findNext('p'))#('h4 + p')[0]) #Adjacent sibling selector
                if textOf(soup.select('h4')[0]) != 'Region':
                    region=''

                r=session.get('http://names.voa.gov/sounds/' + str(counter) + '.mp3')
                r.raise_for_status() #should be no errors

                log('---------------------------------')
                log('ID: ' + str(counter))
                log('Name: ' + name)
                log('Pronounce: ' + repr(pronounce))
                log('Region: ' + region)
                log(str(len(r.content)) + ' bytes')

                with open('data.mp3','wb') as voamp3:
                    voamp3.write(r.content)

                filedesc="{{Information\n" +\
                            "|description= {{VOA pronunciation|term=" + name + "|region=" + region + "|transliteration=" +  pronounce + "}}\n" +\
                            "|date= 2013\n" +\
                            "|source= VOA pronunciation guide: [http://names.voa.gov/modal.phrasedetail.php?id="  + str(counter) + " " + name + "]\n" +\
                            "|author= Jim Tedder\n" +\
                            "|permission= {{PD-USGov-VOA}}\n" +\
                            "|other_versions=\n" +\
                            "}}\n"
                if os.path.exists('data.ogg'):
                    os.remove('data.ogg')
                #call(['avconv', '-i', 'data.mp3', '-acodec', 'libvorbis', '-aq', '7', 'data.webm'])
                call(['avconv', '-i', 'data.mp3', '-acodec', 'libvorbis', '-aq', '7', 'data.ogg']) #use .ogg instead

                if region != '':
                    region = ' from ' + region
                commons.upload(title="En-us-" + name + region + ' pronunciation (Voice of America).ogg',
                               filelocation='data.ogg',
                               text=filedesc,
                               comment='[[Commons:Bots/Requests/Smallbot 9]]: Uploading Voice of America pronunciation files from http://names.voa.gov',
                               uploadifduplicate=False)
                #TODO-upload data.webm as file
            else:
                log('Empty at ' + str(counter))
        counter += 1

except:
    traceback.print_exc()
finally: #
    with open('last.txt','w') as lastfp:
        lastfp.write(str(lastsuccess))
    log('Done.')