"""Utility to bulk load leads.





"""

import csv
import os
from pprint import pprint


def bulkload(dataFileName):
    """Load in data from a correctly prepared CSV file.
    
    Not suitable for web use, checking is not robust enough.
    """
    from models import Lead
    rows = []
    for line in csv.reader(file(dataFileName)):
        rows.append(line)
    print 'slurped in', len(rows), 'rows'
    
    fieldNames = [f.lower() for f in rows[0]]
    data = rows[1:]
    count = 0
    for row in data:
        defined = {}
        for key, value in zip(fieldNames, row):
            if key == 'role':
                key = 'role_id'   #foreign key
            elif key == 'language':
                key = 'language_id'
                
            if value:  #only want non-null values
                defined[key] = value
        
        if defined.has_key('email'):
            try:
                lead = Lead.objects.filter(email=defined['email'])[0]
                print 'found lead: %s' % lead
            except:
                lead = Lead()
                lead.__dict__.update(defined)
                print 'new lead: %s' % lead
            try:
                #lead.save()
                count += 1
                print 'saved', lead.email
            except Exception, e:
                pprint(defined)
                raise
    print 'saved %d leads' % count
    


def strippedContent(textBlock):
    """Return array of lines which are not blank, comments or hash.
    
    Each qualifying line is split on tabs.
    
    >>> txt = '  \\n  \\none\\ttwo  \\n\\n  \\n'
    >>> strippedContent(txt)
    [['one', 'two']]
    >>>
    
    
    """
    
    out = []
    for line in textBlock.split('\n'):
        line = line.strip()
        if line.startswith('#'):
            continue
        if not line:
            continue
        words = line.split('\t')
        #strip any trailing/leading spaces
        words = [x.strip() for x in words]
        out.append(words)
    return out
 
import string   
_allowed = string.lowercase + string.digits

def canonicalFieldName(txt):
    "Try to extract a simple field name"
    if type(txt) is type(u''):
        txt = txt.encode('utf8')
    txt = txt.strip().lower()
    out = ''
    for ch in txt:
        if ch in _allowed:
            out += ch
    return out
    
def matchColumnNames(supplied, expected):
    """Returns updateable column names - ok and missing
    
    Aims to forgive errors in case, funny characters etc.
    """
    
    used = {}  #theirs to ours
    unused = []

    canonicals = {}
    for name in expected:
        key = canonicalFieldName(name)
        canonicals[key] = name
    
    for sup in supplied:
        key = canonicalFieldName(sup)
        if canonicals.has_key(key):
            used[sup] = canonicals[key]
        else:
            unused.append(sup)
    return (used, unused)
    


def process(listOfLists, expectedFields, requiredFields=[]):
    """Extract dictionaries for all expected fields.
    
    This is a first phase of validation.  If they have no required
    columns we can reject the batch outright.  If it's OK at this
    level, process after with Django forms.
    """
    colNames = listOfLists[0]
    data = listOfLists[1:]
    #usedCols maps their column names to our ones.   'First Name' -> 'first_name'
    usedCols, unusedCols = matchColumnNames(colNames, expectedFields)
    succeeded = True
    problems = []
    rows = []
    if not usedCols:
        problems.append("No valid columns found.  Expected columns are: %s" % (",".join(expectedFields)))
    else:
        canonicalUsedCols = set(usedCols.values())
        for req in requiredFields:
            canreq = canonicalFieldName(req)
            if canreq not in canonicalUsedCols:
                problems.append('Required column "%s" not found' % req)
    
    if problems:
        succeeded = False
    else:
        for (j, row) in enumerate(data):
            d = {}
            try:
                for (i, value) in enumerate(row):
                    colName = colNames[i].encode('utf8')
                    if usedCols.has_key(colName):
                        key = usedCols[colName]
                        d[key] = value
            except:
                succeeded = False
                problems.append("Wrong number of columns on row %s" % (j+1))
            rows.append(d)

    return (succeeded, rows, problems)


    


if __name__=='__main__':
    import doctest
    doctest.testmod()
