#----------------------------------------------------------------------
# Name:        rss.py
# Purpose:     Handles the RDF/RSS pages.
#
# Author:      Kristofer Goransson
#
# Created:     2003 Kristofer Goransson
# Copyright:   (c) 2003 
# Licence:     GPL
#---------------------------------------------------------------------- 
import feedparser
import time, string, os, cPickle, siteo
from htmlentitydefs import entitydefs

class RSS:
    def __init__(self, parent, stats):
        self.main = parent
        self.stats = stats
        self._loadUrls()
        self.tottime = 1800
        self.sitenum = 0
        self.dtime = 0
        self.chtime = 1

    #--------------------------------------------
    # Check methods
    #--------------------------------------------
    def check(self):
        #self.readdSites()       #hmm...whats this?
        while 1:
            time.sleep(10)
            if self.sites != []:
                self.chtime = self.tottime/len(self.sites)
                self.dtime += 10
                self.stats.addUptime(10)
                if self.dtime >= self.chtime:
                    self.doCheck()
                    self.dtime = 0

    def doCheck(self, manual=False):
        if not manual:
            site = self.sites[self.sitenum]
            self.main.output('Checking %s...'%site.getName(), 'white')
            self.itemCheck(site)
            if self.sitenum == (len(self.sites)-1):
                self.sitenum = 0
            else:
                self.sitenum += 1
        else:
            for site in self.sites:
                self.main.output('Checking %s...'%site.getName(), 'white')
                self.itemCheck(site, True)

        self.main.output('Check is done!', 'white')

    def itemCheck(self, site, manual=False):         
        if site.getAdded() != 0 or manual == True:
            self.stats.addCheck()
            site.addCheck()
            name = site.getName()
            url = site.getUrl()

            rssitems, encoding, link = self._getRss(url)
            if rssitems != []:
                
                if encoding != None:
                    site.setEncoding(encoding)
                
                site.setLink(link)
                if site.getCache():
                    diff = self._diffRss(site, rssitems[:])
                
                    if diff != []:
                        parsedrss = self._parseRss(site, diff)
                        for user in self.main.getUsers():
                            if user.checkRss(name):
                                if self.main.getUserShowOk(user) and not user.getPaused():
                                    self.main.sendIM(user.getJID(), parsedrss)
                                    time.sleep(1)
                else:
                    self.main.output('Checking %s for the first time'%name, 'white')
                site.setCache(rssitems)
                self.saveUrls()     #remember cache
            else:
                self.main.output('%s No items found!'%name, 'warning')

    def getDtime(self):
        return self.dtime

    def getCHtime(self):
        return self.chtime


    #--------------------------------------------
    # Parse RSS/RDF methods
    #--------------------------------------------
    def _parseRss(self, site, items):
        #name = site.getName()
        link = site.getLink()
        encoding = site.getEncoding()
        lines = "Website: %s\n" %link
        x = 0
        for item in items:
            x += 1
            line = ''
            if item.has_key('title'):
                title = item['title']
                line += '\n'+title+'\n'
            if item.has_key('date'):
                line += item['date']+'\n'
            if item.has_key('link'):
                line += item['link']+'\n'
            if item.has_key('description'):
                desc = item['description']
                
                fbreak = desc.find('\r\n\r\n')
                if fbreak != -1:
                    desc = desc[:fbreak]
                    
                fbreak = desc.find('\n\n')
                if fbreak != -1:
                    desc = desc[:fbreak]

                if len(desc) > 500:
                    desc = desc[:500]
                    desc = desc[:desc.rfind(' ')]
                    desc = desc+'...'
                desc = self._remHtmlEnt(desc)
                line += desc+'\n'
            lines += line.decode(encoding)
            if x == 3:      #####user setting?
                break
        return lines

    #Returns a dict with rdf data
    def _getRss(self, url):
        try:
            data = feedparser.parse(url)
            #print data
        except:
            self.main.output('Timeout %s'%url, 'red')
        else:
            if data['items'] != []:
                if data.has_key('channel'):
                    channel = data['channel']
                    link = channel['link']
                else:
                    link = 'N/A'
                    
                if data.has_key('encoding'):
                    enc = data['encoding']
                else:
                    enc = None
                return data['items'], enc, link
        return [], None, None

    #Diffs old and new rdf data for new items
    def _diffRss(self, site, newitems):
        rssitems = site.getCache()
        for item in rssitems:
            for nitem in newitems:
                if item['title'] == nitem['title']:
                    newitems.remove(nitem)
                    break
        return newitems
    
    #Removes some html entities
    def _remHtmlEnt(self, text):
        entities = ['amp;','quot;']
        text = text.replace('&','')
        for ent in entities:
            text = text.replace(ent,'')
        return text


    #--------------------------------------------
    #Misc site methods
    #--------------------------------------------
    
    #Test new url for valid RSS/RDF info
    #If everything went ok the page is added to the sitelist.
    def testItem(self, name, url, hidden, jid):
        for site in self.sites:
            if site.getName() == name:
                return 'Name already exists!'
            elif site.getUrl() == url:
                return 'Url already exists!'
            
        result, encoding, link = self._getRss(url)
        if result == []:
            return 'Error: Unable to parse %s %s.'%(name, url)

            #    self.main.output('Parse error! No encoding information found. Using default: utf-8', 'red')
            #    encoding = 'utf-8'
        else:
            if encoding == None:
                self.main.output('%s has no enconding information! Using default: latin-1'%url, 'warning')
                encoding = 'latin-1'
                
            site = siteo.Site(name,url,len(self.sites)+1, encoding, link, hidden, jid)
            parsedresult = self._parseRss(site, result)
            site.setCache(result)
            self.sites.append(site)
            self.saveUrls()
            return 'Succesfully parsed!\n%s\n\n%s is now available in the news feeds list'%(parsedresult, name)

    #Removes site from sitelist
    def delSite(self, site):
        self.sites.remove(site)
        x = 0
        for site in self.sites:     #renumbering
            x += 1
            site.setNumber(x)
        self.saveUrls()

    #Returns a site object, if name/num exists
    def getSite(self, name):
        try:
            num = int(name)
        except:
            num = None
            
        if num and num <= len(self.sites):
            return self.sites[num-1]
        else:
            for site in self.sites:
                if site.getName() == name:
                    return site
        return None

    def getSites(self):
        return self.sites


    #--------------------------------------------
    # Latest methods
    #--------------------------------------------

    #Returns latest news from a user sitelist
    def getLatestUser(self, user):
        newslist = []
        for name in user.getRss():
            site = self.getSite(name)
            if site:
                news = self.getLatest(site)
                newslist.append(news)
        return newslist

    #Returns latest news from a site
    def getLatest(self, site):
        if site.getCache():
            items = site.getCache()
            news = self._parseRss(site, items)  #get three news
        else:
            news = 'Website: %s\nNo news available.'%site.getName()
        return news.encode(site.getEncoding())

    #--------------------------------------------
    # Save/Load site methods
    #--------------------------------------------

    #Saves sites
    def saveUrls(self):        
        try:
            file = open('sites.dat','w')
        except:
            self.mine.output('Couldn\'t open sites.dat', 'red')
            sys.exit(0)
        cPickle.dump(self.sites, file)
        file.close()        
            

    def _loadUrls(self):
        self.main.output('Loading news feeds...', 'white')
        if os.path.isfile('sites.dat'):
            try:
                file = open('sites.dat','r')
            except IOError, strerror:
                self.main.output('Error opening sites.dat: '+strerror, 'red')
            else:
                self.sites = cPickle.load(file)
                file.close()
                empty = False
        else:
            self.sites = []
            empty = True

        #############
        #for site in self.sites:
        #    site.setEncoding()
        #self.saveUrls()
            
            
        """try:
            file = open('rssurls.txt','r')
        except:
            self.main.output('Couldn\'t open rssurls.txt', 'red')
            sys.exit(0)
        lines = file.readlines()
        urllist = []
        x = 0
        for line in lines[1:]:
            x += 1
            line = line.replace('\n','')
            line = line.replace('\r','')
            mark = line.find(':')
            name = line[0:mark]
            url = line[mark+1:]
            if url[0] == '!':
                hidden = True
                url = url[1:]
            else:
                hidden = False

            if empty:
                siteobj = siteo.Site(name,url,x,hidden)
                self.sites.append(siteobj)
            else:
                found = False
                for site in self.sites:
                    if name == site.getName():
                        found = True
                        break
                        
                if found == False:
                    siteobj = siteo.Site(name,url,x,hidden)
                    self.sites.append(siteobj)"""
        #if empty:
        #    self.saveUrls()
        self.main.output('Loaded %s news feed(s)'%len(self.sites), 'white')

    #--------------------------------------------
    # Misc methods
    #--------------------------------------------      

    def readdSites(self):
        for site in self.sites:
            name = site.getName()
            for user in self.main.getUsers():
                if user.checkRss(name):
                    site.addAdded()
            
    def getNextNum(self):
        return self.sitenum

    def getNextItem(self):
        return self.sites[self.sitenum]


    """def _timer(self, start):
        if start == 'start':
            self.starttime = time.clock()
        else:
            self.endtime = time.clock()
            difftime = str(self.endtime - self.starttime)
            return  difftime[:difftime.find('.')+2]"""