#!/usr/bin/python
# Copyright (c) 2015 SUSE Linux GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from pprint import pformat
import os, sys, re
import logging
import cmdln
from FileListExtractor import FileListExtractor, Channel
import pickle
import signal
import subprocess
import time

try:
    from xml.etree import cElementTree as ET
except ImportError:
    import cElementTree as ET

BLACKLIST = ['java-.*-ibm.*']
CHANNELS = None
CHANNELS_A = None
CHANNELS_B = None
EXCEPTIONS= None

SP = None

def service_pack(name):
    r = re.compile(".*(sle-module|sle-manager)", flags=re.IGNORECASE)
    if (not SP or r.match(name)):
        return name
    else:
        return re.sub(r'([_\-]12)([_\-:])',r'\1-%s\2'%SP, name)

class SUSEChannel(Channel):
    def __init__(self, name):
        Channel.__init__(self, service_pack(name), service_pack('https://build.suse.de/source/SUSE:Channels/%s/_channel'%name))

    def dump_filename(self):
        return self.url.split('/')[-2]+'.dump'

    def channel_filename(self):
        return self.url.split('/')[-2]+'.channel'

    def pickle_filename(self):
        return self.url.split('/')[-2]+'.p'

class SLE12GAChannel(SUSEChannel):
    def __init__(self, name):
        Channel.__init__(self, name, service_pack('https://build.suse.de/build/SUSE:SLE-12:GA/images/local/_product:%s/_channel'%name))


class Tool(cmdln.Cmdln):
    def __init__(self, *args, **kwargs):
        cmdln.Cmdln.__init__(self, args, kwargs)

        self.ex = FileListExtractor()
        self.ex.save_channels_files = True

    def get_optparser(self):
        parser = cmdln.CmdlnOptionParser(self)
        parser.add_option("--dry", action="store_true", help="dry run")
        parser.add_option("--debug", action="store_true", help="debug output")
        parser.add_option("--verbose", action="store_true", help="verbose")
        parser.add_option("--servicepack", type="string", dest="servicepack", help="service pack", default=None)
        return parser

    def fill_channels(self):
        global CHANNELS, CHANNELS_A, CHANNELS_B, EXCEPTIONS
    # imported and blocked update channels
        CHANNELS_A = dict((SUSEChannel(name), None) for name in """
                      SLE-DESKTOP_12_x86_64
                      SLE-SERVER_12_x86_64
                      SLE-Module-Adv-Systems-Management_12_x86_64
                      SLE-Module-Containers_12_x86_64
                      SLE-Module-Public-Cloud_12_x86_64
                      SLE-Module-Web-Scripting_12_x86_64
                      SLE-SDK_12_x86_64
                      SLE-WE_12_x86_64
                      """.split())

        # imported and blocked GA media
        CHANNELS_B = dict((SLE12GAChannel(name), { 'noupdate' : True }) for name in """
                  SLED-dvd5-DVD-x86_64
                  SLES-dvd5-DVD-x86_64
                  sle-module-adv-systems-management-cd-cd-x86_64
                  sle-module-public-cloud-cd-cd-x86_64
                  sle-module-web-scripting-cd-cd-x86_64
                  sle-sdk-dvd5-DVD-x86_64
                  sle-we-dvd-DVD-x86_64
                  """.split())

        # manual overrides where we got an exception but channel was not updated yet
        EXCEPTIONS = {
            Channel('exceptions', 'file://sle-exceptions'): None
        }

        CHANNELS = dict(CHANNELS_A)
        CHANNELS.update(CHANNELS_B)
        CHANNELS.update(EXCEPTIONS)

        # cloud add-on. Nothing blocked
        # https://redmine.nue.suse.com/issues/2346
        #cloud_whitelist = [ 'openstack-.*' ]
        #CHANNELS[SUSEChannel('12-Cloud-Compute_5_x86_64')] = \
            #    { 'whitelist' : cloud_whitelist }
        #CHANNELS['https://build.suse.de/build/SUSE:SLE-12:GA/images/local/_product:suse-sle12-cloud-compute-cd-cd-x86_64/_channel')] = \
            #    { 'whitelist' : cloud_whitelist }

        # ha add-on. Only selected packages blocked
        ha_whitelist = [ 'drbd', 'pacemaker', 'libpacemaker.*']
        CHANNELS[SUSEChannel('SLE-HA_12_x86_64')] = \
            { 'whitelist' : ha_whitelist }
        CHANNELS[SLE12GAChannel('sle-ha-cd-cd-x86_64')] = \
            { 'noupdate' : True, 'whitelist': ha_whitelist }

        # ha geo add-on. Only selected packages blocked
        ha_geo_whitelist = [ 'booth.*' ]
        CHANNELS[SUSEChannel('SLE-HA-GEO_12_x86_64')] = \
            { 'whitelist' : ha_geo_whitelist }
        CHANNELS[SLE12GAChannel('sle-ha-geo-cd-cd-s390x_x86_64')] = \
            { 'noupdate' : True, 'whitelist': ha_geo_whitelist }

        # manager add-on. Only selected packages blocked
        manager_whitelist = [ 'rhn.*', 'spacewalk-.*' ]
        CHANNELS[SUSEChannel('SLE-Manager-Tools_12_x86_64')] = \
            { 'whitelist' : manager_whitelist }
        CHANNELS[SLE12GAChannel('sle-manager-tools-cd-cd-x86_64')] = \
            { 'noupdate' : True, 'whitelist': manager_whitelist }

    def postoptparse(self):
        logging.basicConfig()
        self.logger = logging.getLogger(self.optparser.prog)
        if (self.options.debug):
            self.logger.setLevel(logging.DEBUG)
            self.ex.debug = True
        elif (self.options.verbose):
            self.logger.setLevel(logging.INFO)
        if (self.options.servicepack):
            global SP
            SP = self.options.servicepack

        self.ex.logger = self.logger
        self.fill_channels()


    def do_list(self, subcmd, opts, *channel_names):
        """${cmd_name}: list channels

        ${cmd_usage}
        ${cmd_option_list}
        """

        if channel_names:
            names = set(channel_names)
            channels = [ c for c in CHANNELS.keys() if c.name in names ]
        else:
            channels = sorted(CHANNELS.keys())
        for c in channels:
            print "%s\n  %s\n  %s"%(c.name, c.url, c.channel_filename())

    @cmdln.option("-f", "--force", action="store_true",
                  help="force something")
    def do_pickle(self, subcmd, opts, *channel_names):
        """${cmd_name}: generate pickle for channels

        ${cmd_usage}
        ${cmd_option_list}
        """

        if channel_names:
            names = set(channel_names)
            channels = [ c for c in CHANNELS.keys() if c.name in names ]
        else:
            channels = sorted(CHANNELS.keys())

        self._pickle(channels, force=opts.force)

    def _pickle(self, channels, force=False):
        self.ex.set_blacklist(BLACKLIST)

        def process(channel, noupdate = False):
            self.logger.debug("processing %s %s", channel.name, noupdate)
            fn = channel.pickle_filename()
            olddata = None
            if os.path.exists(fn):
                if noupdate:
                    return False
                with open(fn, 'rb') as f:
                    olddata = pickle.load(f)
            data = self.ex.readFileLists([channel])
            if olddata is not None:
                missing = olddata['pkgnames'] - data['pkgnames']
                new = data['pkgnames'] - olddata['pkgnames']
                if new:
                    self.logger.info("%s - new: %s", channel.name, ', '.join(new))
                if missing:
                    self.logger.warning("%s - packages vanished: %s", channel.name, ', '.join(missing))
                if not new:
                    self.logger.info("%s - no change"%channel.name)
                    return False
                data = self.ex.merge(olddata, data)
            with open(fn, 'wb') as f:
                pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
            with open(channel.dump_filename(), 'wb') as f:
                f.write(pformat(data))
            return True

        changed = False
        for i in channels or []:
            if not i:
                continue
            args = {}
            if i in CHANNELS and CHANNELS[i] is not None:
                if not force and 'noupdate' in CHANNELS[i]:
                    args['noupdate'] = True
                if 'whitelist' in CHANNELS[i]:
                    self.ex.set_whitelist(CHANNELS[i]['whitelist'])
            if process(i, **args):
                changed = True
            self.ex.set_whitelist(None)

        return changed

    @cmdln.option("-o", "--output", dest='filename', metavar='FILE',
                  help="output to FILE")
    @cmdln.option("--shelve", action="store_true", help="save as shelve file")
    def do_merge(self, subcmd, opts, *files):
        """${cmd_name}: merge .p files after call to pickle

        ${cmd_usage}
        ${cmd_option_list}
        """

        if not opts.filename:
            raise Exception('filename missing')

        return self._merge(opts.filename, files, shelve = opts.shelve)

    def _merge(self, filename, files, shelve = False):

        if not files:
            files = [i.pickle_filename() for i in sorted(CHANNELS.keys())]

        res = {
            'filenames' : dict(),
            'pkgnames' : set(),
        }

        for fn in files:
            with open(fn, 'rb') as f:
                self.logger.debug("merging %s"%fn)
                data = pickle.load(f)
                res = self.ex.merge(res, data)

        tmpfn = filename+'.new'
        if shelve:
            import shelve
            d = shelve.open(tmpfn, flag='n')
            d.update(res)
            d.close()
        else:
            with open(tmpfn, 'wb') as f:
                pickle.dump(res, f, pickle.HIGHEST_PROTOCOL)
        os.rename(tmpfn, filename)

    @cmdln.option("-f", "--force", action="store_true",
                  help="force something")
    @cmdln.option('-c', "--check", metavar="check", action="append", help="which file to check against the rest")
    def do_check_bsk(self, subcmd, opts, *channel_names):
        """${cmd_name}: check bsk consistency

        checks if files in BSK are actually just subpackages of stuff that
        already is in other products. Also prints duplicates and what would be
        left if the duplicates and supackages were removed.

        Only operates locally. Needs previous run of pickle

        ${cmd_usage}
        ${cmd_option_list}
        """
        pkgs = dict() # srcpkg -> set(binpkgs)
        checkpkgs = dict() # srcpkg -> set(binpkgs)

        b2chan = dict() # binpkg -> set(channel)

        if not opts.check:
            opts.check = [SUSEChannel('SLE-BSK_12_x86_64'), SLE12GAChannel('sle-bsk-dvd5-DVD-x86_64')]

        if channel_names:
            names = set(channel_names)
            channels = [ c for c in CHANNELS.keys() if c.name in names ]
        else:
            channels = CHANNELS_A.keys() + CHANNELS_B.keys()

        def parse(dst, channels):
            for channel in channels:
                fn = channel.channel_filename()
                with open(fn, 'rb') as f:
                    root = ET.parse(f).getroot()
                    for binaries in root.findall('binaries'):
                        for node in binaries.findall('binary'):
                            name = node.attrib['name']
                            package = node.attrib['package']
                            if package.startswith('_product:'):
                                continue
                            if name.endswith('-debuginfo') or name.endswith('-debugsource'):
                                continue
                            dst.setdefault(package, set()).add(name)
                            b2chan.setdefault(name, set()).add(channel.name)

        parse(pkgs, channels)
        self._pickle(opts.check)
        parse(checkpkgs, opts.check)

        bsk_all = set(checkpkgs.keys())
        for p in sorted(checkpkgs.keys()):
            if p in pkgs:
                missing = checkpkgs[p] - pkgs[p]
                overlap = pkgs[p] & checkpkgs[p]
                if missing:
                    bsk_all.remove(p)
                    print("separate subpackage %s: %s"%(p, ', '.join(sorted(missing))))
                if overlap:
                    if p in bsk_all:
                        bsk_all.remove(p)
                    for b in overlap:
                        print("duplicate %s: %s"%(b, ', '.join(sorted(b2chan[b]))))

        for p in sorted(bsk_all):
            print "left %s: %s"%(p, ','.join(sorted(checkpkgs[p])))

    def do_check_exported(self, subcmd, opts, *channel_names):
        """${cmd_name}: check OBS exported packages

        check if binary rpms are exported in obs

        ${cmd_usage}
        ${cmd_option_list}
        """

        exported = set()

        import osc.conf

        self.ex._init_osc()

        apiurl = osc.conf.config['apiurl']
        u = osc.core.makeurl(apiurl,
                service_pack('build/openSUSE.org:SUSE:SLE-12:GA/standard/x86_64/_repository').split('/'),
                [ 'view=binaryversions' ])
        r = osc.core.http_GET(u)
        root = ET.parse(r).getroot()
        for node in root.findall('binary'):
            name = node.attrib['name']
            name = name[:-len('.rpm')]
            if name.endswith('-debuginfo') or name.endswith('-debugsource'):
                continue
            if name.endswith('-debuginfo-32bit') or name.endswith('-debugsource-32bit'):
                continue
            exported.add(name)

        if channel_names:
            names = set(channel_names)
            channels = [ c for c in CHANNELS.keys() if c.name in names ]
        else:
            channels = CHANNELS_A.keys() + CHANNELS_B.keys() + EXCEPTIONS.keys()

        b2chan = dict() # binpkg -> set(channel)

        def parse(channels):
            blacklist = re.compile('('+'|'.join(BLACKLIST+['update-test-.*'])+')')
            for channel in channels:
                fn = channel.channel_filename()
                with open(fn, 'rb') as f:
                    root = ET.parse(f).getroot()
                    for binaries in root.findall('binaries'):
                        for node in binaries.findall('binary'):
                            name = node.attrib['name']
                            package = node.attrib['package']
                            if package.startswith('_product:'):
                                continue
                            if name.endswith('-debuginfo') or name.endswith('-debugsource'):
                                continue
                            if name.endswith('-debuginfo-32bit') or name.endswith('-debugsource-32bit'):
                                continue
                            if blacklist.match(name):
                                continue
                            b2chan.setdefault(name, set()).add(channel.name)

        parse(channels)
        needed = set(b2chan.keys())

        for p in sorted(needed-exported):
            print "missing %s: %s"%(p, ', '.join(b2chan[p]))
        for p in sorted(exported-needed):
            print "extra %s"%p

    @cmdln.option('-n', '--interval', metavar="minutes", type="int", help="periodic interval in minutes")
    @cmdln.option('--git-commit', action="store_true", help="commit changes to git")
    def do_update_obs(self, subcmd, opts):
        """${cmd_name}: update files, upload

        ${cmd_usage}
        ${cmd_option_list}
        """

        import osc.core

        class ExTimeout(Exception):
            """raised on timeout"""

        if opts.interval:
            def alarm_called(nr, frame):
                raise ExTimeout()
            signal.signal(signal.SIGALRM, alarm_called)

        while True:
            try:
                channels = sorted(CHANNELS.keys())
                if self._pickle(channels):
                    if opts.git_commit:
                        try:
                            for c in channels:
                                subprocess.call(['git', 'add', c.dump_filename()])
                            subprocess.call(['git', 'commit', '-m', 'update'])
                        except OSError, e:
                            print "### git ERROR: %s"%e
                    dbfile = 'SLE-12-file-package-map.db'
                    self.logger.info("merging")
                    self._merge(dbfile, None, shelve=True)
                    self.logger.info("uploading")
                    self.ex._init_osc()
                    apiurl = 'https://api.opensuse.org'
                    u = osc.core.makeurl(apiurl, [ 'source', service_pack('openSUSE:Backports:SLE-12'), 'rpmlint-backports-data', dbfile ], {})
                    r = osc.core.http_PUT(u, file=dbfile)
                    self.logger.info(r.read())
            except Exception, e:
                print "### ERROR: %s"%e

            if opts.interval:
                print "sleeping %d minutes. Press enter to check now ..."%opts.interval
                signal.alarm(opts.interval*60)
                try:
                    raw_input()
                except ExTimeout:
                    pass
                except EOFError:
                    # no tty available, let's sleep then
                    time.sleep(opts.interval)
                signal.alarm(0)
                continue
            break

if __name__ == "__main__":
    app = Tool()
    sys.exit( app.main() )

# vim: sw=4 et
