"""
[ urlimport.py ]
Enables remote module importing.

version:   0.54m (myevan (http://imp17.com/tc/myevan/guestbook) fixed)
version:   0.53n (alex <alexbodn@012.net.il> fixed)
author:    Jure Vrscaj <jure@codeshift.net>
homepage:  http://urlimport.codeshift.net
license:   GNU GPL

== history ==
v0.54m 2010-03-26 - https id, pw authorization added
v0.53n 2010-03-15 - split perforce class from other urls
v0.52n 2010-03-14 - allow cache disabling for certain urls and hosts
v0.51n 2010-03-13 - allow setting ssl keys/certs for specific urls and hosts
v0.50n 2010-03-12 - fixed bugs with cache building when submodules present
v0.49m 2010-03-10 - pyo, pyc support, mac source newlines, fixed minor bugs
v0.48m 2010-02-26 - pydXX support
v0.47m 2010-02-04 - reset function added
v0.46m 2010-02-03 - perforce bug fix
v0.45m 2010-02-03 - pyd bug fix
v0.44m 2010-01-31 - perforce support
v0.43m 2010-01-25 - pyd support
v0.42m 2010-01-22 - added cache function & hid some logs
v0.42b 2006-12-30 - added support for DOS-style source files - eval() chokes on "\r\n"
v0.42  2006-06-26 - added verbose mode setting
v0.41  2006-06-05 - client ssl certificate support
v0.32  2006-05-10 - ftp, https support :)
v0.31  2006-02-24 - recursion patch: non-packages now have no __path__
                  - load_module now returns the module from sys.modules,
                    in case the module itself was messing with sys.modules
v0.30  2006-02-23 package importing now possible
v0.02  2006-02-23 remote modules now first check own url when they have to import sth
v0.01  2006-02-19 made basic (single-file) importing
v0.00  2006-02-18 playing with path_hooks

Usage:

import sys, urlimport
urlimport.config(**{
    'ssl_key.https://your.url':'key for this url'
    'ssl_cert.https://your.url':'cert for this url',
    'ssl_key.host':'key for this host'
    'ssl_cert.host':'cert for this host',
    'ssl_key':'key for unspecified urls', 
    'ssl_cert':'cert for unspecified urls',
    'cacheDir':'root of the cache dir', # if missing, a temp dir will be created,
                                        # if empty, no cache will be used
    'no_cache.url':True, # don'tcache modules from this url
    'no_cache.host':True, # don'tcache modules from this host
    'p4conn': None,
    'debug': int(level of output text. see debug() function)
})
sys.path.insert(0, "http://your.url") #this may already be there, like setting
                                      #PYTHONPATH
                                      #but there is a danger default __import__
                                      #would try it before
expect to have access to your modules at http://your.url

TODO:
test with perforce: hope i didn't ruin the code.
test with windows/*.pyd??: same hope.
"""

import sys, os, re
import imp, marshal
import base64

from tempfile import mkdtemp
from urllib2 import urlopen

DOT_PYO = ".pyo"
DOT_PYC = ".pyc"
DOT_PY = ".py"
STANDARD_FILES = (DOT_PYO, DOT_PYC, DOT_PY)
DOT_PYD = ".pyd%x%x" % ((sys.hexversion >> 24), (sys.hexversion >> 16) & 0xf)
DOT_SO = ".so"

settings = sys.__dict__.setdefault(
    'urlimport_settings',
    {
    'ssl_cert': '', 
    'ssl_key': '', 
    'cacheDir': mkdtemp(),
    'debug': 0,
    }
)

url_cache = {}

def config(**kwargs):
    """config(key=value) - Set key to value.
       config()          - Display settings.
    """
    settings.update(kwargs)
    for k,v in settings.iteritems():
        debug(" "+str(k)+"="+repr(v), lvl=1 )

def cache_reset():
    """remove the cache contents"""
    cacheDir = settings.get('cacheDir', '')
    if cacheDir:
        for base, dirs, names in os.walk(cacheDir):
            for name in names:
                os.remove(os.path.join(base, name))

def debug(s, pf='| |', lvl=1):
    if lvl <= settings.get('debug', 0):
        print "%s %s" % (pf, s)

class UrlFinder:
    
    re_url_split = re.compile('^(.+):\/\/(.+?)(?::(\d+))?(\/.*)$')
    re_url_ok = re.compile(r'^http://|^ftp://|^https://')
    
    def __init__(self, path):
        if self.re_url_ok.match(path):
            debug("%s: accepting '%s'." % (self.__class__.__name__, path), lvl=2)
            self.path = path.rstrip('/') + '/'
        else:
            debug("%s: rejecting non-matching path item: '%s'" % (self.__class__.__name__, path), lvl=3)
            raise ImportError

    def find_module(self, fullname, mpath=None):
        """try to locate the remote module, do this:
         a) try to get fullname.py from http://self.path/
         b) try to get __init__.py from http://self.path/fullname/
        """

        debug("find_module %s" % fullname, lvl=2)
        head = self.path + fullname.split('.')[-1]

        file_types = \
            [(file_type, None) for file_type in STANDARD_FILES] + \
            [(DOT_SO, None), (DOT_PYD, None)] + \
            [('/__init__%s' % file_type, head + '/') for file_type in STANDARD_FILES]
        for tail, path in file_types:
            url = head + tail
            debug("check_url %s" % url, lvl=2)
            try:
                debug("trying to get '%s'." % url, lvl=1)
                force_cache = url.endswith(DOT_PYD) or url.endswith(DOT_SO)
                fileData, cachedFile = \
                    self.caching_get_file(url, fullname, tail, force_cache)
                if url.endswith(DOT_PYO) or url.endswith(DOT_PYC):
                    debug("find_compiled_module: got '%s'." % url, lvl=1)
                    return UrlCompiledLoader(url, path, fileData)
                elif url.endswith(DOT_PY):
                    debug("find_source_module: got '%s'." % url, lvl=1)
                    return UrlSourceLoader(url, path, fileData)
                else:
                    debug("find_dynamic_module: got '%s'." % url, lvl=1)
                    return UrlDynamicLoader(cachedFile)
            except Exception, e:
                debug("find_module: failed to get '%s'. (%s)" % (url, e), lvl=3)

        debug("not_found_module %s" % (fullname), lvl=2)

    def caching_get_file(self, url, module, tail, force_cache=False):
        """get the file data using cache.
        """
        fileData = None
        cachedFile = None
        proto, host, port, path = self.split_url(url)
        no_cache = settings.get('no_cache.%s' % url, 
                                settings.get('no_cache.%s/' % url, 
                                             settings.get('no_cache.%s' % host, False))) and \
                   not force_cache
        if url in url_cache and not no_cache:
            try:
                fileData = open(url_cache[url], "rb").read()
                debug('found_in_file_cache %s' % url_cache[url], lvl=1)
            except:
                fileData = self.get_file_data(url)
        else:
            fileData = self.get_file_data(url)
        if url not in url_cache and not no_cache:
            cacheDir = settings.get('cacheDir', mkdtemp())
            module = module.replace('.', os.path.sep)
            if tail == DOT_PYD:
                tail = '.pyd'
            cachedFile = os.path.join(cacheDir, module + tail)
            debug('saving in cache path:%s' % cachedFile, pf='|>>>>>|', lvl=1)
            path = os.path.dirname(cachedFile)
            if not os.path.exists(path):
                os.makedirs(path)
            open(cachedFile, "wb").write(fileData)
            url_cache[url] = cachedFile
        return fileData, cachedFile

    def get_file_data(self, url):
        """Download the file data from given url.
        """
        proto, host, port, path = self.split_url(url)
        creds, host = host.split("@") if "@" in host else ("", host)

        key = settings.get('ssl_key.%s' % self.path, 
                           settings.get('ssl_key.%s/' % self.path, 
                                        settings.get('ssl_key.%s' % host,
                                                     settings.get('ssl_key', ''))))
        cert = settings.get('ssl_cert.%s' % self.path, 
                            settings.get('ssl_cert.%s/' % self.path, 
                                         settings.get('ssl_cert.%s' % host,
                                                      settings.get('ssl_cert', ''))))
        debug(cert + ',' + key, pf='|>>>>|', lvl=3)
        if proto == 'https':
            # handle http over ssl with client certificate
            import httplib

            try:
                port = int(port)
            except:
                port = 443

            conn = httplib.HTTPSConnection(host=host, port=port, key_file=key, cert_file=cert,) if key and cert else httplib.HTTPSConnection(host=host, port=port)
            conn.putrequest('GET', path)
            if creds:
                conn.putheader('Authorization', "Basic %s" % base64.encodestring(creds))
            conn.endheaders()
            response = conn.getresponse()
            if response.status != 200:
                raise StandardError, "HTTPS Error: %d"%response.status
            return response.read()
        else:
            # handle everything else
            return urlopen(url).read()

    def split_url(self, url):
        return self.re_url_split.findall(url)[0]
 

class UrlDynamicLoader:
    def __init__(self, cachedFile):
        self.cachedFile = cachedFile

    def load_module(self, fullname):
        """add the new module to sys.modules,
        execute its source and return it
        """
        return imp.load_dynamic(fullname, self.cachedFile)

class UrlSourceLoader:
    def __init__(self, url, path, data):
        self.url = url
        self.path = path
        self.source = data.replace("\r\n", "\n").replace('\r', '\n')
        self._files = {}

    def load_module(self, fullname):
        """add the new module to sys.modules,
        execute its source and return it
        """
        mod = sys.modules.setdefault(fullname, imp.new_module(fullname))

        mod.__file__ = "%s" % self.url
        mod.__loader__ = self
        if self.path:
            mod.__path__ = [self.path]

        for line in self.source.split('\n'):
            debug(line, pf='|>|', lvl=4)

        debug("load_module: executing %s's source..." % fullname, lvl=2)

        exec self.source in mod.__dict__

        mod = sys.modules[fullname]
        return mod

class UrlCompiledLoader:
    def __init__(self, url, path, data):
        self.url = url
        self.path = path
        if data[:4] != imp.get_magic():
            raise ImportError("incompatible python version")
        self.code = marshal.loads(data[8:])
        self._files = {}

    def load_module(self, fullname):
        """add the new module to sys.modules,
        execute it and return it
        """
        mod = sys.modules.setdefault(fullname, imp.new_module(fullname))

        mod.__file__ = "%s" % self.url
        mod.__loader__ = self
        if self.path:
            mod.__path__ = [self.path]

        debug(repr(self.code), pf='|>|', lvl=4)

        debug("load_module: executing %s's code..." % fullname, lvl=2)

        exec self.code in mod.__dict__

        mod = sys.modules[fullname]
        return mod


class PerforceFinder(UrlFinder):
    
    re_url_ok = re.compile(r'^p4://')
    
    def get_file_data(self, url):
        """Download the file data from given url.
        """
        p4conn = settings.get('p4conn', None)
        if not p4conn:
           from P4 import P4
           p4conn = P4()
           p4conn.connect()
           debug("p4 connect", lvl=1)

        fstats = p4conn.run_fstat(url[3:])
        for fstat in fstats:
            try:
                return open(fstat["clientFile"]).read()
            except IOError:
                p4conn.run_sync(url[3:])
                return open(fstat["clientFile"]).read()
            else:
                raise StandardError, "PERFORCE Error"

    def split_url(self, url):
        return "", "", "", ""
 

# register The Hooks
sys.path_hooks = \
    [x for x in sys.path_hooks if x.__name__ != 'UrlFinder'] + [UrlFinder]
sys.path_hooks = \
    [x for x in sys.path_hooks if x.__name__ != 'PerforceFinder'] + [PerforceFinder]

sys.path_importer_cache.clear()

debug("Url and Perforce importing enabled. Add urls to sys.path.", lvl=1)
debug("Use urlimport.config(key=value) to manipulate settings:", lvl=1)

# print settings
config()

debug("", lvl=1)
debug("This stuff is experimental, use at your own risk. Enjoy.", lvl=1)

#if cacheDir:
#    sys.path.append(cacheDir) # is this needed? this should be inserted where
#                              # the url is, not at the end

if __name__ == "__main__":
    # TODO: TEST_CODE
    pass