12. Packages, Libraries, and Modules

Introduction

#-----------------------------
## Python's "module" is the closest equivalent to Perl's "package"


#=== In the file "Alpha.py"
name = "first"

#=== End of file

#=== In the file "Omega.py"

name = "last"
#=== End of file

import Alpha, Omega
print "Alpha is %s, Omega is %s." % (Alpha.name, Omega.name)
#> Alpha is first, Omega is last.
#-----------------------------
# Python does not have an equivalent to "compile-time load"
import sys

# Depending on the implementation, this could use a builtin
# module or load a file with the extension .py, .pyc, pyo, .pyd,
# .so, .dll, or (with imputils) load from other files.
import Cards.Poker

#-----------------------------
#=== In the file Cards/Poker.py
__all__ = ["card_deck", "shuffle"]  # not usually needed
card_deck = []
def shuffle():
    pass

#-----------------------------

Defining a Module's Interface

#-----------------------------
#== In the file "YourModule.py"

__version__ = (1, 0)          # Or higher
__all__ = ["...", "..."]      # Override names included in "... import *"
                              #   Note: 'import *' is considered poor style
                              #   and it is rare to use this variable.
########################
# your code goes here
########################

#-----------------------------
import YourModule             # Import the module into my package
                              #  (does not import any of its symbols)

import YourModule as Module   # Use a different name for the module

from YourModule import *      # Import all module symbols not starting
                              #  with an underscore (default); if __all__
                              #  is defined, only imports those symbols.
                              # Using this is discouraged unless the 
                              #  module is specifically designed for it.

from YourModule import name1, name2, xxx
                              # Import the named symbols from the module

from YourModule import name1 as name2
                              # Import the named object, but use a
                              #  different name to access it locally.

#-----------------------------
__all__ = ["F1", "F2", "List"]
#-----------------------------
__all__ = ["Op_Func", "Table"]
#-----------------------------
from YourModule import Op_Func, Table, F1
#-----------------------------
from YourModule import Functions, Table
#-----------------------------

Trapping Errors in require or use

#-----------------------------
# no import
mod = "module"
try:
    __import__(mod)
except ImportError, err:
    raise ImportError("couldn't load %s: %s" % (mod, err))

# imports into current package
try:
    import module
except ImportError, err:
    raise ImportError("couldn't load 'module': %s" % (err, ))

# imports into current package, if the name is known
try:
    import module
except ImportError, err:
    raise ImportError("couldn't load 'module': %s" % (err, ))

# Use a fixed local name for a named module
mod = "module"
try:
    local_name = __import__(mod)
except ImportError, err:
    raise ImportError("couldn't load %s: %s" % (mod, err))

# Use the given name for the named module.
# (You probably don't need to do this.)
mod = "module"
try:
    globals()[mod] = __import__(mod)
except ImportError, err:
    raise ImportError("couldn't load %s: %s" % (mod, err))

#-----------------------------
DBs = "Giant.Eenie Giant.Meanie Mouse.Mynie Moe".split()
for mod in DBs.split():
    try:
        loaded_module = __import__(mod)
    except ImportError:
        continue
    # __import__ returns a reference to the top-most module
    # Need to get the actual submodule requested.
    for term in mod.split(".")[:-1]:
        loaded_module = getattr(loaded_module, term)
    break
else:
    raise ImportError("None of %s loaded" % DBs)
#-----------------------------

Delaying use Until Run Time

#-----------------------------
import sys
if __name__ == "__main__":
    if len(sys.argv) != 3 or not sys.argv[1].isdigit() \
                          or not sys.argv[2].isdigit():
        raise SystemExit("usage: %s num1 num2" % sys.argv[0])

import Some.Module
import More.Modules
#-----------------------------
if opt_b:
    import math
#-----------------------------
from os import O_EXCL, O_CREAT, O_RDWR

#-----------------------------
import os
O_EXCL = os.O_EXCL
O_CREAT = os.O_CREAT
O_RDWR = os.O_RDWR
#-----------------------------
import os
O_EXCL, O_CREAT, O_RDWR = os.O_EXCL, os.O_CREAT, os.O_RDWR
#-----------------------------
load_module('os', "O_EXCL O_CREAT O_RDWR".split())

def load_module(module_name, symbols):
    module = __import__(module_name)
    for symbol in symbols:
        globals()[symbol] = getattr(module, symbol)
#-----------------------------

Making Variables Private to a Module

#-----------------------------
# Python doesn't have Perl-style packages

# Flipper.py
__version__ = (1, 0)

__all__ = ["flip_boundary", "flip_words"]

Separatrix = ' '  # default to blank

def flip_boundary(sep = None):
    prev_sep = Separatrix
    if sep is not None:
        global Separatrix
        Separatrix = sep
    return prev_sep

def flip_words(line):
    words = line.split(Separatrix)
    words.reverse()
    return Separatrix.join(words)
#-----------------------------

Determining the Caller's Package

#-----------------------------
this_pack = __name__
#-----------------------------
that_pack = sys._getframe(1).f_globals.get("__name__", "<string>")
#-----------------------------
print "I am in package", __name__
#-----------------------------
def nreadline(count, myfile):
    if count <= 0:
        raise ValueError("Count must be > 0")
    return [myfile.readline() for i in range(count)]

def main():
    myfile = open("/etc/termcap")
    a, b, c = nreadline(3, myfile)
    myfile.close()

if __name__ == "__main__":
    main()

# DON'T DO THIS:
import sys

def nreadline(count, handle_name):
    assert count > 0, "count must be > 0"
    locals = sys._getframe(1).f_locals
    if not locals.has_key(handle_name):
        raise AssertionError("need open filehandle")
    infile = locals[handle_name]
    retlist = []
    for line in infile:
        retlist.append(line)
        count -= 1
        if count == 0:
            break
    return retlist

def main():
    FH = open("/etc/termcap")
    a, b, c = nreadline(3, "FH")

if __name__ == "__main__":
    main()
#-----------------------------

Automating Module Clean-Up

#-----------------------------
## There is no direct equivalent in Python to an END block
import time, os, sys

# Tricks to ensure the needed functions exist during module cleanup
def _getgmtime(asctime=time.asctime, gmtime=time.gmtime,
               t=time.time):
    return asctime(gmtime(t()))

class Logfile:
    def __init__(self, file):
        self.file = file

    def _logmsg(self, msg, argv0=sys.argv[0], pid=os.getpid(),
                _getgmtime=_getgmtime):
        # more tricks to keep all needed references
        now = _getgmtime()
        print>>self.file, argv0, pid, now + ":", msg

    def logmsg(self, msg):
        self._logmsg(self.file, msg)

    def __del__(self):
        self._logmsg("shutdown")
        self.file.close()

    def __getattr__(self, attr):
        # forward everything else to the file handle
        return getattr(self.file, attr)

# 0 means unbuffered
LF = Logfile(open("/tmp/mylog", "a+", 0))
logmsg = LF.logmsg

#-----------------------------
## It is more appropriate to use try/finally around the
## main code, so the order of initialization and finalization
## can be specified.
if __name__ == "__main__":
    import logger
    logger.init("/tmp/mylog")
    try:
        main()
    finally:
        logger.close()

#-----------------------------

Keeping Your Own Module Directory

#-----------------------------
#% python -c 'import sys\
for i, name in zip(xrange(sys.maxint), sys.path):\
    print i, repr(name)
#> 0 ''
#> 1 '/usr/lib/python2.2'
#> 2 '/usr/lib/python2.2/plat-linux2'
#> 3 '/usr/lib/python2.2/lib-tk'
#-----------------------------
# syntax for sh, bash, ksh, or zsh
#$ export PYTHONPATH=$HOME/pythonlib

# syntax for csh or tcsh
#% setenv PYTHONPATH ~/pythonlib
#-----------------------------
import sys
sys.path.insert(0, "/projects/spectre/lib")
#-----------------------------
import FindBin
sys.path.insert(0, FindBin.Bin)
#-----------------------------
import FindBin
Bin = "Name"
bin = getattr(FindBin, Bin)
sys.path.insert(0, bin + "/../lib")
#-----------------------------

Preparing a Module for Distribution

#-----------------------------
#% h2xs -XA -n Planets
#% h2xs -XA -n Astronomy::Orbits
#-----------------------------
# @@INCOMPLETE@@
# @@INCOMPLETE@@
# Need a distutils example
#-----------------------------

Speeding Module Loading with SelfLoader

#-----------------------------
# Python compiles a file to bytecode the first time it is imported and 
# stores this compiled form in a .pyc file.  There is thus less need for
# incremental compilation as once there is a .pyc file, the sourcecode
# is only recompiled if it is modified.  

Speeding Up Module Loading with Autoloader

#-----------------------------
# See previous section

Overriding Built-In Functions

#-----------------------------
## Any definition in a Python module overrides the builtin
## for that module

#=== In MyModule
def open():
    pass # TBA
#-----------------------------
from MyModule import open
file = open()
#-----------------------------

Reporting Errors and Warnings Like Built-Ins

#-----------------------------
def even_only(n):
    if n & 1:     # one way to test
        raise AssertionError("%s is not even" % (n,))
    #....

#-----------------------------
def even_only(n):
    if n % 2:    # here's another
        # choice of exception depends on the problem
        raise TypeError("%s is not even" % (n,))
    #....

#-----------------------------
import warnings
def even_only(n):
    if n & 1:           # test whether odd number
        warnings.warn("%s is not even, continuing" % (n))
        n += 1
    #....
#-----------------------------
warnings.filterwarnings("ignore")
#-----------------------------

Referring to Packages Indirectly

#-----------------------------
val = getattr(__import__(packname), varname)
vals =  getattr(__import__(packname), aryname)
getattr(__import__(packname), funcname)("args")

#-----------------------------
# DON'T DO THIS [Use math.log(val, base) instead]
import math
def make_log(n):
   def logn(val):
      return math.log(val, n)
   return logn

# Modifying the global dictionary - this could also be done
# using locals(), or someobject.__dict__
globaldict = globals()
for i in range(2, 1000):
    globaldict["log%s"%i] = make_log(i)

# DON'T DO THIS
for i in range(2,1000):
    exec "log%s = make_log(i)"%i in globals()
    
print log20(400)
#=>2.0
#-----------------------------
blue = colours.blue
someobject.blue = colours.azure  # someobject could be a module...
#-----------------------------

Using h2ph to Translate C #include Files

#-----------------------------
# Python extension modules can be imported and used just like
# a pure python module.
#
# See http://www.cosc.canterbury.ac.nz/~greg/python/Pyrex/ for
# information on how to create extension modules in Pyrex [a
# language that's basically Python with type definitions which
# converts to compiled C code]
#
# See http://www.boost.org/libs/python/doc/ for information on how
# to create extension modules in C++.
#
# See http://www.swig.org/Doc1.3/Python.html for information on how
# to create extension modules in C/C++
#
# See http://docs.python.org/ext/ext.html for information on how to
# create extension modules in C/C++ (manual reference count management).
#
# See http://cens.ioc.ee/projects/f2py2e/ for information on how to
# create extension modules in Fortran
#
# See http://www.scipy.org/Weave for information on how to 
# include inline C code in Python code.
#
# @@INCOMPLETE@@ Need examples of FineTime extensions using the different methods...
#-----------------------------

Using h2xs to Make a Module with C Code

#-----------------------------
# See previous section
#-----------------------------

Documenting Your Module with Pod

#-----------------------------
# To document code, use docstrings. A docstring is a bare string that
# is placed at the beginning of a module or immediately after the 
# definition line of a class, method, or function. Normally, the
# first line is a brief description of the object; if a longer
# description is needed, it commences on the third line (the second
# line being left blank).  Multiline comments should use triple
# quoted strings.
# 
# Docstrings are automagically assigned to an object's __doc__ property.
#
# In other words these three classes are identical:
class Foo(object):
    "A class demonstrating docstrings."

class Foo(object):
    __doc__ = "A class demonstrating docstrings."

class Foo(object):
    pass
Foo.__doc__ = "A class demonstrating docstrings."

# as are these two functions:
def foo():
    "A function demonstrating docstrings."

def foo():
    pass
foo.__doc__ = "A function demonstrating docstrings."

# the pydoc module is used to display a range of information about 
# an object including its docstrings:
import pydoc 
print pydoc.getdoc(int)
pydoc.help(int)

# In the interactive interpreter, objects' documentation can be 
# using the help function:
help(int)

#-----------------------------

Building and Installing a CPAN Module

#-----------------------------
# Recent Python distributions are built and installed with disutils.
# 
# To build and install under unix
# 
# % python setup.py install
# 
# If you want to build under one login and install under another
# 
# % python setup.py build
# $ python setup.py install
# 
# A package may also be available prebuilt, eg, as an RPM or Windows
# installer.  Details will be specific to the operating system.

#-----------------------------
# % python setup.py --prefix ~/python-lib
#-----------------------------

Example: Module Template

#-----------------------------
#== File Some/Module.py

# There are so many differences between Python and Perl that
# it isn't worthwhile trying to come up with an equivalent to
# this Perl code.  The Python code is much smaller, and there's
# no need to have a template.

#-----------------------------

Program: Finding Versions and Descriptions of Installed Modules

#-----------------------------
#% pmdesc
#-----------------------------
import sys, pydoc

def print_module_info(path, modname, desc):
   # Skip files starting with "test_"
   if modname.split(".")[-1].startswith("test_"):
       return
   try:
       # This assumes the modules are safe for importing,
       # in that they don't have side effects.  Could also
       # grep the file for the __version__ line.
       mod = pydoc.safeimport(modname)
   except pydoc.ErrorDuringImport:
       return
   version = getattr(mod, "__version__", "unknown")
   if isinstance(version, type("")):
       # Use the string if it's given
       pass
   else:
       # Assume it's a list of version numbers, from major to minor
       ".".join(map(str, version))
   synopsis, text = pydoc.splitdoc(desc)
   print "%s (%s) - %s" % (modname, version, synopsis)

scanner = pydoc.ModuleScanner()
scanner.run(print_module_info)

#-----------------------------