New Chapter

Well, tomorrow is a big day for me: I'll be starting a new job. I'll be working for a super-secretive company in Cupertino which means that I won't be writing any more tech-related posts. I'm sure the next few weeks will be hectic, so I doubt I'll be up to writing much anyway. But I've got some ideas of what I'd like to write about once things settle down.

I'd say "stay tuned" but I'm pretty sure no one is tuned in to my blog as it is. :)

Python: Enumerating IP Addresses on FreeBSD

As promised in my earlier post on enumerating local interfaces and their IP addresses on MacOS X, this time I'll cover how to do the same on FreeBSD and other operating systems that implement the getifaddrs API. Basically, this is just a python wrapper around the getifaddrs interface using ctypes.

The code is a bit longer than I typically like to include in a blog post, but here it goes:

"""
Wrapper for getifaddrs(3).
"""

import socket
import sys

from collections import namedtuple
from ctypes import *

class sockaddr_in(Structure):
    _fields_ = [
        ('sin_len',     c_uint8),
        ('sin_family',  c_uint8),
        ('sin_port',    c_uint16),
        ('sin_addr',    c_uint8 * 4),
        ('sin_zero',    c_uint8 * 8)
    ]

    def __str__(self):
        assert self.sin_len >= sizeof(sockaddr_in)
        data = ''.join(map(chr, self.sin_addr))
        return socket.inet_ntop(socket.AF_INET, data)

class sockaddr_in6(Structure):
    _fields_ = [
        ('sin6_len',        c_uint8),
        ('sin6_family',     c_uint8),
        ('sin6_port',       c_uint16),
        ('sin6_flowinfo',   c_uint32),
        ('sin6_addr',       c_uint8 * 16),
        ('sin6_scope_id',   c_uint32)
    ]

    def __str__(self):
        assert self.sin6_len >= sizeof(sockaddr_in6)
        data = ''.join(map(chr, self.sin6_addr))
        return socket.inet_ntop(socket.AF_INET6, data)
        
class sockaddr_dl(Structure):
    _fields_ = [
        ('sdl_len',         c_uint8),
        ('sdl_family',      c_uint8),
        ('sdl_index',       c_short),
        ('sdl_type',        c_uint8),
        ('sdl_nlen',        c_uint8),
        ('sdl_alen',        c_uint8),
        ('sdl_slen',        c_uint8),
        ('sdl_data',        c_uint8 * 12)
    ]
    
    def __str__(self):
        assert self.sdl_len >= sizeof(sockaddr_dl)
        addrdata = self.sdl_data[self.sdl_nlen:self.sdl_nlen+self.sdl_alen]
        return ':'.join('%02x' % x for x in addrdata)

class sockaddr_storage(Structure):
    _fields_ = [
        ('sa_len',      c_uint8),
        ('sa_family',   c_uint8),
        ('sa_data',     c_uint8 * 254)
    ]

class sockaddr(Union):
    _anonymous_ = ('sa_storage', )
    _fields_ = [
        ('sa_storage', sockaddr_storage),
        ('sa_sin', sockaddr_in),
        ('sa_sin6', sockaddr_in6),
        ('sa_sdl', sockaddr_dl),
    ]

    def family(self):
        return self.sa_storage.sa_family

    def __str__(self):
        family = self.family()
        if family == socket.AF_INET:
            return str(self.sa_sin)
        elif family == socket.AF_INET6:
            return str(self.sa_sin6)
        elif family == 18:  # AF_LINK
            return str(self.sa_sdl)
        else:
            print family
            raise NotImplementedError, "address family %d not supported" % family


class ifaddrs(Structure):
    pass
ifaddrs._fields_ = [
    ('ifa_next',        POINTER(ifaddrs)),
    ('ifa_name',        c_char_p),
    ('ifa_flags',       c_uint),
    ('ifa_addr',        POINTER(sockaddr)),
    ('ifa_netmask',     POINTER(sockaddr)),
    ('ifa_dstaddr',     POINTER(sockaddr)),
    ('ifa_data',        c_void_p)
]

# Define constants for the most useful interface flags (from if.h).
IFF_UP            = 0x0001
IFF_BROADCAST     = 0x0002
IFF_LOOPBACK      = 0x0008
IFF_POINTTOPOINT  = 0x0010
IFF_RUNNING       = 0x0040
if sys.platform == 'darwin' or 'bsd' in sys.platform:
    IFF_MULTICAST = 0x8000
elif sys.platform == 'linux':
    IFF_MULTICAST = 0x1000

# Load library implementing getifaddrs and freeifaddrs.
if sys.platform == 'darwin':
    libc = cdll.LoadLibrary('libc.dylib')
else:
    libc = cdll.LoadLibrary('libc.so')

# Tell ctypes the argument and return types for the getifaddrs and
# freeifaddrs functions so it can do marshalling for us.
libc.getifaddrs.argtypes = [POINTER(POINTER(ifaddrs))]
libc.getifaddrs.restype = c_int
libc.freeifaddrs.argtypes = [POINTER(ifaddrs)]


def getifaddrs():
    """
    Get local interface addresses.

    Returns generator of tuples consisting of interface name, interface flags,
    address family (e.g. socket.AF_INET, socket.AF_INET6), address, and netmask.
    The tuple members can also be accessed via the names 'name', 'flags',
    'family', 'address', and 'netmask', respectively.
    """
    # Get address information for each interface.
    addrlist = POINTER(ifaddrs)()
    if libc.getifaddrs(pointer(addrlist)) < 0:
        raise OSError

    X = namedtuple('ifaddrs', 'name flags family address netmask')

    # Iterate through the address information.
    ifaddr = addrlist
    while ifaddr and ifaddr.contents:
        # The following is a hack to workaround a bug in FreeBSD
        # (PR kern/152036) and MacOSX wherein the netmask's sockaddr may be
        # truncated.  Specifically, AF_INET netmasks may have their sin_addr
        # member truncated to the minimum number of bytes necessary to
        # represent the netmask.  For example, a sockaddr_in with the netmask
        # 255.255.254.0 may be truncated to 7 bytes (rather than the normal
        # 16) such that the sin_addr field only contains 0xff, 0xff, 0xfe.
        # All bytes beyond sa_len bytes are assumed to be zero.  Here we work
        # around this truncation by copying the netmask's sockaddr into a
        # zero-filled buffer.
        if ifaddr.contents.ifa_netmask:
            netmask = sockaddr()
            memmove(byref(netmask), ifaddr.contents.ifa_netmask,
                    ifaddr.contents.ifa_netmask.contents.sa_len)
            if netmask.sa_family == socket.AF_INET and netmask.sa_len < sizeof(sockaddr_in):
                netmask.sa_len = sizeof(sockaddr_in)
        else:
            netmask = None
            
        try:
            yield X(ifaddr.contents.ifa_name,
                    ifaddr.contents.ifa_flags,
                    ifaddr.contents.ifa_addr.contents.family(),
                    str(ifaddr.contents.ifa_addr.contents),
                    str(netmask) if netmask else None)
        except NotImplementedError:
            # Unsupported address family.
            yield X(ifaddr.contents.ifa_name,
                    ifaddr.contents.ifa_flags,
                    None,
                    None,
                    None)
        ifaddr = ifaddr.contents.ifa_next

    # When we are done with the address list, ask libc to free whatever memory
    # it allocated for the list.
    libc.freeifaddrs(addrlist)

__all__ = ['getifaddrs'] + [n for n in dir() if n.startswith('IFF_')]

As always, this code is released under a BSD-style license.

Euklas

Carnegie Mellon University has produced a tool called Euklas (Eclipse Users' Keystrokes Lessened by Attaching from Samples). From the description:

Euklas enhances Eclipse's JavaScript editor to help users to more successfully employ copy-and-paste strategies for reuse.

I think I'm going to cry.

Python: Enumerating IP Addresses on MacOS X

How do you enumerate the host's local IP addresses from python? This turns out to be a surprisingly common question. Unfortunately, there is no pretty answer; it depends on the host operating system. On Windows, you can wrap the IP Helper GetIpAddrTable using ctypes. On modern Linux, *BSD, or MacOS X systems, you can wrap getifaddrs(). Neither is trivial, though, so I'll save those for a future post.

Luckily, MacOS X provides a simpler way to get the local IP addresses: the system configuration dynamic store. Using pyObjC, which comes pre-installed on every Mac, we can write a straight port of Apple's example in Technical Note TN1145 for retrieving a list of all IPv4 addresses assigned to local interfaces:

from SystemConfiguration import *    # from pyObjC
import socket

def GetIPv4Addresses():
    """
    Get all IPv4 addresses assigned to local interfaces.
    Returns a generator object that produces information
    about each IPv4 address present at the time that the
    function was called.

    For each IPv4 address, the returned generator yields
    a tuple consisting of the interface name, address
    family (always socket.AF_INET), the IP address, and
    the netmask.  The tuple elements may also be accessed
    by the names: "ifname", "family", "address", and
    "netmask".
    """
    ds = SCDynamicStoreCreate(None, 'GetIPv4Addresses', None, None)
    # Get all keys matching pattern State:/Network/Service/[^/]+/IPv4
    pattern = SCDynamicStoreKeyCreateNetworkServiceEntity(None,
                                                          kSCDynamicStoreDomainState,
                                                          kSCCompAnyRegex,
                                                          kSCEntNetIPv4)
    patterns = CFArrayCreate(None, (pattern, ), 1, kCFTypeArrayCallBacks)
    valueDict = SCDynamicStoreCopyMultiple(ds, None, patterns)

    ipv4info = namedtuple('ipv4info', 'ifname family address netmask')

    for serviceDict in valueDict.values():
        ifname = serviceDict[u'InterfaceName']
        for address, netmask in zip(serviceDict[u'Addresses'], serviceDict[u'SubnetMasks']):
            yield ipv4info(ifname, socket.AF_INET, address, netmask)

One interesting point regarding this code is that it doesn't actually inspect interface information in the system configuration dynamic store. The interface-related keys are stored under State:/Network/Interface/, but this code (and Apple's example on which it is based) inspect keys under State:/Network/Service/ instead. However, if you want to get IPv6 addresses then you do have to inspect the system configuration's interface information:

from SystemConfiguration import *    # from pyObjC
import socket
import re
ifnameExtractor = re.compile(r'/Interface/([^/]+)/')

def GetIPv6Addresses():
    """
    Get all IPv6 addresses assigned to local interfaces.
    Returns a generator object that produces information
    about each IPv6 address present at the time that the
    function was called.
    
    For each IPv6 address, the returned generator yields
    a tuple consisting of the interface name, address
    family (always socket.AF_INET6), the IP address, and
    the prefix length.  The tuple elements may also be
    accessed by the names: "ifname", "family", "address",
    and "prefixlen".
    """
    ds = SCDynamicStoreCreate(None, 'GetIPv6Addresses', None, None)
    # Get all keys matching pattern State:/Network/Interface/[^/]+/IPv6
    pattern = SCDynamicStoreKeyCreateNetworkInterfaceEntity(None,
                                                          kSCDynamicStoreDomainState,
                                                          kSCCompAnyRegex,
                                                          kSCEntNetIPv6)
    patterns = CFArrayCreate(None, (pattern, ), 1, kCFTypeArrayCallBacks)
    valueDict = SCDynamicStoreCopyMultiple(ds, None, patterns)

    ipv6info = namedtuple('ipv6info', 'ifname family address prefixlen')

    for key, ifDict in valueDict.items():
        ifname = ifnameExtractor.search(key).group(1)
        for address, prefixlen in zip(ifDict[u'Addresses'], ifDict[u'PrefixLength']):
            yield ipv6info(ifname, socket.AF_INET6, address, prefixlen)

In fact, you could easily adapt the above function to be able to fetch IPv4 addresses from the interface configuration.

Caffeine Deficiency

My wife has been complaining that she always feels nauseous after taking her daily vitamin. On the theory that it was perhaps a result of taking them first thing in the morning on an empty stomach, she tried taking them at night with dinner. I saw with my own eyes what she was talking about: she almost vomited.

So, I told her to start taking mine instead and I'd finish her vitamins off. Well, I'm not doing that again. I don't know what it is about One A Day Women's Active Metabolism vitamins, but they made me sick too. While we hate wasting the money, we just chucked the rest of the bottle because there was no way either of us were ever taking those again. Before we tossed the bottle in the trash, though, we were examining the label for differences between her One A Day Women's Active Metabolism and my One A Day Men's vitamins that might account for the nausea. In particular, we were looking for ingredients that were in higher concentration in the women's vitamins.

There are a few: vitamins D, K, B1, B2, B6, Calcium, and Iron to be exact. But I couldn't find any hard evidence that any of these vitamins could cause the sort of nausea we experienced (at least not in quantities we are likely to be exposed to). But that is when we noticed what makes the women's vitamins earn the "active metabolism" moniker:


Caffeine, and lots of it. Guarana seed is just another source of caffeine, so it is possible that its 50mg is included in the total of 120mg of caffeine in each "vitamin" pill. But if not, that means that 1 One A Day Women's Active Metabolism pill has more caffeine (~170mg) than a cup of coffee (100-150mg) and approaching that of a single Vivarin pill (200mg).

While neither of us drinks much coffee and we try to avoid caffeinated drinks, I doubt that 170mg of coffee in the morning would be enough to induce nausea. After all, millions of people have a cup of coffee in the morning and that is approximately the same amount of caffeine.

But, we never expected to have caffeine added to vitamin pills. I guess that is what we get for not having looked at the label before buying them. We certainly won't be making that mistake again.

All that background out of the way, this experience got me thinking: just how much caffeine do people consume everyday now? Here my wife was getting a base 170mg a day without even knowing it. The marketing for caffeinated products tends to emphasize that it helps you stay "mentally aware"; how long before being constantly-caffeinated is considered the norm and the symptoms of caffeine deficiency are "mentally sluggish"?

At what point does caffeine go from "booster" to baseline? Is there a point in our future wherein caffeine belongs in vitamins and even has a FDA-recommended daily allowance?

Getting the APNS device token

As alluded to in my previous post, this time I'm covering how to get the APNS device token for a given iOS client. Actually, it is pretty straightforward. First, call registerForRemoteNotificationTypes from your application's didFinishLaunchingWithOptions UIApplicationDelegate callback. You need to specify which type of notifications your application will accept. Here is an example:

- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions
{    
    ...

    // Register with the Apple Push Notification Service
    // so we can receive notifications from our server
    // application.  Upon successful registration, our 
    // didRegisterForRemoteNotificationsWithDeviceToken:
    // delegate callback will be invoked with our unique
    // device token.
    UIRemoteNotificationType allowedNotifications = UIRemoteNotificationTypeAlert
                                                  | UIRemoteNotificationTypeSound
                                                  | UIRemoteNotificationTypeBadge;
    [[UIApplication sharedApplication] registerForRemoteNotificationTypes:allowedNotifications];

    ...
    
    return YES;
}

As mentioned in the code comments, the application will then talk to Apple's Push Notification Service in the background and, when the device's unique token has been issued, your application delegate's didRegisterForRemoteNotificationsWithDeviceToken callback is invoked. This is where you actually get the device token.

- (void)application:(UIApplication *)application
        didRegisterForRemoteNotificationsWithDeviceToken:(NSData *)deviceToken
{
    NSLog(@"didRegisterForRemoteNotificationsWithDeviceToken: %@", deviceToken);
    // Stash the deviceToken data somewhere to send it to
    // your application server.
    ...
}

Once your iOS client application knows its own device token, it needs to send it to your application server so that the application server can push notifications back to the client later. How you do this depends on the architecture of your client-server communication.

At this point, I should add the device tokens can change. So, I recommend repeating the above logic every time your iOS client application starts so your application server always gets the latest device token for the user's terminal.

I would be remiss to not mention error handling. It is possible that the registerForRemoteNotificationTypes call will fail. The most obvious way it could fail is if the user does not have access to the 3G or WiFi networks and, as a result, cannot communicate with Apple's Push Notification Service; for example, when the device is in Airplane Mode with the wireless signals turned off.

In this case, the didFailToRegisterForRemoteNotificationsWithError delegate callback is invoked instead of didRegisterForRemoteNotificationsWithDeviceToken. In which case, you probably want to retry the registration later when network connectivity is restored.

Using pyapns with django

There is a handy daemon for sending push notifications to iOS-based mobile clients via Apple's Push Notification Service; it is called pyapns. It is implemented in python but, since it runs as a standalone XML-RPC server process, that fact is largely irrelevant. The important facts are that:

• It properly and fully implements the client interface to APNs, including the requirement for maintaining a persistent connection with Apple's servers rather than repeatedly setting-up and tearing-down SSL connections.
  
  
• It includes client libraries for communicating with the pyapns daemon from python and ruby, although any language that can speak XML-RPC (including C) will work too.
  

The way it works is: you first start the pyapns daemon process. This process acts as a XML-RPC server handling requests from your application(s), packing them into Apple's binary APNS protocol, and sending them to Apple to deliver to the iPhone, iPad, or iPod.

In order for your applications to send a push notification request, though, they must first tell pyapns which client certificate it should use to authenticate with the APNS servers. Here is a decent guide on how to obtain a client certificate. Once you have a certificate, you have to use the pyapns client library's configure and provision APIs to tell the pyapns daemon process to use your certificate.

If you are implementing your application in django, you can accomplish the configuration and provisioning directly from your django settings.py file like so:

# Configuration for connecting to the local pyapns daemon,
# including our certificate for pushing notifications to
# mobile terminals via APNS.
PYAPNS_CONFIG = {
    'HOST': 'http://localhost:7077/',
    'TIMEOUT': 15,
    'INITIAL': [
        ('MyAppName', 'path/to/cert/apns_sandbox.pem', 'sandbox'),
    ]
}

The pyapns python client library will automatically configure and provision itself from these settings. So, assuming you know the APNS device token of the mobile device you want to send a notification to, all you need to do to send a push notification is to call the pyapns.client.notify() function.

If only it were so easy. One complication arises in that the pyapns provisioning and configuration state is split between the client library and the pyapns daemon process. As a result, there are two scenarios to be wary of:

1. The django application is restarted. In this case, the client library, which is part of your django application, loses its state and tries to re-configure and re-provision itself from your django settings. Luckily, since the client library will re-read the configuration and provisioning settings from settings.py and seamlessly resume communication with the pyapns daemon.
   
   However, as noted in the pyapns documentation, "attempts to provision the same application id multiple times are ignored." As a result, if you change pyapns configuration in the settings.py file and restart django, you need to restart the pyapns daemon too for the new settings to take effect. Otherwise, if the settings are unchanged, the client library will seamlessly resume communication with the pyapns daemon.
   
   
2. The pyapns daemon is restarted. In this case, the client library thinks it has already configured and provisioned the daemon, but the daemon has lost this configuration due to restart. As a result, any attempt to send a push notification will fail as the daemon does not know how to establish the connection with Apple's Push Notification service.
   

As I mentioned above, the first scenario isn't a big deal. If you have to restart your web application or the web server for some reason, the connection between the pyapns client library and the daemon process will automatically resume right where it left off. In the rare case that you changed the pyapns settings in your django settings.py file, you need to restart both the django application and the pyapns daemon process for the new settings to take effect.

The latter scenario, though, is a bigger problem because it is impossible to detect until it is too late: that is, it doesn't manifest itself until you try to send a push notification and fail. Luckily, however, we can catch the failure condition and resolve the problem automatically. Specifically, if the pyapns client library fails to send a push notification to the daemon process due to the daemon process not being configured or provisioned, we can force the client library to re-configure and re-provision and retry.

So here you go, a wrapper around the pyapns client library to automatically recover when the backend pyapns daemon has been restarted:

"""
Wrappers for the pyapns client to simplify sending APNS
notifications, including support for re-configuring the
pyapns daemon after a restart.
"""

import pyapns.client
import time
import logging

log = logging.getLogger('APNS')

def notify(apns_token, message, badge=None, sound=None):
    """Push notification to device with the given message

    @param apns_token - The device's APNS-issued unique token
    @param message - The message to display in the
                     notification window
    """
    notification = {'aps': {'alert': message}}
    if badge is not None:
        notification['aps']['badge'] = int(badge)
    if sound is not None:
        notification['aps']['sound'] = str(sound)
    for attempt in range(4):
        try:
            pyapns.client.notify('MyAppId', apns_token,
                                 notification)
            break
        except (pyapns.client.UnknownAppID,
                pyapns.client.APNSNotConfigured):
            # This can happen if the pyapns server has been
            # restarted since django started running.  In
            # that case, we need to clear the client's
            # configured flag so we can reconfigure it from
            # our settings.py PYAPNS_CONFIG settings.
            if attempt == 3:
                log.exception()
            pyapns.client.OPTIONS['CONFIGURED'] = False
            pyapns.client.configure({})
            time.sleep(0.5)

Since I glossed over it in this post, I'll cover how to get the APNS device token for a mobile device in my next post. The device token acts as an address, telling Apple's Push Notification service which mobile device it should deliver your notification message to.