Derrick Petzold

Open source enthusiast and developer


An auto random character field for Django

Views: 1,002 Comments: 0

Posted October 20, 2012 by derrick


Sometime ago I wrote about generating external ids. An external id is an unique id not related to the database id. For example something a link shortening service would use - http://bit.ly/RaFx8j. The original post had the logic in a model Manager and I did get some feedback saying it would be better implemented as a Django Field. Finally after just a couple of years I had a chance to implement it as such. It is based on the AutoSlugField from the excellent django-extensions app.

class RandomCharField(BaseUniqueField):

    def __init__(self, *args, **kwargs):
        kwargs.setdefault('blank', True)
        kwargs.setdefault('editable', False)

        self.lower = kwargs.pop('lower', False)
        self.digits_only = kwargs.pop('digits_only', False)
        self.alpha_only = kwargs.pop('alpha_only', False)
        self.include_punctuation = kwargs.pop('include_punctuation', False)
        self.length = kwargs.pop('length', 8)
        kwargs['max_length'] = self.length

        # legacy
        kwargs.pop('include_digits', False)

        if self.digits_only:
            self.valid_chars = string.digits
        else:
            self.valid_chars = string.lowercase

            if not self.lower:
                self.valid_chars += string.uppercase

            if not self.alpha_only:
                self.valid_chars += string.digits

                if self.include_punctuation:
                   self.valid_chars += string.punctuation

        super(RandomCharField, self).__init__(*args, **kwargs)

    def generate_chars(self, *args, **kwargs):
        return ''.join([random.choice(list(self.valid_chars)) for x in range(self.length)])

    def pre_save(self, model_instance, add):
        if not add:
            return getattr(model_instance, self.attname)

        initial = self.generate_chars()
        value = self.find_unique(model_instance, initial, self.generate_chars)
        setattr(model_instance, self.attname, value)
        return value

    def get_internal_type(self):
        return "CharField"

    def south_field_triple(self):
        "Returns a suitable description of this field for South."
        # We'll just introspect the _actual_ field.
        from south.modelsinspector import introspector
        field_class = '%s.RandomCharField' % (self.__module__)
        args, kwargs = introspector(self)
        kwargs.update({
            'alpha_only': repr(self.alpha_only),
            'digits_only': repr(self.digits_only),
            'include_punctuation': repr(self.include_punctuation),
            'length': repr(self.length),
            'lower': repr(self.lower),
        })
        # That's our definition!
        return (field_class, args, kwargs)

Below is some sample output:

>>> class RandomCharTestModel(models.Model):
>>>     chars = RandomCharField(length=6)
BVm9GE

>>> class RandomCharTestModelAlpha(models.Model):
>>>     chars = RandomCharField(length=12, alpha_only=True)
CxPWKJHDPnNO

>>> class RandomCharTestModelDigits(models.Model):
>>>     chars = RandomCharField(length=4, digits_only=True)
7097

>>> class RandomCharTestModelPunctuation(models.Model):
>>>     chars = RandomCharField(length=12, include_punctuation=True)
k[ZS.TR,0LHO    

>>> class RandomCharTestModelLower(models.Model):
>>>     chars = RandomCharField(length=12, lower=True, alpha_only=True)
pzolbemetmok

>>> class RandomCharTestModelLowerAlphaDigits(models.Model):
>>>     chars = RandomCharField(length=12, lower=True, include_punctuation=False)
wfaytk3msiin
For the full implementation including tests see my django utilitiy library on github.

Django request logging and json

Views: 6,680 Comments: 0

Posted January 15, 2012 by derrick


I wrote a decorator to make the request available to logging by using a LoggerAdpater and RequestInfo make the request object dict-like.

class RequestInfo(object):

    def __init__(self, request):
        self.request = request

    def __getitem__(self, name):

        if name == 'request.host':
            return socket.gethostname()

        if name.startswith('request.meta.'):
            val = name.split('.')[2]
            try:
                return self.request.META[val.upper()]
            except KeyError as e:
                return None
        return eval('self.%s' % (name))

    def _get_attrs(self, obj):
        attrs = []
        for attr in dir(obj):
            try:
                if not attr.startswith('_') and \
                        not callable(getattr(obj, attr)):
                    attrs.append(attr)
            except AttributeError:
                pass
        return attrs

    def __iter__(self):
        keys = ['request.host']
        keys.extend(['request.%s' % (a) for a in
                self._get_attrs(self.request)])
        keys.extend(['request.session.%s' % (a) for a in
            self._get_attrs(self.request.session)])
        keys.extend(['request.user.%s' % (a) for a in
            self._get_attrs(self.request.user)])
        keys.extend(['request.meta.%s' % (a.lower()) for a in
            self.request.META.keys()])
        return keys.__iter__()

def logger(name):
    def wrap(func):
        def caller(*args, **kwargs):
            request = None
            for arg in args:
                if isinstance(arg, HttpRequest):
                    request = arg
            if 'logger' not in kwargs:
                if request is not None:
                    kwargs['logger'] = logging.LoggerAdapter(
                            logging.getLogger(name), RequestInfo(request))
                else:
                    kwargs['logger'] = logging.getLogger(name)
            return func(*args, **kwargs)
        return caller
    return wrap
its meant to be used with the views like this
@log.logger(__name__)
def home(request, logger=None):
    logger.info('user performed some action')
update your logging.yaml to use the new format. For more info on that please see my previous post.
request-json:
    (): log.CustomJsonFormatter
    format: "'created', 'request.host', 'module', 'funcName', 'lineno', 'levelname', 'request.meta.remote_addr',  'request.meta.http_user_agent', 'request.user.username', 'request.path', 'message'"
and the resulting log message should look something like this
{"host": "fastcgi22", "request.meta.http_user_agent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.15 (KHTML, like Gecko) Ubuntu/11.04 Chromium/18.0.996.0 Chrome/18.0.996.0 Safari/535.15", "request.path": "/", "created": 1326706577.275584, "module": "log", "funcName": "caller", "request.user.username": "derrick", "lineno": 114, "request.meta.remote_addr": "76.93.216.138", "message": "user performed some action", "levelname": "INFO"}
Now django requests are fully integrated with logging. Happy Trails.
Download the source.

Generating external ids with django

Views: 2,306 Comments: 0
Tags:

Posted September 21, 2010 by derrick


Sometimes you want an alternative to the traditional consecutive database ids i.e. to keep people from walking your site. Below is the code to generate an external id (a random 5 character field) when a model instance is created. I have used this on a couple of sites now and I am pretty happy with it.

models.py

from django.contrib.auth.models import User
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.db import models

import random

class YourModelManager(models.Manager):

    @staticmethod
    def _random_id(prefix='', length=5):
        alphabet = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
        for x in range(length):
            prefix += random.choice(alphabet)
        return prefix

    @classmethod
    def _generate_external_id(cls):
        external_id = cls._random_id()
        while (YourModel.objects.filter(external_id=external_id).count() > 0):
            external_id = cls._random_id()
        return external_id

    def create(self, **kwargs):
        kwargs['external_id'] = self._generate_external_id()
        return self.get_query_set().create(**kwargs)

    def get_or_create(self, **kwargs):
        try:
            return self.get_query_set().get(**kwargs), False
        except self.model.DoesNotExist:
            kwargs['external_id'] = self._generate_external_id()
            return self.get_query_set().get_or_create(**kwargs)

class YourModel(models.Model):
    external_id = models.CharField(max_length=5, unique=True)
    objects = YourModelManager()
Download the source.

Django Facebook Authentication Backend

Views: 4,159 Comments: 2

Posted July 15, 2010 by derrick


Here is a Django authentication backend I wrote using Facebook's amazingly simple Graph API. It logs the user in using their Facebook credentials so you site doesn't have to worry about creating user profiles, validating, etc. See

Define the facebook tokens in settings.py and replace with the name of your app. You will probably want to modify the scope on the authorize link in the template, see the authentication permissions link.

backends.py

from django.conf import settings
from django.contrib.auth import models as auth_models

import cgi
import urllib
import simplejson

from <app_name> import models

class FacebookBackend:

    def authenticate(self, token=None):

        facebook_session = models.FacebookSession.objects.get(
            access_token=token,
        )

        profile = facebook_session.query('me')

        try:
            user = auth_models.User.objects.get(username=profile['id'])
        except auth_models.User.DoesNotExist, e:
            user = auth_models.User(username=profile['id'])

        user.set_unusable_password()
        user.email = profile['email']
        user.first_name = profile['first_name']
        user.last_name = profile['last_name']
        user.save()

        try:
            models.FacebookSession.objects.get(uid=profile['id']).delete()
        except models.FacebookSession.DoesNotExist, e:
            pass

        facebook_session.uid = profile['id']
        facebook_session.user = user
        facebook_session.save()

        return user

    def get_user(self, user_id):

        try:
            return auth_models.User.objects.get(pk=user_id)
        except auth_models.User.DoesNotExist:
            return None

models.py

from django.db import models
from django.contrib.auth.models import User

class FacebookSessionError(Exception):
    def __init__(self, error_type, message):
        self.message = message
        self.type = error_type
    def get_message(self):
        return self.message
    def get_type(self):
        return self.type
    def __unicode__(self):
        return u'%s: "%s"' % (self.type, self.message)

class FacebookSession(models.Model):

    access_token = models.CharField(max_length=103, unique=True)
    expires = models.IntegerField(null=True)

    user = models.ForeignKey(User, null=True)
    uid = models.BigIntegerField(unique=True, null=True)

    class Meta:
        unique_together = (('user', 'uid'), ('access_token', 'expires'))

    def query(self, object_id, connection_type=None, metadata=False):
        import urllib
        import simplejson

        url = 'https://graph.facebook.com/%s' % (object_id)
        if connection_type:
            url += '/%s' % (connection_type)

        params = {'access_token': self.access_token}
        if metadata:
            params['metadata'] = 1

        url += '?' + urllib.urlencode(params)
        response = simplejson.load(urllib.urlopen(url))
        if 'error' in response:
            error = response['error']
            raise FacebookSessionError(error['type'], error['message'])
        return response

views.py

from django.contrib import auth
from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response
from django.template import RequestContext

import cgi
import simplejson
import urllib

from <app_name> import settings

def login(request):
    error = None

    if request.user.is_authenticated():
        return HttpResponseRedirect('/yay/')

    if request.GET:
        if 'code' in request.GET:
            args = {
                'client_id': settings.FACEBOOK_APP_ID,
                'redirect_uri': settings.FACEBOOK_REDIRECT_URI,
                'client_secret': settings.FACEBOOK_API_SECRET,
                'code': request.GET['code'],
            }

            url = 'https://graph.facebook.com/oauth/access_token?' + \
                    urllib.urlencode(args)
            response = cgi.parse_qs(urllib.urlopen(url).read())
            access_token = response['access_token'][0]
            expires = response['expires'][0]

            facebook_session = models.FacebookSession.objects.get_or_create(
                access_token=access_token,
            )[0]

            facebook_session.expires = expires
            facebook_session.save()

            user = auth.authenticate(token=access_token)
            if user:
                if user.is_active:
                    auth.login(request, user)
                    return HttpResponseRedirect('/yay/')
                else:
                    error = 'AUTH_DISABLED'
            else:
                error = 'AUTH_FAILED'
        elif 'error_reason' in request.GET:
            error = 'AUTH_DENIED'

    template_context = {'settings': settings, 'error': error}
    return render_to_response('login.html', template_context, context_instance=RequestContext(request))

settings.py

FACEBOOK_APP_ID = ''
FACEBOOK_API_KEY = ''
FACEBOOK_API_SECRET = ''
FACEBOOK_REDIRECT_URI = 'http://example.com/login/'

AUTHENTICATION_BACKENDS = (
    '<app_name>.backends.FacebookBackend',
)

login.html

{% if error %}
      {% if error == 'AUTH_FAILED' %}
          <p>Authentication failed</p>
      {% else %}{% if error == 'AUTH_DISABLED' %}
          <p>Your account is disabled</p>
      {% else %}{% if error == 'AUTH_DENIED' %}
          <p>You did not allow access</p>
       {% endif %}{% endif %}{% endif %}
  {% else %}
    <a href="https://graph.facebook.com/oauth/authorize?client_id={{ settings.FACEBOOK_APP_ID }}&redirect_uri={{ settings.FACEBOOK_REDIRECT_URI }}&scope=publish_stream,email&display=popup">
      <img src="http://developers.facebook.com/images/devsite/login-button.png"/>
    </a>
  {% endif %}
Download the source.

Comparison of IN, GROUP BY and COUNT using Hibernate, Django and SQLAlchemy

Views: 3,219 Comments: 0

Posted July 7, 2010 by derrick


The other day I wrote about how to do a IN and GROUP BY query using Java's de facto ORM, Hibernate. I thought it would be interesting to see how other ORMs handled the same query. This is the query I want to generate:
SELECT COUNT(*),state FROM download_request WHERE id IN (<id list>) GROUP BY state;
Below is the code, output and SQL generated for the three ORMs.

Hibernate

class HibernateDAO implements ApplicationDAO {
public Map getStateCounts(final Collection ids) {
  HibernateSession hibernateSession = new HibernateSession();
  Session session = hibernateSession.getSession();
  Criteria criteria = session.createCriteria(DownloadRequestEntity.class)
	.add(Restrictions.in("id", ids));
  ProjectionList projectionList = Projections.projectionList();
  projectionList.add(Projections.groupProperty("state"));
  projectionList.add(Projections.rowCount());
  criteria.setProjection(projectionList);
  List results = criteria.list();
  Map stateMap = new HashMap();
  for(Object[] obj: results) {
      	DownloadState downloadState = (DownloadState)obj[0];
       	stateMap.put(downloadState.getDescription().toLowerCase(), (Integer)obj[1]);
  }
  hibernateSession.closeSession();
  return stateMap;
}
public static void main(String args[]) {
    HibernateDAO downloadRequestDAO = new HibernateDAO();	
    Collection ids = new ArrayList();
    for (int i = 1000;  i < 1010; i++ )
        ids.add(i);
    Map stateCounts =  downloadRequestDAO.getStateCounts(ids);
    for (String state: stateCounts.keySet()) {
        System.out.println(state + ": " + stateCounts.get(state));
    }
}
}

Output

failed: 5
downloaded: 1
completed: 4

SQL

select this_.state as y0_, count(*) as y1_ from download_request this_ 
where this_.id in (1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009) 
group by this_.state

Django

counts = models.DownloadRequest.objects.filter(
    id__in=range(1000, 1010),
).values('state').annotate(Count('state'))
for count in counts:
    print count

Output

{'state': u'FAILED', 'state__count': 5}
{'state': u'COMPLETED', 'state__count': 4}
{'state': u'DOWNLOADED', 'state__count': 1}

SQL

SELECT `download_request`.`state`, COUNT(`download_request`.`state`) 
AS `state__count` FROM `download_request` 
WHERE `download_request`.`id` IN (1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009) 
GROUP BY `download_request`.`state` ORDER BY NULL

SQLAlchmey

query = session.query(
    func.count(DownloadRequest.state), DownloadRequest.state,
).filter(
    DownloadRequest.id.in_(range(1000,1010)),
).group_by(DownloadRequest.state)
for count in query.all():
    print count

Output

(4L, 'COMPLETED')
(1L, 'DOWNLOADED')
(5L, 'FAILED')

SQL

SELECT count(download_request.state) AS count_1, download_request.state 
AS download_request_state FROM download_request 
WHERE download_request.id IN (1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009) 
GROUP BY download_request.state
As you can see SQLAlchemy is the most similar to SQL, django's is the briefest and Hibernate (obviously) is the most Java-like. Of the three I'd say I like SQLAlchemy the best as it is the most similar to SQL and me being from an SQL background it is the most natural. However all three get the job done and it is always great to have options.
Download the source.

derrickpetzold.com

My home on the web. Uses the Django web framework, uwsgi as the WSGI server, nginx as the media server and load balancer, pygments for the syntax highlighting. Author and designer.

crowdtube.tv

crowdtube.tv
CrowdTube.tv streams trending videos to your browser like a never ending tv show. This was Cory Shaw's awesome idea. I was responsible for the backend development.

dmusic.bz

dMusic.bz is Pandora clone written using Django and JavaScript. It uses last.fm to find the music relationships. Author and designer.

ilovephotos.com

ilovephotos.com
ilovephotos.com is a photo sharing and tagging website. Facial detection was run on the photos so the bounding boxes were already drawn around the faces eliminating a step in a tedious process. I authored views and the process to run facial detection on the photos and upload them to S3.

kindfish.com

kindfish.com
Kindfish.com was BlueLava's first photo site. It was event based and slideshows would be created from the albums. I authored the views and process to generate the slideshows from the photos on EC2.

Below is a snippet of my resume. Click here to view the full version in pdf, its proper format.

captcha