GMail cleanup

This Python script cleans All Mail GMail folder by moving emails to Trash if there are more than 33 emails from same sender or email is more than 55 days old (values can be adjusted by editing code). IMAP must be enabled in order to work.

#!/usr/bin/env python
from datetime import datetime, timedelta
from email.utils import parsedate, parseaddr
import imaplib
import re
import socket

socket.setdefaulttimeout(30)

IMAP_SERVER = 'imap.gmail.com'
IMAP_PORT = 993


def main():
    clean('email1@gmail.com', 'password1')
    clean('email2@gmail.com', 'password2')


def clean(user, pwd):
    mbox = imaplib.IMAP4_SSL(IMAP_SERVER, IMAP_PORT)
    rc, resp = mbox.login(user, pwd)
    print rc, resp
    r, data = mbox.select("[Gmail]/All Mail")
    assert r == 'OK'
    r, data = mbox.expunge()
    assert r == 'OK'
    r, (ids,) = mbox.search(None, 'ALL')
    assert r == 'OK'
    r, data = mbox.fetch(
        '1:*',
        '(INTERNALDATE FLAGS BODY.PEEK[HEADER.FIELDS (FROM)])')
    assert r == 'OK'
    threads = {}
    for row in data:
        if row == ')':
            continue
        uid_date, subject = row
        m = re.compile(
            r'(\d+) \(INTERNALDATE "(.*)" FLAGS \((.*)\) '
            'BODY\[HEADER\.FIELDS\ \(FROM\)\]\ \{\d+\}').match(uid_date)
        assert m, uid_date
        uid, date, flags = m.groups()
        if '\\Flagged' in flags:
            continue
        date = parsedate(date)
        uid = int(uid)
        _, frm = parseaddr(subject)
        threads.setdefault(frm, []).append((date, uid))
    uids = []
    for key, lst in sorted(threads.items(), key=lambda x: len(x[1])):
        # date descending
        lst.sort(reverse=True)
        for no, (date, uid) in enumerate(lst):
            if no > 33 or datetime.now() - datetime(*date[:6]) > timedelta(55):
                print key, date, uid
                uids.append(uid)
    # uids.sort(reverse=True)
    print len(uids)
    if uids:
        r, data = mbox.store(
            #','.join(str(uid) for uid in uids), '+FLAGS', '\\Deleted')
            ','.join(str(uid) for uid in uids), '+X-GM-LABELS', '\\Trash')
        assert r == 'OK'
    r, data = mbox.expunge()
    assert r == 'OK'
    print 'ok'
    mbox.close()
    mbox.logout()


if __name__ == '__main__':
    main()

XFCE reboot on Ctrl+Alt+Delete

xfconf-query -c xfce4-keyboard-shortcuts -n -p "/commands/custom/Delete" -t string -s "xfce4-session-logout -r"

Exclude the only specific field from django serialization

MyModel._meta.get_field('my_field').serialize = False

Sort Firefox Bookmarks

% sqlite3 ~/.mozilla/firefox/*.default/places.sqlite 'update moz_bookmarks set position=(select count(*) from moz_bookmarks as b where lower(b.title) < lower(moz_bookmarks.title));'

Autologin in FreeBSD without GDM

Autologin and startx without GDM or other display manager:

1. Add this line to /etc/gettytab to describe new terminal line with autologin

   al.username:al=username:tc=std.230400:

   , where <username> is user to log in automatically. "al.username" is the terminal name which can be any.
2. Change ttyv0 line in /dev/ttys from this:

   ttyv0 "/usr/libexec/getty Pc" xterm on secure

   to this

   ttyv0 "/usr/libexec/getty al.username" xterm on secure

   to spawn auto logged in terminal at Alt+F1 console.
3. Set login shell on ttyv0 to start X11 immediately

   echo 'if $tty == ttyv0 startx' >> ~/.login

4. Reboot.
5. X11 will start automatically for <username>.

Django html5lib validating middleware

Django middleware to validate every HTML response with html5lib.

Usage

1. Save to middleware.py
2. Add 'middleware.ProfileMiddleware' to MIDDLEWARE_CLASSES

Listing

from html5lib.html5parser import HTMLParser
from html5lib import treebuilders
from django.http import HttpResponse


class ValidateMiddleware(object):
    def process_request(self, request):
        pass

    def process_response(self, request, response):
        if response.content and 'html' in response['Content-Type'] \
                and 'disable-validation' not in request.GET:
            # validate
            treebuilder = treebuilders.getTreeBuilder("simpleTree")
            parser = HTMLParser(tree=treebuilder, strict=True)
            try:
                parser.parse(response.content)
            except Exception:
                pass
            if parser.errors:
                # format output
                out = []
                lines = response.content.splitlines()
                for (row, col), e, d in parser.errors:
                    out.append('%s, %s' % (e, d))
                    for x in range(max(0, row - 3), min(len(lines), row + 1)):
                        out.append(lines[x])
                    out.append(' ' * col + '^')
                out = '\n'.join(out)
                return HttpResponse(out, mimetype='text/plain')
        return response

    def process_view(self, request, callback, callback_args, callback_kwargs):
        return callback(request, *callback_args, **callback_kwargs)

Django Profiling Middleware

This Django middleware can be used to collect and print accumulated profiling statistics on a running site. As opposed to popular individual view profiling tools this one profiles all existing views as they are being executed and saves statistics for later usage. On production (DEBUG=False) only each 20-th request is profiled to keep CPU overhead low.

Usage

1. Save to profile.py
2. Add 'profile.ProfileMiddleware' to MIDDLEWARE_CLASSES
3. ...run requests
4. Visit http://mysite/profile/ to print accumulated statistics
5. Visit http://mysite/myview?profile to profile this view only and print statistics immediately

Listing

from cStringIO import StringIO
from django.conf import settings
from django.http import HttpResponse
from os import path
from random import randrange
import cProfile
import pstats

# see pstats.Stats.print_stats
RESTRICTIONS = 30,
# to keep stats in memory
STATS_FILE = None
# to dump accumulated stats to file after every request
STATS_FILE = '/tmp/mysite.profile'


class ProfileMiddleware(object):
    empty = True
    prof = cProfile.Profile()

    def process_request(self, request):
        if request.path.startswith('/profile/'):
            # return collected stats
            if not (settings.DEBUG or request.user.is_staff):
                return HttpResponseForbidden(
                    'Forbidden', content_type='text/plain')
            return self.print_stats()

    def process_response(self, request, response):
        return response

    def process_view(self, request, callback, callback_args, callback_kwargs):
        if 'profile' in request.GET:
            # profile this view and print results
            self.prof.clear()
            self.prof.runcall(
                callback, request, *callback_args, **callback_kwargs)
            self.empty = False
            return self.print_stats(None)
        if not self.empty and not settings.DEBUG and randrange(20):
            # run without profiling
            return callback(request, *callback_args, **callback_kwargs)
        # run with profiling
        self.empty = False
        if STATS_FILE:
            self.prof.clear()
        retval = self.prof.runcall(
            callback, request, *callback_args, **callback_kwargs)
        if STATS_FILE:
            # merge stats
            self.save()
        return retval

    def save(self):
        stats = pstats.Stats(self.prof)
        stats.strip_dirs()
        if path.exists(STATS_FILE):
            try:
                stats.add(STATS_FILE)
            except (EOFError, ValueError):
                pass
        stats.dump_stats(STATS_FILE)

    def print_stats(self, input_file=STATS_FILE):
        # handle ourselfves
        out = StringIO()
        if input_file:
            stats = pstats.Stats(input_file, stream=out)
        else:
            if self.empty:
                return HttpResponse('No data yet.', content_type='text/plain')
            stats = pstats.Stats(self.prof, stream=out).strip_dirs()
        stats.sort_stats('cumulative').print_stats(*RESTRICTIONS)
        stats_str = out.getvalue()
        return HttpResponse(stats_str, content_type='text/plain')

See also

• Profiling Django
• Profiling Middleware snippet #1
• Profiling Middleware snippet #2