Commit d83e41cd authored by Francois Marier's avatar Francois Marier

Commit our version of gitdm for release stats

This comes from git://
parent f3b00ce0
The code in this directory can be distributed under the terms of the GNU
General Public License, version 2.
# Stuff for dealing with configuration files.
# This code is part of the LWN git data miner.
# Copyright 2007-11 Eklektix, Inc.
# Copyright 2007-11 Jonathan Corbet <>
# This file may be distributed under the terms of the GNU General
# Public License, version 2.
import sys, re, datetime, os.path
import database
# Read a line and strip out junk.
def ReadConfigLine (file):
line = file.readline ()
if not line:
return None
line = line.split('#')[0] # Get rid of any comments
line = line.strip () # and extra white space
if len (line) == 0: # we got rid of everything
return ReadConfigLine (file)
return line
# Give up and die.
def croak (message):
sys.stderr.write (message + '\n')
sys.exit (1)
# Read a list of email aliases.
def ReadEmailAliases (name):
file = open (name, 'r')
except IOError:
croak ('Unable to open email alias file %s' % (name))
line = ReadConfigLine (file)
while line:
m = re.match ('^("[^"]+"|\S+)\s+(.+)$', line)
if not m or len (m.groups ()) != 2:
croak ('Funky email alias line "%s"' % (line))
if m and (2).find ('@') <= 0:
croak ('Non-addresses in email alias "%s"' % (line))
database.AddEmailAlias ( (1).replace ('"', ''), (2))
line = ReadConfigLine (file)
file.close ()
# The Email/Employer map
EMMpat = re.compile (r'^([^\s]+)\s+([^<]+)\s*(<\s*(\d+-\d+-\d+)\s*)?$')
def ReadEmailEmployers (name):
file = open (name, 'r')
except IOError:
croak ('Unable to open email/employer file %s' % (name))
line = ReadConfigLine (file)
while line:
m = EMMpat.match (line)
if not m:
croak ('Funky email/employer line "%s"' % (line))
email = (1)
company = (2).strip ()
enddate = ParseDate ( (4))
database.AddEmailEmployerMapping (email, company, enddate)
line = ReadConfigLine (file)
file.close ()
def ParseDate (cdate):
if not cdate:
return None
sdate = cdate.split ('-')
return (int (sdate[0]), int (sdate[1]), int (sdate[2]))
def ReadGroupMap (fname, employer):
file = open (fname, 'r')
except IOError:
croak ('Unable to open group map file %s' % (fname))
line = ReadConfigLine (file)
while line:
database.AddEmailEmployerMapping (line, employer)
line = ReadConfigLine (file)
file.close ()
# Read in a virtual employer description.
def ReadVirtual (file, name):
ve = database.VirtualEmployer (name)
line = ReadConfigLine (file)
while line:
sl = line.split (None, 1)
first = sl[0]
if first == 'end': ()
# Zap the "%" syntactic sugar if it's there
if first[-1] == '%':
first = first[:-1]
percent = int (first)
except ValueError:
croak ('Bad split value "%s" for virtual empl %s' % (first, name))
if not (0 < percent <= 100):
croak ('Bad split value "%s" for virtual empl %s' % (first, name))
ve.addsplit (' '.join (sl[1:]), percent/100.0)
line = ReadConfigLine (file)
# We should never get here
croak ('Missing "end" line for virtual employer %s' % (name))
# Read an overall config file.
def ConfigFile (name, confdir):
file = open (name, 'r')
except IOError:
croak ('Unable to open config file %s' % (name))
line = ReadConfigLine (file)
while line:
sline = line.split (None, 2)
if len (sline) < 2:
croak ('Funky config line: "%s"' % (line))
if sline[0] == 'EmailAliases':
ReadEmailAliases (os.path.join (confdir, sline[1]))
elif sline[0] == 'EmailMap':
ReadEmailEmployers (os.path.join (confdir, sline[1]))
elif sline[0] == 'GroupMap':
if len (sline) != 3:
croak ('Funky group map line "%s"' % (line))
ReadGroupMap (os.path.join (confdir, sline[1]), sline[2])
elif sline[0] == 'VirtualEmployer':
ReadVirtual (file, ' '.join (sline[1:]))
croak ('Unrecognized config line: "%s"' % (line))
line = ReadConfigLine (file)
The code in this directory makes up the "git data miner," a simple hack
which attempts to figure things out from the revision history in a git
gitdm is a python script and doesn't need to be proper installed like other
normal programs. You just have to adjust your PATH variable, pointing it to
the directory of gitdm or alternatively create a symbolic link of the script
inside /usr/bin.
Before actually run gitdm you may want also to update the configuration file
(gitdm.config) with the needed information.
Run it like this:
git log -p -M [details] | gitdm [options]
The [details] tell git which changesets are of interest; the [options] can
-a If a patch contains signoff lines from both Andrew Morton
and Linus Torvalds, omit Linus's.
-b dir Specify the base directory to fetch the configuration files.
-c file Specify the name of the gitdm configuration file.
By default, "./gitdm.config" is used.
-d Omit the developer reports, giving employer information
-D Rather than create the usual statistics, create a
file (datelc) providing lines changed per day, where the first column
displays the changes happened only on that day and the second sums
the day it happnened with the previous ones. This option is suitable
for feeding to a tool like gnuplot.
-h file Generate HTML output to the given file
-l num Only list the top <num> entries in each report.
-o file Write text output to the given file (default is stdout).
-r pat Only generate statistics for changes to files whose
name matches the given regular expression.
-s Ignore Signed-off-by lines which match the author of
each patch.
-u Group all unknown developers under the "(Unknown)"
-x file Export raw statistics as CSV.
-w Aggregate the data by weeks instead of months in the
CSV file when -x is used.
-z Dump out the hacker database to "database.dump".
A typical command line used to generate the "who write 2.6.x" LWN articles
looks like:
git log -p -M v2.6.19..v2.6.20 | \
gitdm -u -s -a -o results -h results.html
The main purpose of the configuration file is to direct the mapping of
email addresses onto employers. Please note that the config file parser is
exceptionally stupid and unrobust at this point, but it gets the job done.
Blank lines and lines beginning with "#" are ignored. Everything else
specifies a file with some sort of mapping:
EmailAliases file
Developers often post code under a number of different email
addresses, but it can be desirable to group them all together in
the statistics. An EmailAliases file just contains a bunch of
lines of the form:
alias@address canonical@address
Any patches originating from alias@address will be treated as if
they had come from canonical@address.
It may happen that some people set their git user data in the
following form: " <Joe Hacker>". The
"Joe Hacker" is then considered as the email... but gitdm says
it is a "Funky" email. An alias line in the following form can
be used to alias these commits aliased to the correct email
"Joe Hacker"
EmailMap file
Map email addresses onto employers. These files contain lines
[user@]domain employer [< yyyy-mm-dd]
If the "user@" portion is missing, all email from the given domain
will be treated as being associated with the given employer. If a
date is provided, the entry is only valid up to that date;
otherwise it is considered valid into the indefinite future. This
feature can be useful for properly tracking developers' work when
they change employers but do not change email addresses.
GroupMap file employer
This is a variant of EmailMap provided for convenience; it contains
email addresses only, all of which are associated with the given
VirtualEmployer name
nn% employer1
This construct (which appears in the main configuration file)
allows causes the creation of a fake employer with the given
"name". It directs that any contributions attributed to that
employer should be split to other (real) employers using the given
percentages. The functionality works, but is primitive - there is,
for example, no check to ensure that the percentages add up to
something rational.
A few other tools have been added to this repository:
Reads a set of commits, then generates a graphviz file charting the
flow of patches into the mainline. Needs to be smarter, but, then,
so does everything else in this directory.
Simple brute-force crawler which outputs the names of any files
which have not been touched since the original (kernel) commit.
I needed to be able to quickly associate a given commit with the
major release which contains it. First attempt used
"git tags --contains="; after it ran for a solid week, I concluded
there must be a better way. This tool just reads through the repo,
remembering tags, and creating a Python dictionary containing the
association. The result is an ugly 10mb pickle file, but, even so,
it's still a better way.
Crawls through a directory hierarchy, counting how many lines of
code are associated with each major release. Needs the pickle file
from committags to get the job done.
Gitdm was written by Jonathan Corbet; many useful contributions have come
from Greg Kroah-Hartman.
Please note that this tool is provided in the hope that it will be useful,
but it is not put forward as an example of excellence in design or
implementation. Hacking on gitdm tends to stop the moment it performs
whatever task is required of it at the moment. Patches to make it less
hacky, less ugly, and more robust are welcome.
Jonathan Corbet
# Generate a database of commits and major versions they went into.
# committags [git-args]
# This code is part of the LWN git data miner.
# Copyright 2007-11 Eklektix, Inc.
# Copyright 2007-11 Jonathan Corbet <>
# This file may be distributed under the terms of the GNU General
# Public License, version 2.
import sys
import re
import os
import pickle
git = 'git log --decorate '
if len(sys.argv) > 1:
git += ' '.join(sys.argv[1:])
input = os.popen(git, 'r')
DB = { }
Tag = 'None'
tagline = re.compile(r'^commit ([\da-f]+) .*tag: (v2\.6\.\d\d)')
commit = re.compile(r'^commit ([\da-f]+)')
for line in input.readlines():
if not line.startswith('commit'):
continue # This makes it go faster
m =
if m:
DB[] = Tag =
m =
if m:
DB[] = Tag
print 'Found %d commits' % (len(DB.keys()))
out = open('committags.db', 'w')
pickle.dump(DB, out)
# aggregate per-month statistics for people
import sys, datetime
class CSVStat:
def __init__ (self, name, employer, date): = name
self.employer = employer
self.added = self.removed = 0 = date
def accumulate (self, p):
self.added = self.added + p.added
self.removed = self.removed + p.removed
PeriodCommitHash = { }
def AccumulatePatch (p, Aggregate):
date = "%.2d-%.2d-01"%(,
if (Aggregate == 'week'):
date = "%.2d-%.2d"%([0],[1])
authdatekey = "%s-%s"%(, date)
if authdatekey not in PeriodCommitHash:
empl = (,
stat = CSVStat (, empl, date)
PeriodCommitHash[authdatekey] = stat
stat = PeriodCommitHash[authdatekey]
stat.accumulate (p)
def OutputCSV (file):
if file is None:
file.write ("Name\tAffliation\tDate\tAdded\tRemoved\n")
for date, stat in PeriodCommitHash.items():
# sanitise names " is common and \" sometimes too
empl_name = ("\"", ".").replace ("\\", ".")
author_name = ("\"", ".").replace ("\\", ".")
file.write ("\"%s\"\t\"%s\"\t%s\t%d\t%d\n"%(author_name, empl_name,, \
stat.added, stat.removed))
# The "database".
# This code is part of the LWN git data miner.
# Copyright 2007-11 Eklektix, Inc.
# Copyright 2007-11 Jonathan Corbet <>
# This file may be distributed under the terms of the GNU General
# Public License, version 2.
import sys, datetime
class Hacker:
def __init__ (self, name, id, elist, email): = name = id
self.employer = [ elist ] = [ email ]
self.added = self.removed = 0
self.patches = [ ]
self.signoffs = [ ] = [ ]
self.tested = [ ]
self.reports = [ ]
self.testcred = self.repcred = 0
def addemail (self, email, elist): (email)
self.employer.append (elist)
HackersByEmail[email] = self
def emailemployer (self, email, date):
for i in range (0, len (
if[i] == email:
for edate, empl in self.employer[i]:
if edate > date:
return empl
print 'OOPS. ',, self.employer,, email, date
return None # Should not happen
def addpatch (self, patch):
self.added += patch.added
self.removed += patch.removed
self.patches.append (patch)
# There's got to be a better way.
def addsob (self, patch):
self.signoffs.append (patch)
def addreview (self, patch): (patch)
def addtested (self, patch):
self.tested.append (patch)
def addreport (self, patch):
self.reports.append (patch)
def reportcredit (self, patch):
self.repcred += 1
def testcredit (self, patch):
self.testcred += 1
HackersByName = { }
HackersByEmail = { }
HackersByID = { }
MaxID = 0
def StoreHacker (name, elist, email):
global MaxID
id = MaxID
MaxID += 1
h = Hacker (name, id, elist, email)
HackersByName[name] = h
HackersByEmail[email] = h
HackersByID[id] = h
return h
def LookupEmail (addr):
return HackersByEmail[addr]
except KeyError:
return None
def LookupName (name):
return HackersByName[name]
except KeyError:
return None
def LookupID (id):
return HackersByID[id]
except KeyError:
return None
def AllHackers ():
return HackersByID.values ()
# return [h for h in HackersByID.values ()] # if (h.added + h.removed) > 0]
def DumpDB ():
out = open ('database.dump', 'w')
names = HackersByName.keys ()
names.sort ()
for name in names:
h = HackersByName[name]
out.write ('%4d %s %d p (+%d -%d) sob: %d\n' % (,,
len (h.patches),
h.added, h.removed,
len (h.signoffs)))
for i in range (0, len (
out.write ('\t%s -> \n' % ([i]))
for date, empl in h.employer[i]:
out.write ('\t\t %d-%d-%d %s\n' % (date.year, date.month,,
# Employer info.
class Employer:
def __init__ (self, name): = name
self.added = self.removed = self.count = self.changed = 0
self.sobs = 0
self.hackers = [ ]
def AddCSet (self, patch):
self.added += patch.added
self.removed += patch.removed
self.changed += max(patch.added, patch.removed)
self.count += 1
if not in self.hackers:
self.hackers.append (
def AddSOB (self):
self.sobs += 1
Employers = { }
def GetEmployer (name):
return Employers[name]
except KeyError:
e = Employer (name)
Employers[name] = e
return e
def AllEmployers ():
return Employers.values ()
# Certain obnoxious developers, who will remain nameless (because we
# would never want to run afoul of Thomas) want their work split among
# multiple companies. Let's try to cope with that. Let's also hope
# this doesn't spread.
class VirtualEmployer (Employer):
def __init__ (self, name):
Employer.__init__ (self, name)
self.splits = [ ]
def addsplit (self, name, fraction):
self.splits.append ((name, fraction))
# Go through and (destructively) apply our credits to the
# real employer. Only one level of weirdness is supported.
def applysplits (self):
for name, fraction in self.splits:
real = GetEmployer (name)
real.added += int (self.added*fraction)
real.removed += int (self.removed*fraction)
real.changed += int (self.changed*fraction)
real.count += int (self.count*fraction)
self.__init__ (name) # Reset counts just in case
def store (self):
if Employers.has_key (
print Employers[]
sys.stderr.write ('WARNING: Virtual empl %s overwrites another\n'
% (
if len (self.splits) == 0:
sys.stderr.write ('WARNING: Virtual empl %s has no splits\n'
% (
# Should check that they add up too, but I'm lazy
Employers[] = self
# Mix all the virtual employers into their real destinations.
def MixVirtuals ():
for empl in AllEmployers ():
if isinstance (empl, VirtualEmployer):
empl.applysplits ()
# The email map.
EmailAliases = { }
def AddEmailAlias (variant, canonical):
if EmailAliases.has_key (variant):
sys.stderr.write ('Duplicate email alias for %s\n' % (variant))
EmailAliases[variant] = canonical
def RemapEmail (email):
email = email.lower ()
return EmailAliases[email]
except KeyError:
return email
# Email-to-employer mapping.
EmailToEmployer = { }
nextyear = () + datetime.timedelta (days = 365)
def AddEmailEmployerMapping (email, employer, end = nextyear):
if end is None:
end = nextyear
email = email.lower ()
empl = GetEmployer (employer)
l = EmailToEmployer[email]
for i in range (0, len(l)):
date, xempl = l[i]
if date == end: # probably both nextyear
print 'WARNING: duplicate email/empl for %s' % (email)
if date > end:
l.insert (i, (end, empl))
l.append ((end, empl))
except KeyError:
EmailToEmployer[email] = [(end, empl)]
def MapToEmployer (email, unknown = 0):
# Somebody sometimes does s/@/ at /; let's fix it.
email = email.lower ().replace (' at ', '@')
return EmailToEmployer[email]
except KeyError:
namedom = email.split ('@')
if len (namedom) < 2:
print 'Oops...funky email %s' % email
return [(nextyear, GetEmployer ('Funky'))]
s = namedom[1].split ('.')
for dots in range (len (s) - 2, -1, -1):
addr = '.'.join (s[dots:])
return EmailToEmployer[addr]
except KeyError:
if unknown:
return [(nextyear, GetEmployer ('(Unknown)'))]
return [(nextyear, GetEmployer (email))]
def LookupEmployer (email, mapunknown = 0):
elist = MapToEmployer (email, mapunknown)
return elist # GetEmployer (ename)
# Another quick hack of a script to find files unchanged
# since a given commit.
# This code is part of the LWN git data miner.
# Copyright 2007-11 Eklektix, Inc.
# Copyright 2007-11 Jonathan Corbet <>
# This file may be distributed under the terms of the GNU General
# Public License, version 2.
import sys, os
OriginalSin = '1da177e4c3f41524e886b7f1b8a0c1fc7321cac2'
def CheckFile(file):
git = os.popen('git log --pretty=oneline -1 ' + file, 'r')
line = git.readline()
if line.startswith(OriginalSin):
print file
# Here we just plow through all the files.
if len(sys.argv) != 2:
sys.stderr.write('Usage: findoldfiles directory\n')
files = os.popen('/usr/bin/find . -type f', 'r')
for file in files.readlines():
if file.find('.git/') < 0:
This diff is collapsed.
# This is a sample gitdm configuration file.
# EmailAliases lets us cope with developers who use more
# than one address.
EmailAliases sample-config/aliases
# EmailMap does the main work of mapping addresses onto
# employers.
EmailMap sample-con