tests/sdcard/plot_sdcard.py - platform/system/extras - Git at Google

 #!/usr/bin/python2.5
 #
 # Copyright 2009, The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.


 """plot_sdcard: A module to plot the results of an sdcard perf test.

 Requires Gnuplot python v 1.8

 Typical usage:
  -t x axis is time
  -i x axis is iteration
  -p profile data generated by profile_sdcard.sh

 ./plot_sdcard.py -t /tmp/data.txt
 ./plot_sdcard.py -i /tmp/data.txt
 ./plot_sdcard.py -p

 python interpreter
 >>> import plot_sdcard as p
 >>> (metadata, data) = p.Parse('/tmp/data.txt')
 >>> p.PlotIterations(metadata, data)
 >>> p.PlotTimes(metadata, data)

 """

 import getopt
 from itertools import izip
 import re
 import sys
 import Gnuplot
 import numpy


 class DataSet(object):
   """Dataset holds the summary and data (time,value pairs)."""

   def __init__(self, line):
     res = re.search(('# StopWatch ([\w]+) total/cumulative '
                      'duration ([0-9.]+). Samples: ([0-9]+)'), line)
     self.time = []
     self.data = []
     self.name = res.group(1)
     self.duration = float(res.group(2))
     self.iteration = int(res.group(3))
     self.summary = re.match('([a-z_]+)_total', self.name)

   def __repr__(self):
     return str(zip(self.time, self.data))

   def Add(self, time, value):
     self.time.append(time)
     self.data.append(value)

   def RescaleTo(self, length):
     factor = len(self.data) / length

     if factor > 1:
       new_time = []
       new_data = []
       accum = 0.0
       idx = 1
       for t, d in izip(self.time, self.data):
         accum += d
         if idx % factor == 0:
           new_time.append(t)
           new_data.append(accum / factor)
           accum = 0
         idx += 1
       self.time = new_time
       self.data = new_data


 class Metadata(object):
   def __init__(self):
     self.kernel = ''
     self.command_line = ''
     self.sched = ''
     self.name = ''
     self.fadvise = ''
     self.iterations = 0
     self.duration = 0.0
     self.complete = False

   def Parse(self, line):
     if line.startswith('# Kernel:'):
       self.kernel = re.search('Linux version ([0-9.]+-[^ ]+)', line).group(1)
     elif line.startswith('# Command:'):
       self.command_line = re.search('# Command: [/\w_]+ (.*)', line).group(1)
       self.command_line = self.command_line.replace(' --', '-')
       self.command_line = self.command_line.replace(' -d', '')
       self.command_line = self.command_line.replace('--test=', '')
     elif line.startswith('# Iterations'):
       self.iterations = int(re.search('# Iterations: ([0-9]+)', line).group(1))
     elif line.startswith('# Fadvise'):
       self.fadvise = re.search('# Fadvise: ([\w]+)', line).group(1)
     elif line.startswith('# Sched'):
       self.sched = re.search('# Sched features: ([\w]+)', line).group(1)
       self.complete = True

   def AsTitle(self):
     return '%s-duration:%f\\n-%s\\n%s' % (
         self.kernel, self.duration, self.command_line, self.sched)

   def UpdateWith(self, dataset):
     self.duration = max(self.duration, dataset.duration)
     self.name = dataset.name


 def Parse(filename):
   """Parse a file with the collected data.

   The data must be in 2 rows (x,y).

   Args:
     filename: Full path to the file.
   """

   f = open(filename, 'r')

   metadata = Metadata()
   data = []  # array of dataset
   dataset = None

   for num, line in enumerate(f):
     try:
       line = line.strip()
       if not line: continue

       if not metadata.complete:
         metadata.Parse(line)
         continue

       if re.match('[a-z_]', line):
         continue

       if line.startswith('# StopWatch'):  # Start of a new dataset
         if dataset:
           if dataset.summary:
             metadata.UpdateWith(dataset)
           else:
             data.append(dataset)

         dataset = DataSet(line)
         continue

       if line.startswith('#'):
         continue

       # must be data at this stage
       try:
         (time, value) = line.split(None, 1)
       except ValueError:
         print 'skipping line %d: %s' % (num, line)
         continue

       if dataset and not dataset.summary:
         dataset.Add(float(time), float(value))

     except Exception:
       print 'Error parsing line %d' % num, sys.exc_info()[0]
       raise
   data.append(dataset)
   if not metadata.complete:
     print """Error missing metadata. Did you mount debugfs?
     [adb shell mount -t debugfs none /sys/kernel/debug]"""
     sys.exit(1)
   return (metadata, data)


 def PlotIterations(metadata, data):
   """Plot the duration of the ops against iteration.

   If you are plotting data with widely different runtimes you probably want to
   use PlotTimes instead.

   For instance when readers and writers are in the same mix, the
   readers will go thru 100 iterations much faster than the
   writers. The load test tries to be smart about that but the final
   iterations of the writers will likely be done w/o any influence from
   the readers.

   Args:
     metadata: For the graph's title.
     data: pair of to be plotted.
   """

   gp = Gnuplot.Gnuplot(persist=1)
   gp('set data style lines')
   gp.clear()
   gp.xlabel('iterations')
   gp.ylabel('duration in second')
   gp.title(metadata.AsTitle())
   styles = {}
   line_style = 1

   for dataset in data:
     dataset.RescaleTo(metadata.iterations)
     x = numpy.arange(len(dataset.data), dtype='int_')
     if not dataset.name in styles:
       styles[dataset.name] = line_style
       line_style += 1
       d = Gnuplot.Data(x, dataset.data,
                        title=dataset.name,
                        with_='lines ls %d' % styles[dataset.name])
     else:  # no need to repeat a title that exists already.
       d = Gnuplot.Data(x, dataset.data,
                        with_='lines ls %d' % styles[dataset.name])

     gp.replot(d)
   gp.hardcopy('/tmp/%s-%s-%f.png' %
               (metadata.name, metadata.kernel, metadata.duration),
               terminal='png')


 def PlotTimes(metadata, data):
   """Plot the duration of the ops against time elapsed.

   Args:
     metadata: For the graph's title.
     data: pair of to be plotted.
   """

   gp = Gnuplot.Gnuplot(persist=1)
   gp('set data style impulses')
   gp('set xtics 1')
   gp.clear()
   gp.xlabel('seconds')
   gp.ylabel('duration in second')
   gp.title(metadata.AsTitle())
   styles = {}
   line_style = 1

   for dataset in data:
     x = numpy.array(dataset.time, dtype='float_')
     if not dataset.name in styles:
       styles[dataset.name] = line_style
       line_style += 1
       d = Gnuplot.Data(x, dataset.data,
                        title=dataset.name,
                        with_='impulses ls %d' % styles[dataset.name])
     else:  # no need to repeat a title that exists already.
       d = Gnuplot.Data(x, dataset.data,
                        with_='impulses ls %d' % styles[dataset.name])

     gp.replot(d)
   gp.hardcopy('/tmp/%s-%s-%f.png' %
               (metadata.name, metadata.kernel, metadata.duration),
               terminal='png')


 def PlotProfile():
   """Plot the time of a run against the number of processes."""
   (metadata, data) = Parse('/tmp/sdcard-scalability.txt')
   gp = Gnuplot.Gnuplot(persist=1)
   gp('set data style impulses')
   gp('set xtics 1')
   gp('set pointsize 2')
   gp.clear()
   gp.xlabel('writer process')
   gp.ylabel('duration in second')
   gp.title(metadata.AsTitle())

   dataset = data[0]
   x = numpy.array(dataset.time, dtype='int_')
   d = Gnuplot.Data(x, dataset.data,
                    title=dataset.name,
                    with_='linespoints')
   gp.replot(d)
   gp.hardcopy('/tmp/%s-%s-%f.png' %
               (metadata.name, metadata.kernel, metadata.duration),
               terminal='png')


 def Usage():
   """Print this module's usage."""
   print """
   To plot the result using the iter number of the x axis:

     plot_sdcard.py -i /tmp/data.txt

   To plot the result using time for the x axis:

     plot_sdcard.py -t /tmp/data.txt

   To plot the result from the profiler:

     profile_sdcard.sh
     plot_sdcard.py -p

   """
   sys.exit(2)


 def main(argv):
   try:
     (optlist, args) = getopt.getopt(argv[1:],
                                     'itp', ['iteration', 'time', 'profile'])
   except getopt.GetoptError, err:
     print str(err)
     Usage()

   for flag, val in optlist:
     if flag in ('-i', '--iteration'):
       (metadata, data) = Parse(args[0])
       PlotIterations(metadata, data)
       sys.exit(0)
     elif flag in ('-t', '--time'):
       (metadata, data) = Parse(args[0])
       PlotTimes(metadata, data)
       sys.exit(0)
     elif flag in ('-p', '--profile'):
       PlotProfile()
       sys.exit(0)
   Usage()


 if __name__ == '__main__':
   if Gnuplot.__version__ != "1.8":
     print "Gnuplot should be 1.8. See REAME file"
     sys.exit(2)
   main(sys.argv)
	#!/usr/bin/python2.5
	#
	# Copyright 2009, The Android Open Source Project
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.


	"""plot_sdcard: A module to plot the results of an sdcard perf test.

	Requires Gnuplot python v 1.8

	Typical usage:
	-t x axis is time
	-i x axis is iteration
	-p profile data generated by profile_sdcard.sh

	./plot_sdcard.py -t /tmp/data.txt
	./plot_sdcard.py -i /tmp/data.txt
	./plot_sdcard.py -p

	python interpreter
	>>> import plot_sdcard as p
	>>> (metadata, data) = p.Parse('/tmp/data.txt')
	>>> p.PlotIterations(metadata, data)
	>>> p.PlotTimes(metadata, data)

	"""

	import getopt
	from itertools import izip
	import re
	import sys
	import Gnuplot
	import numpy


	class DataSet(object):
	"""Dataset holds the summary and data (time,value pairs)."""

	def __init__(self, line):
	res = re.search(('# StopWatch ([\w]+) total/cumulative '
	'duration ([0-9.]+). Samples: ([0-9]+)'), line)
	self.time = []
	self.data = []
	self.name = res.group(1)
	self.duration = float(res.group(2))
	self.iteration = int(res.group(3))
	self.summary = re.match('([a-z_]+)_total', self.name)

	def __repr__(self):
	return str(zip(self.time, self.data))

	def Add(self, time, value):
	self.time.append(time)
	self.data.append(value)

	def RescaleTo(self, length):
	factor = len(self.data) / length

	if factor > 1:
	new_time = []
	new_data = []
	accum = 0.0
	idx = 1
	for t, d in izip(self.time, self.data):
	accum += d
	if idx % factor == 0:
	new_time.append(t)
	new_data.append(accum / factor)
	accum = 0
	idx += 1
	self.time = new_time
	self.data = new_data


	class Metadata(object):
	def __init__(self):
	self.kernel = ''
	self.command_line = ''
	self.sched = ''
	self.name = ''
	self.fadvise = ''
	self.iterations = 0
	self.duration = 0.0
	self.complete = False

	def Parse(self, line):
	if line.startswith('# Kernel:'):
	self.kernel = re.search('Linux version ([0-9.]+-[^ ]+)', line).group(1)
	elif line.startswith('# Command:'):
	self.command_line = re.search('# Command: [/\w_]+ (.*)', line).group(1)
	self.command_line = self.command_line.replace(' --', '-')
	self.command_line = self.command_line.replace(' -d', '')
	self.command_line = self.command_line.replace('--test=', '')
	elif line.startswith('# Iterations'):
	self.iterations = int(re.search('# Iterations: ([0-9]+)', line).group(1))
	elif line.startswith('# Fadvise'):
	self.fadvise = re.search('# Fadvise: ([\w]+)', line).group(1)
	elif line.startswith('# Sched'):
	self.sched = re.search('# Sched features: ([\w]+)', line).group(1)
	self.complete = True

	def AsTitle(self):
	return '%s-duration:%f\\n-%s\\n%s' % (
	self.kernel, self.duration, self.command_line, self.sched)

	def UpdateWith(self, dataset):
	self.duration = max(self.duration, dataset.duration)
	self.name = dataset.name


	def Parse(filename):
	"""Parse a file with the collected data.

	The data must be in 2 rows (x,y).

	Args:
	filename: Full path to the file.
	"""

	f = open(filename, 'r')

	metadata = Metadata()
	data = [] # array of dataset
	dataset = None

	for num, line in enumerate(f):
	try:
	line = line.strip()
	if not line: continue

	if not metadata.complete:
	metadata.Parse(line)
	continue

	if re.match('[a-z_]', line):
	continue

	if line.startswith('# StopWatch'): # Start of a new dataset
	if dataset:
	if dataset.summary:
	metadata.UpdateWith(dataset)
	else:
	data.append(dataset)

	dataset = DataSet(line)
	continue

	if line.startswith('#'):
	continue

	# must be data at this stage
	try:
	(time, value) = line.split(None, 1)
	except ValueError:
	print 'skipping line %d: %s' % (num, line)
	continue

	if dataset and not dataset.summary:
	dataset.Add(float(time), float(value))

	except Exception:
	print 'Error parsing line %d' % num, sys.exc_info()[0]
	raise
	data.append(dataset)
	if not metadata.complete:
	print """Error missing metadata. Did you mount debugfs?
	[adb shell mount -t debugfs none /sys/kernel/debug]"""
	sys.exit(1)
	return (metadata, data)


	def PlotIterations(metadata, data):
	"""Plot the duration of the ops against iteration.

	If you are plotting data with widely different runtimes you probably want to
	use PlotTimes instead.

	For instance when readers and writers are in the same mix, the
	readers will go thru 100 iterations much faster than the
	writers. The load test tries to be smart about that but the final
	iterations of the writers will likely be done w/o any influence from
	the readers.

	Args:
	metadata: For the graph's title.
	data: pair of to be plotted.
	"""

	gp = Gnuplot.Gnuplot(persist=1)
	gp('set data style lines')
	gp.clear()
	gp.xlabel('iterations')
	gp.ylabel('duration in second')
	gp.title(metadata.AsTitle())
	styles = {}
	line_style = 1

	for dataset in data:
	dataset.RescaleTo(metadata.iterations)
	x = numpy.arange(len(dataset.data), dtype='int_')
	if not dataset.name in styles:
	styles[dataset.name] = line_style
	line_style += 1
	d = Gnuplot.Data(x, dataset.data,
	title=dataset.name,
	with_='lines ls %d' % styles[dataset.name])
	else: # no need to repeat a title that exists already.
	d = Gnuplot.Data(x, dataset.data,
	with_='lines ls %d' % styles[dataset.name])

	gp.replot(d)
	gp.hardcopy('/tmp/%s-%s-%f.png' %
	(metadata.name, metadata.kernel, metadata.duration),
	terminal='png')


	def PlotTimes(metadata, data):
	"""Plot the duration of the ops against time elapsed.

	Args:
	metadata: For the graph's title.
	data: pair of to be plotted.
	"""

	gp = Gnuplot.Gnuplot(persist=1)
	gp('set data style impulses')
	gp('set xtics 1')
	gp.clear()
	gp.xlabel('seconds')
	gp.ylabel('duration in second')
	gp.title(metadata.AsTitle())
	styles = {}
	line_style = 1

	for dataset in data:
	x = numpy.array(dataset.time, dtype='float_')
	if not dataset.name in styles:
	styles[dataset.name] = line_style
	line_style += 1
	d = Gnuplot.Data(x, dataset.data,
	title=dataset.name,
	with_='impulses ls %d' % styles[dataset.name])
	else: # no need to repeat a title that exists already.
	d = Gnuplot.Data(x, dataset.data,
	with_='impulses ls %d' % styles[dataset.name])

	gp.replot(d)
	gp.hardcopy('/tmp/%s-%s-%f.png' %
	(metadata.name, metadata.kernel, metadata.duration),
	terminal='png')


	def PlotProfile():
	"""Plot the time of a run against the number of processes."""
	(metadata, data) = Parse('/tmp/sdcard-scalability.txt')
	gp = Gnuplot.Gnuplot(persist=1)
	gp('set data style impulses')
	gp('set xtics 1')
	gp('set pointsize 2')
	gp.clear()
	gp.xlabel('writer process')
	gp.ylabel('duration in second')
	gp.title(metadata.AsTitle())

	dataset = data[0]
	x = numpy.array(dataset.time, dtype='int_')
	d = Gnuplot.Data(x, dataset.data,
	title=dataset.name,
	with_='linespoints')
	gp.replot(d)
	gp.hardcopy('/tmp/%s-%s-%f.png' %
	(metadata.name, metadata.kernel, metadata.duration),
	terminal='png')


	def Usage():
	"""Print this module's usage."""
	print """
	To plot the result using the iter number of the x axis:

	plot_sdcard.py -i /tmp/data.txt

	To plot the result using time for the x axis:

	plot_sdcard.py -t /tmp/data.txt

	To plot the result from the profiler:

	profile_sdcard.sh
	plot_sdcard.py -p

	"""
	sys.exit(2)


	def main(argv):
	try:
	(optlist, args) = getopt.getopt(argv[1:],
	'itp', ['iteration', 'time', 'profile'])
	except getopt.GetoptError, err:
	print str(err)
	Usage()

	for flag, val in optlist:
	if flag in ('-i', '--iteration'):
	(metadata, data) = Parse(args[0])
	PlotIterations(metadata, data)
	sys.exit(0)
	elif flag in ('-t', '--time'):
	(metadata, data) = Parse(args[0])
	PlotTimes(metadata, data)
	sys.exit(0)
	elif flag in ('-p', '--profile'):
	PlotProfile()
	sys.exit(0)
	Usage()


	if __name__ == '__main__':
	if Gnuplot.__version__ != "1.8":
	print "Gnuplot should be 1.8. See REAME file"
	sys.exit(2)
	main(sys.argv)