Toponaming: performance testing

src/Mod/Test/TestPerf.py

@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+# ***************************************************************************
+# * *
+# * Copyright (c) 2024 [email protected] *
+# * *
+# * This file is part of FreeCAD. *
+# * *
+# * FreeCAD is free software: you can redistribute it and/or modify it *
+# * under the terms of the GNU Lesser General Public License as *
+# * published by the Free Software Foundation, either version 2.1 of the *
+# * License, or (at your option) any later version. *
+# * *
+# * FreeCAD is distributed in the hope that it will be useful, but *
+# * WITHOUT ANY WARRANTY; without even the implied warranty of *
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU *
+# * Lesser General Public License for more details. *
+# * *
+# * You should have received a copy of the GNU Lesser General Public *
+# * License along with FreeCAD. If not, see *
+# * <https://www.gnu.org/licenses/>. *
+# * *
+# ***************************************************************************
+
+import sys
+import unittest
+import FreeCAD as App
+import Part
+
+try:
+ from guppy import hpy
+
+ Memtest = True
+except ImportError:
+ Memtest = False
+
+try:
+ import cProfile
+
+ Pyprofile = True
+except ImportError:
+ Pyprofile = False
+
+
+class PerfTestCase(unittest.TestCase):
+ """
+ Special Test Case that takes a list of filenames after the "--pass" parameter to FreeCAD, and
+ runs a performance test by opening them, starting instrumentation, calling recompute(), and
+ then saving results.
+
+ Intended to be run as "<perf profiling> FreeCAD -t TestPerf --pass <modelname>
+
+ External perf profiling requires Python 3.12 or better, and a linux platform.
+ cProfile profiling and guppy memory information can run anywhere.
+ """
+
+ def setUp(self):
+ if "--pass" in sys.argv:
+ self.fileList = sys.argv[sys.argv.index("--pass") + 1 :]
+ else:
+ raise FileNotFoundError("Must provide filename parameter(s) via --pass")
+ if Part.Shape().ElementMapVersion == "":
+ self.tnp = ""
+ else:
+ self.tnp = ".tnp"
+ if Memtest:
+ # Use filename of first model with ".mprofile" appended for python memory use info.
+ self.memfile = open(self.fileList[0] + self.tnp + ".mprofile", "w", encoding="utf-8")
+
+ def testAll(self):
+ if Pyprofile:
+ # Generate a cProfile file as a python only time profile.
+ profile = cProfile.Profile()
+ profile.enable()
+ try:
+ # This is Python 3.12 on supported platforms ( linux ) only so that if we are run under
+ # an external 'perf' command, we report the python data. This can be extremely useful,
+ # because it contains not only time consumed, but python and c++ calls that took place
+ # so deep analysis can be performed on the resulting file. See calling script in
+ # tools/profile/perftest.sh for a wrapper.
+ sys.activate_stack_trampoline("perf")
+ except AttributeError:
+ pass # Totally okay if we don't have that, we can use the cProfile if it's there.
+
+ # Walk all files after the --pass. Normally one to avoid result intermingling.
+ for fileName in self.fileList:
+ doc = App.openDocument(fileName)
+ doc.recompute() # The heart of the performance measurement.
+ if Memtest:
+ # If guppy is available, take a heap snapshot and save it. Note that if multiple
+ # files are provided then their heap data sets will be appended to the same file.
+ dumpdata = hpy().heap()
+ dumpdata.stat.dump(self.memfile)
+ self.memfile.flush()
+ App.closeDocument(doc.Name)
+
+ try:
+ sys.deactivate_stack_trampoline()
+ except AttributeError:
+ pass
+ if Pyprofile:
+ profile.disable()
+ # Use filename of first model with ".cprofile" appended for python profiling information.
+ profile.dump_stats(self.fileList[0] + self.tnp + ".cprofile")
+ if Memtest:
+ self.memfile.close()

tools/profile/perftest.sh

@@ -0,0 +1,36 @@
+#! /bin/bash
+
+# Example file to drive the performance profiling python test from the shell.
+# Built to support the Topological Naming Problem fixes from early 2024;
+# this will likely need tweaking for your use.
+
+notnp=<Path to your first executable to profile goes here> #<dir>/bin/FreeCAD${cmd}
+tnp=<Path to your second executable to profile goes here> #<dir/bin/FreeCAD${cmd}
+results="results.txt" # File to append measurements to. We do not clear so xargs works on script
+
+perf record -o "$1.perf" $notnp -t TestPerf --pass "$@"
+perf record -o "$1.tnp.perf" $tnp -t TestPerf --pass "$@"
+
+# For interactive walking of the details using perf:
+#perf report -i $1.perf
+
+# After the two test runs above, process the resulting files to get the numbers.
+# For perf, use the 'script' command to pull the info out of the file, and then do a little
+# old school unix manipulation
+times=($(perf script -F time -i "$1.perf" | sed -e's/://' -e'1p;$!d')) # first, last timestamps
+totaltime=$(echo ${times[1]} - ${times[0]} | bc)
+memory=$(grep .size: "$1.mprofile" | cut -f2 -d\ )
+memory2=$(perf script --header -i "$1.perf" | grep "data size" | cut -f2 -d:)
+
+timestnp=($(perf script -F time -i "$1.tnp.perf" | sed -e's/://' -e'1p;$!d')) # first, last times
+totaltimetnp=$(echo ${timestnp[1]} - ${timestnp[0]} | bc)
+memorytnp=$(grep .size: "$1.tnp.mprofile" | cut -f2 -d\ )
+memory2tnp=$(perf script --header -i "$1.tnp.perf" | grep "data size" | cut -f2 -d:)
+
+# To calculate in this script instead of externally, you could do something like this:
+# delta=$(echo ${totaltimetnp} - ${totaltime} | bc)
+# percent=$(echo "scale=6; ${delta} / ${totaltime} * 100" | bc)
+
+# Summarize the run of one document into a CSV line suitable for importing into a spreadsheet.
+echo $totaltime,$totaltimetnp,$memory,$memorytnp,$memory2,$memory2tnp,"$1" >>"${results}"
+