[jsonunittest] add a new jsonunittest module to make unittest produce JSON data (closes #184774)

This module proposes 2 classes (JsonTestResult and JsonTestRunner) that should be used together to run unittest and produce JSON output (on stdout by default).

authorDavid Douard <david.douard@logilab.fr>
changeset07d7716808ae
branchdefault
phasedraft
hiddenyes
parent revision#34363df41323 fix assertIsNotNone py< 2.7 implementation
child revision<not specified>
files modified by this revision
jsonunittest.py
test/unittest_jsonunittest.py
# HG changeset patch
# User David Douard <david.douard@logilab.fr>
# Date 1382447484 -7200
# Tue Oct 22 15:11:24 2013 +0200
# Node ID 07d7716808ae75cd099c0e58c6ad71c810b1ec40
# Parent 34363df4132345f7bcc41526a2956f9a6487a909
[jsonunittest] add a new jsonunittest module to make unittest produce JSON data (closes #184774)

This module proposes 2 classes (JsonTestResult and JsonTestRunner) that
should be used together to run unittest and produce JSON output (on stdout by default).

diff --git a/jsonunittest.py b/jsonunittest.py
@@ -0,0 +1,162 @@
1 +# -*- coding: utf-8 -*-
2 +# copyright 2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
3 +# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
4 +#
5 +# This file is part of logilab-common.
6 +#
7 +# logilab-common is free software: you can redistribute it and/or modify it under
8 +# the terms of the GNU Lesser General Public License as published by the Free
9 +# Software Foundation, either version 2.1 of the License, or (at your option) any
10 +# later version.
11 +#
12 +# logilab-common is distributed in the hope that it will be useful, but WITHOUT
13 +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 +# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
15 +# details.
16 +#
17 +# You should have received a copy of the GNU Lesser General Public License along
18 +# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
19 +
20 +# disable camel case warning
21 +# pylint: disable=C0103
22 +
23 +import os
24 +import sys
25 +from time import time
26 +
27 +import unittest as unittest_legacy
28 +if not getattr(unittest_legacy, "__package__", None):
29 +    try:
30 +        import unittest2 as unittest
31 +        from unittest2 import SkipTest
32 +        from unittest2.util import strclass
33 +        from unittest2.result import failfast
34 +    except ImportError:
35 +        raise ImportError("You have to install python-unittest2 to use %s" % __name__)
36 +else:
37 +    import unittest
38 +    from unittest import SkipTest
39 +    from unittest.util import strclass
40 +    from unittest.result import failfast
41 +
42 +
43 +class JsonTestResult(unittest.TestResult):
44 +    """A unitttest TestResult class that produces JSON structured output
45 +    """
46 +    def __init__(self, stream, descriptions, verbosity):
47 +        super(JsonTestResult, self).__init__(stream, descriptions, verbosity)
48 +        self.stream = stream
49 +        self.buffer = True
50 +        self.verbosity = verbosity
51 +        self.verbose = verbosity > 1
52 +        self.descriptions = descriptions
53 +        self._jsondata = []
54 +
55 +    def getDescription(self, test):
56 +        doc_first_line = test.shortDescription()
57 +        if self.descriptions and doc_first_line:
58 +            return '\n'.join((str(test), doc_first_line))
59 +        else:
60 +            return str(test)
61 +
62 +    def startTest(self, test):
63 +        super(JsonTestResult, self).startTest(test)
64 +        module = sys.modules.get(test.__module__)
65 +        path = os.path.abspath(module.__file__)
66 +        if path not in [e['path'] for e in self._jsondata]:
67 +            self._jsondata.append({
68 +                'cases': [],
69 +                'name': module.__name__,
70 +                'path': os.path.abspath(module.__file__),
71 +                'desc': module.__doc__,
72 +                })
73 +        [jsondata] = [e for e in self._jsondata if e['path']==path]
74 +        cases = jsondata['cases']
75 +        tcname = strclass(test.__class__).rsplit('.', 1)[-1]
76 +        if tcname not in [e['name'] for e in cases]:
77 +            cases.append({
78 +                    'name': tcname,
79 +                    'desc': test.__class__.__doc__,
80 +                    'tests': [],
81 +                    })
82 +        [currentcase] = [e for e in cases if e['name']==tcname]
83 +        currentcase['tests'].append({'name': test._testMethodName.rsplit('.', 1)[-1],
84 +                                     'desc': test._testMethodDoc})
85 +        self._json = currentcase['tests'][-1]
86 +
87 +    def stopTest(self, test):
88 +        self._json['stdout'] = self._stdout_buffer and self._stdout_buffer.getvalue()
89 +        self._json['stderr'] = self._stderr_buffer and self._stderr_buffer.getvalue()
90 +        super(JsonTestResult, self).stopTest(test)
91 +
92 +    def addSuccess(self, test):
93 +        super(JsonTestResult, self).addSuccess(test)
94 +        self._json['status'] = 'OK'
95 +
96 +    @failfast
97 +    def addError(self, test, err):
98 +        super(JsonTestResult, self).addError(test, err)
99 +        self._mirrorOutput = False
100 +        self._json['status'] = 'ERROR'
101 +        self._json['err'] = self._exc_info_to_string(err, test)
102 +
103 +    @failfast
104 +    def addFailure(self, test, err):
105 +        super(JsonTestResult, self).addFailure(test, err)
106 +        self._mirrorOutput = False
107 +        self._json['status'] = 'FAIL'
108 +        exctype, exc, tb = err
109 +        self._json['message'] = str(exc)
110 +        self._json['err'] = self._exc_info_to_string(err, test)
111 +
112 +    def addSkip(self, test, reason):
113 +        super(JsonTestResult, self).addSkip(test, reason)
114 +        self._json['status'] = 'SKIPPED'
115 +        self._json['reason'] = reason
116 +
117 +    def addExpectedFailure(self, test, err):
118 +        super(JsonTestResult, self).addExpectedFailure(test, err)
119 +        self._json['status'] = 'OK'
120 +        self._json['err'] = self._exc_info_to_string(err, test)
121 +
122 +    @failfast
123 +    def addUnexpectedSuccess(self, test):
124 +        super(JsonTestResult, self).addUnexpectedSuccess(test)
125 +        self._json['status'] = 'FAIL'
126 +        self._json['message'] = 'unexpected success'
127 +
128 +class JsonTestRunner(unittest.TextTestRunner):
129 +    """A unitttest TestRunner that produces JSON structured output
130 +    """
131 +    resultclass = JsonTestResult
132 +    def run(self, test):
133 +        "Run the given test case or test suite."
134 +        result = self._makeResult()
135 +        result.buffer = True
136 +        unittest.signals.registerResult(result)
137 +
138 +        startTime = time()
139 +        startTestRun = getattr(result, 'startTestRun', None)
140 +        if startTestRun is not None:
141 +            startTestRun()
142 +        try:
143 +            test(result)
144 +        finally:
145 +            stopTestRun = getattr(result, 'stopTestRun', None)
146 +            if stopTestRun is not None:
147 +                stopTestRun()
148 +        stopTime = time()
149 +        timeTaken = stopTime - startTime
150 +
151 +        jsdata = {'modules': result._jsondata,
152 +                  'start': startTime,
153 +                  'duration': timeTaken,
154 +                  }
155 +        if result.wasSuccessful():
156 +            jsdata['status'] = 'OK'
157 +        else:
158 +            jsdata['status'] = 'FAILED'
159 +        import json
160 +        self.stream.writeln(json.dumps(jsdata))
161 +        self.stream.writeln("")
162 +        return result
diff --git a/test/unittest_jsonunittest.py b/test/unittest_jsonunittest.py
@@ -0,0 +1,154 @@
163 +# copyright 2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
164 +# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
165 +#
166 +# This file is part of logilab-common.
167 +#
168 +# logilab-common is free software: you can redistribute it and/or modify it under
169 +# the terms of the GNU Lesser General Public License as published by the Free
170 +# Software Foundation, either version 2.1 of the License, or (at your option) any
171 +# later version.
172 +#
173 +# logilab-common is distributed in the hope that it will be useful, but WITHOUT
174 +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
175 +# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
176 +# details.
177 +#
178 +# You should have received a copy of the GNU Lesser General Public License along
179 +# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
180 +"""unittest module for logilab.comon.jsonunittest"""
181 +
182 +import os
183 +import sys
184 +from cStringIO import StringIO
185 +import json
186 +
187 +from logilab.common.jsonunittest import JsonTestRunner
188 +from logilab.common.pytest import NonStrictTestLoader
189 +
190 +from unittest import TestSuite, main, TestCase, TestLoader
191 +
192 +class JsonTestRunnerTC(TestCase):
193 +    class SimpleTC(TestCase):
194 +        def test_ok(self):
195 +            self.assertTrue(True)
196 +        def test_fail(self):
197 +            self.assertTrue(False)
198 +        def test_fail_w_msg(self):
199 +            self.assertTrue(False, "False really is not True")
200 +
201 +    class PrintTC(TestCase):
202 +        def test_ok(self):
203 +            "A test for OK"
204 +            print "a print statement"
205 +            self.assertTrue(True)
206 +        def test_fail(self):
207 +            print "a print statement"
208 +            self.assertTrue(False)
209 +        def test_fail_w_msg(self):
210 +            print "a print statement"
211 +            self.assertTrue(False, "False really is not True")
212 +    module = None
213 +
214 +    def setUp(self):
215 +        if self.module is None:
216 +            self.module = JsonTestRunnerTC
217 +            self.loader = TestLoader()
218 +            self.output = StringIO()
219 +            self.runner = JsonTestRunner(stream=self.output)
220 +            self.testsuite = self.loader.loadTestsFromModule(self.module)
221 +            self.result = self.runner.run(self.testsuite)
222 +
223 +    def test_json_struct(self):
224 +        testsuite = self.testsuite
225 +        self.assertEqual(2, len(testsuite._tests))
226 +        for test in testsuite._tests:
227 +            self.assertEqual(3, len(test._tests))
228 +
229 +        result = self.result
230 +        self.assertEqual(6, result.testsRun)
231 +        self.assertEqual(4, len(result.failures))
232 +        self.assertEqual(0, len(result.errors))
233 +
234 +        jsondata = json.loads(self.output.getvalue())
235 +
236 +        self.assertIsInstance(jsondata, dict)
237 +        for key in 'duration', 'start', 'modules', 'status':
238 +            self.assertIn(key, jsondata)
239 +
240 +        self.assertEqual("FAILED", jsondata['status'])
241 +
242 +        self.assertIsInstance(jsondata['modules'], list)
243 +        self.assertEqual(1, len(jsondata['modules']))
244 +
245 +        for test in jsondata['modules']:
246 +            for key in 'name', 'cases', 'path', 'desc':
247 +                self.assertIn(key, test)
248 +
249 +            self.assertIsInstance(test['cases'], list)
250 +            self.assertEqual(2, len(test['cases']))
251 +
252 +            for case in test['cases']:
253 +                for key in 'name', 'desc', 'tests':
254 +                    self.assertIn(key, case)
255 +                self.assertIsInstance(case['tests'], list)
256 +                self.assertEqual(3, len(case['tests']))
257 +
258 +    def find(self, value, seq, key='name'):
259 +        found = [elt for elt in seq if elt[key] == value]
260 +        self.assertEqual(1, len(found))
261 +        return found[0]
262 +
263 +    def test_simple_tc(self):
264 +        jsondata = json.loads(self.output.getvalue())
265 +        testmodule = self.find('unittest_jsonunittest', jsondata['modules'])
266 +        case = self.find('SimpleTC', testmodule['cases'])
267 +
268 +        testok = self.find('test_ok', case['tests'])
269 +        self.assertEqual('OK', testok['status'])
270 +        self.assertEqual('', testok['stdout'])
271 +        self.assertEqual('', testok['stderr'])
272 +        self.assertEqual(None, testok['desc'])
273 +
274 +        testfail = self.find('test_fail', case['tests'])
275 +        self.assertEqual('FAIL', testfail['status'])
276 +        self.assertEqual('', testfail['stdout'])
277 +        self.assertEqual('', testfail['stderr'])
278 +        self.assertEqual(None, testfail['desc'])
279 +        self.assertEqual('False is not true', testfail['message'])
280 +
281 +        testfail = self.find('test_fail_w_msg', case['tests'])
282 +        self.assertEqual('FAIL', testfail['status'])
283 +        self.assertEqual('', testfail['stdout'])
284 +        self.assertEqual('', testfail['stderr'])
285 +        self.assertEqual(None, testfail['desc'])
286 +        self.assertEqual('False really is not True', testfail['message'])
287 +
288 +    def test_wprint_tc(self):
289 +        jsondata = json.loads(self.output.getvalue())
290 +        testmodule = self.find('unittest_jsonunittest', jsondata['modules'])
291 +        case = self.find('PrintTC', testmodule['cases'])
292 +
293 +        testok = self.find('test_ok', case['tests'])
294 +        self.assertEqual('OK', testok['status'])
295 +        self.assertEqual('a print statement', testok['stdout'].strip())
296 +        self.assertEqual('', testok['stderr'])
297 +        self.assertEqual("A test for OK", testok['desc'])
298 +
299 +        testfail = self.find('test_fail', case['tests'])
300 +        self.assertEqual('FAIL', testfail['status'])
301 +        self.assertEqual('a print statement', testok['stdout'].strip())
302 +        self.assertEqual('', testfail['stderr'])
303 +        self.assertEqual(None, testfail['desc'])
304 +        self.assertEqual('False is not true', testfail['message'])
305 +
306 +        testfail = self.find('test_fail_w_msg', case['tests'])
307 +        self.assertEqual('FAIL', testfail['status'])
308 +        self.assertEqual('a print statement', testok['stdout'].strip())
309 +        self.assertEqual('', testfail['stderr'])
310 +        self.assertEqual(None, testfail['desc'])
311 +        self.assertEqual('False really is not True', testfail['message'])
312 +
313 +
314 +
315 +if __name__ == '__main__':
316 +    main()