build / android / pylib / results / json_results.py [blame]

# Copyright 2014 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


import collections
import itertools
import json
import logging
import time

from pylib.base import base_test_result

def GenerateResultsDict(test_run_results, global_tags=None):
  """Create a results dict from |test_run_results| suitable for writing to JSON.
  Args:
    test_run_results: a list of base_test_result.TestRunResults objects.
  Returns:
    A results dict that mirrors the one generated by
      base/test/launcher/test_results_tracker.cc:SaveSummaryAsJSON.
  """
  # Example json output.
  # {
  #   "global_tags": [],
  #   "all_tests": [
  #     "test1",
  #     "test2",
  #    ],
  #   "disabled_tests": [],
  #   "per_iteration_data": [
  #     {
  #       "test1": [
  #         {
  #           "status": "SUCCESS",
  #           "elapsed_time_ms": 1,
  #           "output_snippet": "",
  #           "output_snippet_base64": "",
  #           "losless_snippet": "",
  #         },
  #         ...
  #       ],
  #       "test2": [
  #         {
  #           "status": "FAILURE",
  #           "elapsed_time_ms": 12,
  #           "output_snippet": "",
  #           "output_snippet_base64": "",
  #           "losless_snippet": "",
  #         },
  #         ...
  #       ],
  #     },
  #     {
  #       "test1": [
  #         {
  #           "status": "SUCCESS",
  #           "elapsed_time_ms": 1,
  #           "output_snippet": "",
  #           "output_snippet_base64": "",
  #           "losless_snippet": "",
  #         },
  #       ],
  #       "test2": [
  #         {
  #           "status": "FAILURE",
  #           "elapsed_time_ms": 12,
  #           "output_snippet": "",
  #           "output_snippet_base64": "",
  #           "losless_snippet": "",
  #         },
  #       ],
  #     },
  #     ...
  #   ],
  # }

  all_tests = set()
  per_iteration_data = []
  test_run_links = {}

  for test_run_result in test_run_results:
    iteration_data = collections.defaultdict(list)
    if isinstance(test_run_result, list):
      results_iterable = itertools.chain(*(t.GetAll() for t in test_run_result))
      for tr in test_run_result:
        test_run_links.update(tr.GetLinks())

    else:
      results_iterable = test_run_result.GetAll()
      test_run_links.update(test_run_result.GetLinks())

    for r in results_iterable:
      result_dict = {
          'status': r.GetType(),
          'elapsed_time_ms': r.GetDuration(),
          'output_snippet': r.GetLog(),
          'losless_snippet': True,
          'output_snippet_base64': '',
          'links': r.GetLinks(),
      }
      iteration_data[r.GetName()].append(result_dict)

    all_tests = all_tests.union(set(iteration_data.keys()))
    per_iteration_data.append(iteration_data)

  return {
    'global_tags': global_tags or [],
    'all_tests': sorted(list(all_tests)),
    # TODO(jbudorick): Add support for disabled tests within base_test_result.
    'disabled_tests': [],
    'per_iteration_data': per_iteration_data,
    'links': test_run_links,
  }


def GenerateJsonTestResultFormatDict(test_run_results, interrupted):
  """Create a results dict from |test_run_results| suitable for writing to JSON.

  Args:
    test_run_results: a list of base_test_result.TestRunResults objects.
    interrupted: True if tests were interrupted, e.g. timeout listing tests
  Returns:
    A results dict that mirrors the standard JSON Test Results Format.
  """

  tests = {}
  counts = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'CRASH': 0, 'TIMEOUT': 0}

  for test_run_result in test_run_results:
    if isinstance(test_run_result, list):
      results_iterable = itertools.chain(*(t.GetAll() for t in test_run_result))
    else:
      results_iterable = test_run_result.GetAll()

    for r in results_iterable:
      element = tests
      for key in r.GetName().split('.'):
        if key not in element:
          element[key] = {}
        element = element[key]

      element['expected'] = 'PASS'

      if r.GetType() == base_test_result.ResultType.PASS:
        result = 'PASS'
      elif r.GetType() == base_test_result.ResultType.SKIP:
        result = 'SKIP'
      elif r.GetType() == base_test_result.ResultType.CRASH:
        result = 'CRASH'
      elif r.GetType() == base_test_result.ResultType.TIMEOUT:
        result = 'TIMEOUT'
      else:
        result = 'FAIL'

      if 'actual' in element:
        element['actual'] += ' ' + result
      else:
        counts[result] += 1
        element['actual'] = result
        if result == 'FAIL':
          element['is_unexpected'] = True

      if r.GetDuration() != 0:
        element['time'] = r.GetDuration()

  # Fill in required fields.
  return {
      'interrupted': interrupted,
      'num_failures_by_type': counts,
      'path_delimiter': '.',
      'seconds_since_epoch': time.time(),
      'tests': tests,
      'version': 3,
  }


def GenerateJsonResultsFile(test_run_result, file_path, global_tags=None,
                            **kwargs):
  """Write |test_run_result| to JSON.

  This emulates the format of the JSON emitted by
  base/test/launcher/test_results_tracker.cc:SaveSummaryAsJSON.

  Args:
    test_run_result: a base_test_result.TestRunResults object.
    file_path: The path to the JSON file to write.
  """
  with open(file_path, 'w') as json_result_file:
    json_result_file.write(json.dumps(
        GenerateResultsDict(test_run_result, global_tags=global_tags),
        **kwargs))
    logging.info('Generated json results file at %s', file_path)


def GenerateJsonTestResultFormatFile(test_run_result, interrupted, file_path,
                                     **kwargs):
  """Write |test_run_result| to JSON.

  This uses the official Chromium Test Results Format.

  Args:
    test_run_result: a base_test_result.TestRunResults object.
    interrupted: True if tests were interrupted, e.g. timeout listing tests
    file_path: The path to the JSON file to write.
  """
  with open(file_path, 'w') as json_result_file:
    json_result_file.write(
        json.dumps(
            GenerateJsonTestResultFormatDict(test_run_result, interrupted),
            **kwargs))
    logging.info('Generated json results file at %s', file_path)


def ParseResultsFromJson(json_results):
  """Creates a list of BaseTestResult objects from JSON.

  Args:
    json_results: A JSON dict in the format created by
                  GenerateJsonResultsFile.
  """

  def string_as_status(s):
    if s in base_test_result.ResultType.GetTypes():
      return s
    return base_test_result.ResultType.UNKNOWN

  results_list = []
  testsuite_runs = json_results['per_iteration_data']
  for testsuite_run in testsuite_runs:
    for test, test_runs in testsuite_run.items():
      results_list.extend(
          [base_test_result.BaseTestResult(test,
                                           string_as_status(tr['status']),
                                           duration=tr['elapsed_time_ms'],
                                           log=tr.get('output_snippet'))
          for tr in test_runs])
  return results_list