base / win / embedded_i18n / create_string_rc.py [blame]

#!/usr/bin/env python3
# Copyright 2012 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Generates .h and .rc files for strings extracted from a .grd file.

This script generates an rc file and header (NAME.{rc,h}) to be included in
a build target. The rc file includes translations for strings pulled from the
given .grd file(s) and their corresponding localized .xtb files.

To specify strings that will be extracted, the script pointed to by the
argument "extract-datafile" should contain one or both of the following global
variables:

STRING_IDS is a list of strings IDs we want to import from the .grd files and
include in the generated RC file. These strings are universal for all brands.

MODE_SPECIFIC_STRINGS: is a dictionary of strings for which there are brand
specific values. This mapping provides brand- and mode-specific string ids for a
given input id as described here:

{
  resource_id_1: {  # A resource ID for use with GetLocalizedString.
    brand_1: [  # 'google_chrome', for example.
      string_id_1,  # Strings listed in order of the brand's modes, as
      string_id_2,  # specified in install_static::InstallConstantIndex.
      ...
      string_id_N,
    ],
    brand_2: [  # 'chromium', for example.
      ...
    ],
  },
  resource_id_2:  ...
}

Note: MODE_SPECIFIC_STRINGS cannot be specified if STRING_IDS is not specified.

"""

# The generated header file includes IDs for each string, but also has values to
# allow getting a string based on a language offset.  For example, the header
# file looks like this:
#
# #define IDS_L10N_OFFSET_AR 0
# #define IDS_L10N_OFFSET_BG 1
# #define IDS_L10N_OFFSET_CA 2
# ...
# #define IDS_L10N_OFFSET_ZH_TW 41
#
# #define IDS_MY_STRING_AR 1600
# #define IDS_MY_STRING_BG 1601
# ...
# #define IDS_MY_STRING_BASE IDS_MY_STRING_AR
#
# This allows us to lookup an an ID for a string by adding IDS_MY_STRING_BASE
# and IDS_L10N_OFFSET_* for the language we are interested in.
#

from __future__ import print_function

import argparse
import collections
import glob
import io
import os
import sys
from xml import sax

BASEDIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(1, os.path.join(BASEDIR, '../../../tools/grit'))
sys.path.insert(2, os.path.join(BASEDIR, '../../../tools/python'))

from grit.extern import tclib

class GrdHandler(sax.handler.ContentHandler):
  """Extracts selected strings from a .grd file.

  Attributes:
    messages: A dict mapping string identifiers to their corresponding messages
      (key "text") and transconsole ids (key "tc_id").
    referenced_xtb_files: A list of all xtb files referenced inside the .grd
      file.
  """
  def __init__(self, string_id_set):
    """Constructs a handler that reads selected strings from a .grd file.

    The dict attribute |messages| is populated with the strings that are read.

    Args:
      string_id_set: An optional set of message identifiers to extract; all
      messages are extracted if empty.
    """
    sax.handler.ContentHandler.__init__(self)
    self.messages = collections.defaultdict(dict)
    self.referenced_xtb_files = []
    self.__id_set = string_id_set
    self.__message_name = None
    self.__element_stack = []
    self.__text_scraps = []

    # contains the text in the format required by transconsole to generate the
    # corresponding TC fingerprint.
    self.__tc_text_scraps = []

    self.__characters_callback = None

  def startElement(self, name, attrs):
    self.__element_stack.append(name)
    if name == 'message':
      self.__OnOpenMessage(attrs.getValue('name'))
    elif name == 'ph':
      self.__OnOpenPlaceholder(attrs.getValue('name'))
    elif name == 'file':
      parent = self.__element_stack[-2]
      if parent == 'translations':
        self.__OnAddXtbFile(attrs.getValue('path'))

  def endElement(self, name):
    popped = self.__element_stack.pop()
    assert popped == name
    if name == 'message':
      self.__OnCloseMessage()

  def characters(self, content):
    if self.__characters_callback:
      self.__characters_callback(self.__element_stack[-1], content)

  def __IsExtractingMessage(self):
    """Returns True if a message is currently being extracted."""
    return self.__message_name is not None

  def __OnOpenMessage(self, message_name):
    """Invoked at the start of a <message> with message's name."""
    assert not self.__IsExtractingMessage()
    self.__message_name = (message_name if (not (self.__id_set) or
                           message_name in self.__id_set)
                           else None)
    if self.__message_name:
      self.__characters_callback = self.__OnMessageText

  def __OnOpenPlaceholder(self, ph_name):
    """Invoked at the start of a <ph> with the `name` attribute."""
    if self.__IsExtractingMessage():
      # TC uses the `name` attribute as part of the fingerprint
      # generation.
      self.__tc_text_scraps.append(ph_name)

  def __OnMessageText(self, containing_element, message_text):
    """Invoked to handle a block of text for a message."""
    if message_text and (containing_element == 'message' or
                         containing_element == 'ph'):
      self.__text_scraps.append(message_text)
      if containing_element == 'message':
        self.__tc_text_scraps.append(message_text)

  def __OnCloseMessage(self):
    """Invoked at the end of a message."""
    if self.__IsExtractingMessage():
      self.messages[self.__message_name]["text"] = ''.join(
          self.__text_scraps).strip()

      # Generate the message ID for each source string to correlate it with its
      # TC translations in the .xtb files.
      self.messages[self.__message_name]["tc_id"] = tclib.GenerateMessageId(
          ''.join(self.__tc_text_scraps).strip())

      self.__message_name = None
      self.__text_scraps = []
      self.__tc_text_scraps = []
      self.__characters_callback = None

  def __OnAddXtbFile(self, xtb_file_path):
    """Adds the xtb file path of a 'file'."""
    if os.path.splitext(xtb_file_path)[1].lower() == '.xtb':
      self.referenced_xtb_files.append(xtb_file_path)

class XtbHandler(sax.handler.ContentHandler):
  """Extracts selected translations from an .xtd file.

  Populates the |lang| and |translations| attributes with the language and
  selected strings of an .xtb file. Instances may be re-used to read the same
  set of translations from multiple .xtb files.

  Attributes:
    translations: A mapping of translation ids to strings.
    lang: The language parsed from the .xtb file.
  """
  def __init__(self, translation_ids):
    """Constructs an instance to parse the given strings from an .xtb file.

    Args:
      translation_ids: a mapping of translation ids to their string
        identifiers list for the translations to be extracted.
    """
    sax.handler.ContentHandler.__init__(self)
    self.lang = None
    self.translations = None
    self.__translation_ids = translation_ids
    self.__element_stack = []
    self.__string_ids = None
    self.__text_scraps = []

    # The count of the `ph` tags.
    self.__ph_count = 0

    self.__characters_callback = None

  def startDocument(self):
    # Clear the lang and translations since a new document is being parsed.
    self.lang = ''
    self.translations = {}

  def startElement(self, name, attrs):
    self.__element_stack.append(name)
    # translationbundle is the document element, and hosts the lang id.
    if len(self.__element_stack) == 1:
      assert name == 'translationbundle'
      self.__OnLanguage(attrs.getValue('lang'))
    if name == 'translation':
      self.__OnOpenTranslation(attrs.getValue('id'))
    elif name == 'ph':
      self.__OnOpenPlaceholder()

  def endElement(self, name):
    popped = self.__element_stack.pop()
    assert popped == name
    if name == 'translation':
      self.__OnCloseTranslation()

  def characters(self, content):
    if self.__characters_callback:
      self.__characters_callback(self.__element_stack[-1], content)

  def __IsExtractingTranslation(self):
    """Returns `True` if a translation is currently being extracted."""
    return self.__string_ids is not None

  def __OnLanguage(self, lang):
    self.lang = lang.replace('-', '_').upper()

  def __OnOpenTranslation(self, translation_id):
    assert not self.__IsExtractingTranslation()
    self.__string_ids = self.__translation_ids.get(translation_id)
    if self.__string_ids:
      self.__characters_callback = self.__OnTranslationText

  def __OnOpenPlaceholder(self):
    if self.__IsExtractingTranslation():
      # The XTB files contain `ph` tags instead of placeholders, so we add the
      # placeholders in the format `$1` in place of the `ph` tags here.
      self.__ph_count += 1
      self.__text_scraps.append('$' + str(self.__ph_count))

  def __OnTranslationText(self, containing_element, message_text):
    if message_text and containing_element == 'translation':
      self.__text_scraps.append(message_text)

  def __OnCloseTranslation(self):
    if self.__IsExtractingTranslation():
      translated_string = ''.join(self.__text_scraps).strip()
      for string_id in self.__string_ids:
        self.translations[string_id] = translated_string
      self.__string_ids = None
      self.__text_scraps = []
      self.__ph_count = 0
      self.__characters_callback = None


class StringRcMaker(object):
  """Makes .h and .rc files containing strings and translations."""
  def __init__(self, inputs, expected_xtb_input_files, header_file, rc_file,
    brand, first_resource_id, string_ids_to_extract, mode_specific_strings):
    """Constructs a maker.

    Args:
      inputs: A list of (grd_file, xtb_dir) pairs containing the source data.
      expected_xtb_input_files: A list of xtb files that are expected to exist
        in the inputs folders. If there is a discrepency between what exists
        and what is expected the script will fail.
      header_file: The location of the header file to write containing all the
        defined string IDs.
      rc_file: The location of the rc file to write containing all the string
        resources.
      brand: The brand to check against when extracting mode-specific strings.
      first_resource_id: The starting ID for the generated string resources.
      string_ids_to_extract: The IDs of strings we want to import from the .grd
        files and include in the generated RC file. These strings are universal
        for all brands.
      mode_specific_strings: A dictionary of strings that have conditional
        values based on the brand's install mode. Refer to the documentation at
        the top of this file for more information on the format of the
        dictionary.
    """
    self.inputs = inputs
    self.expected_xtb_input_files = expected_xtb_input_files
    self.expected_xtb_input_files.sort()
    self.header_file = header_file
    self.rc_file = rc_file
    self.brand = brand
    self.first_resource_id = first_resource_id;
    self.string_id_set = set(string_ids_to_extract)
    self.mode_specific_strings = mode_specific_strings
    self.__AddModeSpecificStringIds()

  def MakeFiles(self):
    translated_strings = self.__ReadSourceAndTranslatedStrings()
    self.__WriteRCFile(translated_strings)
    self.__WriteHeaderFile(translated_strings)

  class __TranslationData(object):
    """A container of information about a single translation."""
    def __init__(self, resource_id_str, language, translation):
      self.resource_id_str = resource_id_str
      self.language = language
      self.translation = translation

    def __lt__(self, other):
      """Allow __TranslationDatas to be sorted by id then by language."""
      return (self.resource_id_str, self.language) < (other.resource_id_str,
                                                      other.language)

  def __AddModeSpecificStringIds(self):
    """Adds the mode-specific strings for all of the current brand's install
    modes to self.string_id_set."""
    for string_id, brands in self.mode_specific_strings.items():
      brand_strings = brands.get(self.brand)
      if not brand_strings:
        raise RuntimeError(
            'No strings declared for brand \'%s\' in MODE_SPECIFIC_STRINGS for '
            'message %s' % (self.brand, string_id))
      self.string_id_set.update(brand_strings)

  def __ReadSourceAndTranslatedStrings(self):
    """Reads the source strings and translations from all inputs."""
    translated_strings = []
    all_xtb_files = []
    for grd_file, xtb_dir in self.inputs:
      # Get the name of the grd file sans extension.
      source_name = os.path.splitext(os.path.basename(grd_file))[0]
      # Compute a glob for the translation files.
      xtb_pattern = os.path.join(os.path.dirname(grd_file), xtb_dir,
                                 '%s*.xtb' % source_name)
      local_xtb_files = [x.replace('\\', '/') for x in glob.glob(xtb_pattern)]
      all_xtb_files.extend(local_xtb_files)
      translated_strings.extend(
        self.__ReadSourceAndTranslationsFrom(grd_file, local_xtb_files))
    translated_strings.sort()
    all_xtb_files.sort()

    if self.expected_xtb_input_files != all_xtb_files:
      extra = list(set(all_xtb_files) - set(self.expected_xtb_input_files))
      missing = list(set(self.expected_xtb_input_files) - set(all_xtb_files))
      error = '''Asserted file list does not match.

Expected input files:
{}
Actual input files:
{}
Missing input files:
{}
Extra input files:
{}
'''
      print(error.format('\n'.join(self.expected_xtb_input_files),
                         '\n'.join(all_xtb_files), '\n'.join(missing),
                         '\n'.join(extra)))
      sys.exit(1)
    return translated_strings

  def __ReadSourceAndTranslationsFrom(self, grd_file, xtb_files):
    """Reads source strings and translations for a .grd file.

    Reads the source strings and all available translations for the messages
    identified by self.string_id_set (or all the messages if self.string_id_set
    is empty). The source string is used where translations are missing.

    Args:
      grd_file: Path to a .grd file.
      xtb_files: List of paths to .xtb files.

    Returns:
      An unsorted list of __TranslationData instances.
    """
    sax_parser = sax.make_parser()

    # Read the source (en-US) string from the .grd file.
    grd_handler = GrdHandler(self.string_id_set)
    sax_parser.setContentHandler(grd_handler)
    sax_parser.parse(grd_file)
    source_strings = grd_handler.messages

    grd_file_path = os.path.dirname(grd_file)
    source_xtb_files = []
    for xtb_file in grd_handler.referenced_xtb_files:
      relative_xtb_file_path = (
        os.path.join(grd_file_path, xtb_file).replace('\\', '/'))
      source_xtb_files.append(relative_xtb_file_path)
    missing_xtb_files = list(set(source_xtb_files) - set(xtb_files))

    # Manually put the source strings as en-US in the list of translated
    # strings.
    translated_strings = []
    for string_id, string_data in source_strings.items():
      translated_strings.append(self.__TranslationData(string_id,
                                                       'EN_US',
                                                       string_data["text"]))

    # Multiple source strings may have the same message text; hence the
    # message id is mapped to a list of string ids instead of a single value.
    translation_ids = {}
    for (string_id, string_data) in source_strings.items():
      translation_ids.setdefault(string_data["tc_id"], []).append(string_id);

    # Track any xtb files that appear in the xtb folder but are not present in
    # the grd file.
    extra_xtb_files = []
    # Gather the translated strings from the .xtb files. Use the en-US string
    # for any message lacking a translation.
    xtb_handler = XtbHandler(translation_ids)
    sax_parser.setContentHandler(xtb_handler)
    for xtb_filename in xtb_files:
      if not xtb_filename in source_xtb_files:
        extra_xtb_files.append(xtb_filename)
      sax_parser.parse(xtb_filename)
      for string_id, string_data in source_strings.items():
        translated_string = xtb_handler.translations.get(string_id,
                                                         string_data["text"])
        translated_strings.append(self.__TranslationData(string_id,
                                                         xtb_handler.lang,
                                                         translated_string))
    if missing_xtb_files or extra_xtb_files:
      if missing_xtb_files:
        missing_error = ("There were files that were found in the .grd file "
                         "'{}' but do not exist on disk:\n{}")
        print(missing_error.format(grd_file, '\n'.join(missing_xtb_files)))

      if extra_xtb_files:
        extra_error = ("There were files that exist on disk but were not found "
                       "in the .grd file '{}':\n{}")
        print(extra_error.format(grd_file, '\n'.join(extra_xtb_files)))

      sys.exit(1)
    return translated_strings

  def __WriteRCFile(self, translated_strings):
    """Writes a resource file with the strings provided in |translated_strings|.
    """
    HEADER_TEXT = (
      u'#include "%s"\n\n'
      u'STRINGTABLE\n'
      u'BEGIN\n'
      ) % os.path.basename(self.header_file)

    FOOTER_TEXT = (
      u'END\n'
    )

    with io.open(self.rc_file,
                 mode='w',
                 encoding='utf-16',
                 newline='\n') as outfile:
      outfile.write(HEADER_TEXT)
      for translation in translated_strings:
        # Escape special characters for the rc file.
        escaped_text = (translation.translation.replace('"', '""')
                       .replace('\t', '\\t')
                       .replace('\n', '\\n'))
        outfile.write(u'  %s "%s"\n' %
                      (translation.resource_id_str + '_' + translation.language,
                       escaped_text))
      outfile.write(FOOTER_TEXT)

  def __WriteHeaderFile(self, translated_strings):
    """Writes a .h file with resource ids."""
    # TODO(grt): Stream the lines to the file rather than building this giant
    # list of lines first.
    lines = []
    do_languages_lines = ['\n#define DO_LANGUAGES']
    installer_string_mapping_lines = ['\n#define DO_STRING_MAPPING']
    do_mode_strings_lines = ['\n#define DO_MODE_STRINGS']

    # Write the values for how the languages ids are offset.
    seen_languages = set()
    offset_id = 0
    for translation_data in translated_strings:
      lang = translation_data.language
      if lang not in seen_languages:
        seen_languages.add(lang)
        lines.append('#define IDS_L10N_OFFSET_%s %s' % (lang, offset_id))
        do_languages_lines.append('  HANDLE_LANGUAGE(%s, IDS_L10N_OFFSET_%s)'
                                  % (lang.replace('_', '-').lower(), lang))
        offset_id += 1
      else:
        break

    # Write the resource ids themselves.
    resource_id = self.first_resource_id
    for translation_data in translated_strings:
      lines.append('#define %s %s' % (translation_data.resource_id_str + '_' +
                                      translation_data.language,
                                      resource_id))
      resource_id += 1

    # Handle mode-specific strings.
    for string_id, brands in self.mode_specific_strings.items():
      # Populate the DO_MODE_STRINGS macro.
      brand_strings = brands.get(self.brand)
      if not brand_strings:
        raise RuntimeError(
            'No strings declared for brand \'%s\' in MODE_SPECIFIC_STRINGS for '
            'message %s' % (self.brand, string_id))
      do_mode_strings_lines.append(
        '  HANDLE_MODE_STRING(%s_BASE, %s)'
        % (string_id, ', '.join([ ('%s_BASE' % s) for s in brand_strings])))

    # Generate defines for the specific strings to extract or take all of the
    # strings found in the translations.
    if self.string_id_set:
      string_ids_to_write = self.string_id_set;
    else:
      string_ids_to_write = {t.resource_id_str for t in translated_strings}

    # Write out base ID values.
    for string_id in sorted(string_ids_to_write):
      lines.append('#define %s_BASE %s_%s' % (string_id,
                                              string_id,
                                              translated_strings[0].language))
      installer_string_mapping_lines.append('  HANDLE_STRING(%s_BASE, %s)'
                                            % (string_id, string_id))

    with open(self.header_file, 'w') as outfile:
      outfile.write('\n'.join(lines))
      outfile.write('\n#ifndef RC_INVOKED')
      outfile.write(' \\\n'.join(do_languages_lines))
      outfile.write(' \\\n'.join(installer_string_mapping_lines))
      outfile.write(' \\\n'.join(do_mode_strings_lines))
      # .rc files must end in a new line
      outfile.write('\n#endif  // ndef RC_INVOKED\n')


def BuildArgumentParser():
  parser = argparse.ArgumentParser(
    description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.add_argument('-b',
                      help='identifier of the browser brand (e.g., chromium).'
                      'This argument is mandatory if the module file included'
                      'by --extract-datafile contains MODE_SPECIFIC_STRINGS',
                      dest='brand')
  parser.add_argument('-i', action='append',
                      required=True,
                      help='path to .grd file',
                      dest='input_grd_files')
  parser.add_argument('-r', action='append',
                      required=True,
                      help='relative path to .xtb dir for each .grd file',
                      dest='input_xtb_relative_paths')
  parser.add_argument('-x', action='append',
                      required=True,
                      help='expected xtb input files to read',
                      dest='expected_xtb_input_files')
  parser.add_argument('--header-file',
                      required=True,
                      help='path to generated .h file to write',
                      dest='header_file')
  parser.add_argument('--rc-file',
                      required=True,
                      help='path to generated .rc file to write',
                      dest='rc_file')
  parser.add_argument('--first-resource-id',
                      type=int,
                      required=True,
                      help='first id for the generated string resources',
                      dest='first_resource_id')
  parser.add_argument('--extract-datafile',
                      help='the python file execute that will define the '
                      'specific strings to extract from the source .grd file.'
                      'The module should contain a global array STRING_IDS '
                      'that specifies which string IDs need to be extracted '
                      '(if no global member by that name exists, then all the '
                      'strings are extracted). It may also optionally contain '
                      'a dictionary MODE_SPECIFIC_STRINGS which defines the '
                      'mode-specific strings to use for a given brand that is '
                      'extracted.',
                      dest='extract_datafile')

  return parser


def main():
  parser = BuildArgumentParser()
  args = parser.parse_args()
  # Extract all the strings from the given grd by default.
  string_ids_to_extract = []
  mode_specific_strings = {}

  # Check to see if an external module containing string extraction information
  # was specified.
  extract_datafile = args.extract_datafile
  if extract_datafile:
    datafile_locals = dict();
    exec(open(extract_datafile).read(), globals(), datafile_locals)
    if 'STRING_IDS' in datafile_locals:
      string_ids_to_extract = datafile_locals['STRING_IDS']
    if 'MODE_SPECIFIC_STRINGS' in datafile_locals:
      if not string_ids_to_extract:
        parser.error('MODE_SPECIFIC_STRINGS was specified in file ' +
          extract_datafile + ' but there were no specific STRING_IDS '
          'specified for extraction')
      mode_specific_strings = datafile_locals['MODE_SPECIFIC_STRINGS']

  brand = args.brand
  if brand:
    if not mode_specific_strings:
      parser.error('A brand was specified (' + brand + ') but no mode '
        'specific strings were given.')
    valid_brands = [b for b in
      next(iter(mode_specific_strings.values())).keys()]
    if not brand in valid_brands:
      parser.error('A brand was specified (' + brand + ') but it is not '
        'a valid brand [' + ', '.join(valid_brands) + '].')
  elif mode_specific_strings:
    parser.error('MODE_SPECIFIC_STRINGS were specified but no brand was '
      'given.')

  grd_files = args.input_grd_files
  xtb_relative_paths = args.input_xtb_relative_paths

  if len(grd_files) != len(xtb_relative_paths):
    parser.error('Mismatch in number of grd files ({}) and xtb relative '
                 'paths ({})'.format(len(grd_files), len(xtb_relative_paths)))

  inputs = zip(grd_files, xtb_relative_paths)

  StringRcMaker(inputs, args.expected_xtb_input_files, args.header_file,
    args.rc_file,  brand, args.first_resource_id, string_ids_to_extract,
    mode_specific_strings).MakeFiles()
  return 0

if '__main__' == __name__:
  sys.exit(main())