/external/v8/tools/js2c.py

  1#!/usr/bin/env python
  2#
  3# Copyright 2006-2008 the V8 project authors. All rights reserved.
  4# Redistribution and use in source and binary forms, with or without
  5# modification, are permitted provided that the following conditions are
  6# met:
  7#
  8#     * Redistributions of source code must retain the above copyright
  9#       notice, this list of conditions and the following disclaimer.
 10#     * Redistributions in binary form must reproduce the above
 11#       copyright notice, this list of conditions and the following
 12#       disclaimer in the documentation and/or other materials provided
 13#       with the distribution.
 14#     * Neither the name of Google Inc. nor the names of its
 15#       contributors may be used to endorse or promote products derived
 16#       from this software without specific prior written permission.
 17#
 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 29
 30# This is a utility for converting JavaScript source code into C-style
 31# char arrays. It is used for embedded JavaScript code in the V8
 32# library.
 33
 34import os, re, sys, string
 35import jsmin
 36import bz2
 37
 38
 39def ToCAsciiArray(lines):
 40  result = []
 41  for chr in lines:
 42    value = ord(chr)
 43    assert value < 128
 44    result.append(str(value))
 45  return ", ".join(result)
 46
 47
 48def ToCArray(lines):
 49  result = []
 50  for chr in lines:
 51    result.append(str(ord(chr)))
 52  return ", ".join(result)
 53
 54
 55def RemoveCommentsAndTrailingWhitespace(lines):
 56  lines = re.sub(r'//.*\n', '\n', lines) # end-of-line comments
 57  lines = re.sub(re.compile(r'/\*.*?\*/', re.DOTALL), '', lines) # comments.
 58  lines = re.sub(r'\s+\n+', '\n', lines) # trailing whitespace
 59  return lines
 60
 61
 62def ReadFile(filename):
 63  file = open(filename, "rt")
 64  try:
 65    lines = file.read()
 66  finally:
 67    file.close()
 68  return lines
 69
 70
 71def ReadLines(filename):
 72  result = []
 73  for line in open(filename, "rt"):
 74    if '#' in line:
 75      line = line[:line.index('#')]
 76    line = line.strip()
 77    if len(line) > 0:
 78      result.append(line)
 79  return result
 80
 81
 82def LoadConfigFrom(name):
 83  import ConfigParser
 84  config = ConfigParser.ConfigParser()
 85  config.read(name)
 86  return config
 87
 88
 89def ParseValue(string):
 90  string = string.strip()
 91  if string.startswith('[') and string.endswith(']'):
 92    return string.lstrip('[').rstrip(']').split()
 93  else:
 94    return string
 95
 96
 97EVAL_PATTERN = re.compile(r'\beval\s*\(')
 98WITH_PATTERN = re.compile(r'\bwith\s*\(')
 99
100
101def Validate(lines, file):
102  lines = RemoveCommentsAndTrailingWhitespace(lines)
103  # Because of simplified context setup, eval and with is not
104  # allowed in the natives files.
105  eval_match = EVAL_PATTERN.search(lines)
106  if eval_match:
107    raise ("Eval disallowed in natives: %s" % file)
108  with_match = WITH_PATTERN.search(lines)
109  if with_match:
110    raise ("With statements disallowed in natives: %s" % file)
111
112
113def ExpandConstants(lines, constants):
114  for key, value in constants:
115    lines = key.sub(str(value), lines)
116  return lines
117
118
119def ExpandMacros(lines, macros):
120  # We allow macros to depend on the previously declared macros, but
121  # we don't allow self-dependecies or recursion.
122  for name_pattern, macro in reversed(macros):
123    pattern_match = name_pattern.search(lines, 0)
124    while pattern_match is not None:
125      # Scan over the arguments
126      height = 1
127      start = pattern_match.start()
128      end = pattern_match.end()
129      assert lines[end - 1] == '('
130      last_match = end
131      arg_index = [0]  # Wrap state into array, to work around Python "scoping"
132      mapping = { }
133      def add_arg(str):
134        # Remember to expand recursively in the arguments
135        replacement = ExpandMacros(str.strip(), macros)
136        mapping[macro.args[arg_index[0]]] = replacement
137        arg_index[0] += 1
138      while end < len(lines) and height > 0:
139        # We don't count commas at higher nesting levels.
140        if lines[end] == ',' and height == 1:
141          add_arg(lines[last_match:end])
142          last_match = end + 1
143        elif lines[end] in ['(', '{', '[']:
144          height = height + 1
145        elif lines[end] in [')', '}', ']']:
146          height = height - 1
147        end = end + 1
148      # Remember to add the last match.
149      add_arg(lines[last_match:end-1])
150      result = macro.expand(mapping)
151      # Replace the occurrence of the macro with the expansion
152      lines = lines[:start] + result + lines[end:]
153      pattern_match = name_pattern.search(lines, start + len(result))
154  return lines
155
156class TextMacro:
157  def __init__(self, args, body):
158    self.args = args
159    self.body = body
160  def expand(self, mapping):
161    result = self.body
162    for key, value in mapping.items():
163        result = result.replace(key, value)
164    return result
165
166class PythonMacro:
167  def __init__(self, args, fun):
168    self.args = args
169    self.fun = fun
170  def expand(self, mapping):
171    args = []
172    for arg in self.args:
173      args.append(mapping[arg])
174    return str(self.fun(*args))
175
176CONST_PATTERN = re.compile(r'^const\s+([a-zA-Z0-9_]+)\s*=\s*([^;]*);$')
177MACRO_PATTERN = re.compile(r'^macro\s+([a-zA-Z0-9_]+)\s*\(([^)]*)\)\s*=\s*([^;]*);$')
178PYTHON_MACRO_PATTERN = re.compile(r'^python\s+macro\s+([a-zA-Z0-9_]+)\s*\(([^)]*)\)\s*=\s*([^;]*);$')
179
180
181def ReadMacros(lines):
182  constants = []
183  macros = []
184  for line in lines:
185    hash = line.find('#')
186    if hash != -1: line = line[:hash]
187    line = line.strip()
188    if len(line) is 0: continue
189    const_match = CONST_PATTERN.match(line)
190    if const_match:
191      name = const_match.group(1)
192      value = const_match.group(2).strip()
193      constants.append((re.compile("\\b%s\\b" % name), value))
194    else:
195      macro_match = MACRO_PATTERN.match(line)
196      if macro_match:
197        name = macro_match.group(1)
198        args = map(string.strip, macro_match.group(2).split(','))
199        body = macro_match.group(3).strip()
200        macros.append((re.compile("\\b%s\\(" % name), TextMacro(args, body)))
201      else:
202        python_match = PYTHON_MACRO_PATTERN.match(line)
203        if python_match:
204          name = python_match.group(1)
205          args = map(string.strip, python_match.group(2).split(','))
206          body = python_match.group(3).strip()
207          fun = eval("lambda " + ",".join(args) + ': ' + body)
208          macros.append((re.compile("\\b%s\\(" % name), PythonMacro(args, fun)))
209        else:
210          raise ("Illegal line: " + line)
211  return (constants, macros)
212
213
214HEADER_TEMPLATE = """\
215// Copyright 2011 Google Inc. All Rights Reserved.
216
217// This file was generated from .js source files by SCons.  If you
218// want to make changes to this file you should either change the
219// javascript source files or the SConstruct script.
220
221#include "v8.h"
222#include "natives.h"
223#include "utils.h"
224
225namespace v8 {
226namespace internal {
227
228  static const byte sources[] = { %(sources_data)s };
229
230%(raw_sources_declaration)s\
231
232  template <>
233  int NativesCollection<%(type)s>::GetBuiltinsCount() {
234    return %(builtin_count)i;
235  }
236
237  template <>
238  int NativesCollection<%(type)s>::GetDebuggerCount() {
239    return %(debugger_count)i;
240  }
241
242  template <>
243  int NativesCollection<%(type)s>::GetIndex(const char* name) {
244%(get_index_cases)s\
245    return -1;
246  }
247
248  template <>
249  int NativesCollection<%(type)s>::GetRawScriptsSize() {
250    return %(raw_total_length)i;
251  }
252
253  template <>
254  Vector<const char> NativesCollection<%(type)s>::GetRawScriptSource(int index) {
255%(get_raw_script_source_cases)s\
256    return Vector<const char>("", 0);
257  }
258
259  template <>
260  Vector<const char> NativesCollection<%(type)s>::GetScriptName(int index) {
261%(get_script_name_cases)s\
262    return Vector<const char>("", 0);
263  }
264
265  template <>
266  Vector<const byte> NativesCollection<%(type)s>::GetScriptsSource() {
267    return Vector<const byte>(sources, %(total_length)i);
268  }
269
270  template <>
271  void NativesCollection<%(type)s>::SetRawScriptsSource(Vector<const char> raw_source) {
272    ASSERT(%(raw_total_length)i == raw_source.length());
273    raw_sources = raw_source.start();
274  }
275
276}  // internal
277}  // v8
278"""
279
280
281RAW_SOURCES_COMPRESSION_DECLARATION = """\
282  static const char* raw_sources = NULL;
283"""
284
285
286RAW_SOURCES_DECLARATION = """\
287  static const char* raw_sources = reinterpret_cast<const char*>(sources);
288"""
289
290
291GET_INDEX_CASE = """\
292    if (strcmp(name, "%(id)s") == 0) return %(i)i;
293"""
294
295
296GET_RAW_SCRIPT_SOURCE_CASE = """\
297    if (index == %(i)i) return Vector<const char>(raw_sources + %(offset)i, %(raw_length)i);
298"""
299
300
301GET_SCRIPT_NAME_CASE = """\
302    if (index == %(i)i) return Vector<const char>("%(name)s", %(length)i);
303"""
304
305def JS2C(source, target, env):
306  ids = []
307  debugger_ids = []
308  modules = []
309  # Locate the macros file name.
310  consts = []
311  macros = []
312  for s in source:
313    if 'macros.py' == (os.path.split(str(s))[1]):
314      (consts, macros) = ReadMacros(ReadLines(str(s)))
315    else:
316      modules.append(s)
317
318  minifier = jsmin.JavaScriptMinifier()
319
320  module_offset = 0
321  all_sources = []
322  for module in modules:
323    filename = str(module)
324    debugger = filename.endswith('-debugger.js')
325    lines = ReadFile(filename)
326    lines = ExpandConstants(lines, consts)
327    lines = ExpandMacros(lines, macros)
328    Validate(lines, filename)
329    lines = minifier.JSMinify(lines)
330    id = (os.path.split(filename)[1])[:-3]
331    if debugger: id = id[:-9]
332    raw_length = len(lines)
333    if debugger:
334      debugger_ids.append((id, raw_length, module_offset))
335    else:
336      ids.append((id, raw_length, module_offset))
337    all_sources.append(lines)
338    module_offset += raw_length
339  total_length = raw_total_length = module_offset
340
341  if env['COMPRESSION'] == 'off':
342    raw_sources_declaration = RAW_SOURCES_DECLARATION
343    sources_data = ToCAsciiArray("".join(all_sources))
344  else:
345    raw_sources_declaration = RAW_SOURCES_COMPRESSION_DECLARATION
346    if env['COMPRESSION'] == 'bz2':
347      all_sources = bz2.compress("".join(all_sources))
348    total_length = len(all_sources)
349    sources_data = ToCArray(all_sources)
350
351  # Build debugger support functions
352  get_index_cases = [ ]
353  get_raw_script_source_cases = [ ]
354  get_script_name_cases = [ ]
355
356  i = 0
357  for (id, raw_length, module_offset) in debugger_ids + ids:
358    native_name = "native %s.js" % id
359    get_index_cases.append(GET_INDEX_CASE % { 'id': id, 'i': i })
360    get_raw_script_source_cases.append(GET_RAW_SCRIPT_SOURCE_CASE % {
361        'offset': module_offset,
362        'raw_length': raw_length,
363        'i': i
364        })
365    get_script_name_cases.append(GET_SCRIPT_NAME_CASE % {
366        'name': native_name,
367        'length': len(native_name),
368        'i': i
369        })
370    i = i + 1
371
372  # Emit result
373  output = open(str(target[0]), "w")
374  output.write(HEADER_TEMPLATE % {
375    'builtin_count': len(ids) + len(debugger_ids),
376    'debugger_count': len(debugger_ids),
377    'sources_data': sources_data,
378    'raw_sources_declaration': raw_sources_declaration,
379    'raw_total_length': raw_total_length,
380    'total_length': total_length,
381    'get_index_cases': "".join(get_index_cases),
382    'get_raw_script_source_cases': "".join(get_raw_script_source_cases),
383    'get_script_name_cases': "".join(get_script_name_cases),
384    'type': env['TYPE']
385  })
386  output.close()
387
388def main():
389  natives = sys.argv[1]
390  type = sys.argv[2]
391  compression = sys.argv[3]
392  source_files = sys.argv[4:]
393  JS2C(source_files, [natives], { 'TYPE': type, 'COMPRESSION': compression })
394
395if __name__ == "__main__":
396  main()