PageRenderTime 53ms CodeModel.GetById 16ms RepoModel.GetById 0ms app.codeStats 0ms

/Languages/IronPython/IronPython/Runtime/FunctionCode.cs

http://github.com/IronLanguages/main
C# | 1247 lines | 843 code | 180 blank | 224 comment | 138 complexity | a3bf9bde1474d1e6a7401b7e8eeb8a9e MD5 | raw file
Possible License(s): CPL-1.0, BSD-3-Clause, ISC, GPL-2.0, MPL-2.0-no-copyleft-exception

Large files files are truncated, but you can click here to view the full file

  1. /* ****************************************************************************
  2. *
  3. * Copyright (c) Microsoft Corporation.
  4. *
  5. * This source code is subject to terms and conditions of the Apache License, Version 2.0. A
  6. * copy of the license can be found in the License.html file at the root of this distribution. If
  7. * you cannot locate the Apache License, Version 2.0, please send an email to
  8. * dlr@microsoft.com. By using this source code in any fashion, you are agreeing to be bound
  9. * by the terms of the Apache License, Version 2.0.
  10. *
  11. * You must not remove this notice, or any other, from this software.
  12. *
  13. *
  14. * ***************************************************************************/
  15. #if FEATURE_CORE_DLR
  16. using System.Linq.Expressions;
  17. using Microsoft.Scripting.Ast;
  18. #else
  19. using Microsoft.Scripting.Ast;
  20. #endif
  21. using System;
  22. using System.Collections.Generic;
  23. using System.Diagnostics;
  24. using System.Reflection;
  25. using System.Runtime.CompilerServices;
  26. using System.Threading;
  27. using Microsoft.Scripting;
  28. using Microsoft.Scripting.Debugging.CompilerServices;
  29. using Microsoft.Scripting.Generation;
  30. using Microsoft.Scripting.Interpreter;
  31. using Microsoft.Scripting.Runtime;
  32. using Microsoft.Scripting.Utils;
  33. using IronPython.Compiler;
  34. using IronPython.Runtime.Operations;
  35. using IronPython.Runtime.Types;
  36. namespace IronPython.Runtime {
  37. /// <summary>
  38. /// Represents a piece of code. This can reference either a CompiledCode
  39. /// object or a Function. The user can explicitly call FunctionCode by
  40. /// passing it into exec or eval.
  41. /// </summary>
  42. [PythonType("code")]
  43. [DebuggerDisplay("{co_name}, FileName = {co_filename}")]
  44. public class FunctionCode : IExpressionSerializable, ICodeFormattable {
  45. [PythonHidden]
  46. internal Delegate Target, LightThrowTarget; // the current target for the function. This can change based upon adaptive compilation, recursion enforcement, and tracing.
  47. internal Delegate _normalDelegate; // the normal delegate - this can be a compiled or interpreted delegate.
  48. private Compiler.Ast.ScopeStatement _lambda; // the original DLR lambda that contains the code
  49. internal readonly string _initialDoc; // the initial doc string
  50. private readonly int _localCount; // the number of local variables in the code
  51. private readonly int _argCount; // cached locally because it's used during calls w/ defaults
  52. private bool _compilingLight; // true if we're compiling for light exceptions
  53. private int _exceptionCount;
  54. // debugging/tracing support
  55. private LambdaExpression _tracingLambda; // the transformed lambda used for tracing/debugging
  56. internal Delegate _tracingDelegate; // the delegate used for tracing/debugging, if one has been created. This can be interpreted or compiled.
  57. /// <summary>
  58. /// This is both the lock that is held while enumerating the threads or updating the thread accounting
  59. /// information. It's also a marker CodeList which is put in place when we are enumerating the thread
  60. /// list and all additions need to block.
  61. ///
  62. /// This lock is also acquired whenever we need to calculate how a function's delegate should be created
  63. /// so that we don't race against sys.settrace/sys.setprofile.
  64. /// </summary>
  65. private static CodeList _CodeCreateAndUpdateDelegateLock = new CodeList();
  66. /// <summary>
  67. /// Constructor used to create a FunctionCode for code that's been serialized to disk.
  68. ///
  69. /// Code constructed this way cannot be interpreted or debugged using sys.settrace/sys.setprofile.
  70. ///
  71. /// Function codes created this way do support recursion enforcement and are therefore registered in the global function code registry.
  72. /// </summary>
  73. internal FunctionCode(PythonContext context, Delegate code, Compiler.Ast.ScopeStatement scope, string documentation, int localCount) {
  74. _normalDelegate = code;
  75. _lambda = scope;
  76. _argCount = CalculateArgumentCount();
  77. _initialDoc = documentation;
  78. // need to take this lock to ensure sys.settrace/sys.setprofile is not actively changing
  79. lock (_CodeCreateAndUpdateDelegateLock) {
  80. SetTarget(AddRecursionCheck(context, code));
  81. }
  82. RegisterFunctionCode(context);
  83. }
  84. /// <summary>
  85. /// Constructor to create a FunctionCode at runtime.
  86. ///
  87. /// Code constructed this way supports both being interpreted and debugged. When necessary the code will
  88. /// be re-compiled or re-interpreted for that specific purpose.
  89. ///
  90. /// Function codes created this way do support recursion enforcement and are therefore registered in the global function code registry.
  91. ///
  92. /// the initial delegate provided here should NOT be the actual code. It should always be a delegate which updates our Target lazily.
  93. /// </summary>
  94. internal FunctionCode(PythonContext context, Delegate initialDelegate, Compiler.Ast.ScopeStatement scope, string documentation, bool? tracing, bool register) {
  95. _lambda = scope;
  96. Target = LightThrowTarget = initialDelegate;
  97. _initialDoc = documentation;
  98. _localCount = scope.Variables == null ? 0 : scope.Variables.Count;
  99. _argCount = CalculateArgumentCount();
  100. if (tracing.HasValue) {
  101. if (tracing.Value) {
  102. _tracingDelegate = initialDelegate;
  103. } else {
  104. _normalDelegate = initialDelegate;
  105. }
  106. }
  107. if (register) {
  108. RegisterFunctionCode(context);
  109. }
  110. }
  111. private static PythonTuple SymbolListToTuple(IList<string> vars) {
  112. if (vars != null && vars.Count != 0) {
  113. object[] tupleData = new object[vars.Count];
  114. for (int i = 0; i < vars.Count; i++) {
  115. tupleData[i] = vars[i];
  116. }
  117. return PythonTuple.MakeTuple(tupleData);
  118. } else {
  119. return PythonTuple.EMPTY;
  120. }
  121. }
  122. private static PythonTuple StringArrayToTuple(string[] closureVars) {
  123. if (closureVars != null && closureVars.Length != 0) {
  124. return PythonTuple.MakeTuple((object[])closureVars);
  125. } else {
  126. return PythonTuple.EMPTY;
  127. }
  128. }
  129. /// <summary>
  130. /// Registers the current function code in our global weak list of all function codes.
  131. ///
  132. /// The weak list can be enumerated with GetAllCode().
  133. ///
  134. /// Ultimately there are 3 types of threads we care about races with:
  135. /// 1. Other threads which are registering function codes
  136. /// 2. Threads calling sys.settrace which require the world to stop and get updated
  137. /// 3. Threads running cleanup (thread pool thread, or call to gc.collect).
  138. ///
  139. /// The 1st two must have perfect synchronization. We cannot have a thread registering
  140. /// a new function which another thread is trying to update all of the functions in the world. Doing
  141. /// so would mean we could miss adding tracing to a thread.
  142. ///
  143. /// But the cleanup thread can run in parallel to either registrying or sys.settrace. The only
  144. /// thing it needs to take a lock for is updating our accounting information about the
  145. /// number of code objects are alive.
  146. /// </summary>
  147. private void RegisterFunctionCode(PythonContext context) {
  148. if (_lambda == null) {
  149. return;
  150. }
  151. WeakReference codeRef = new WeakReference(this);
  152. CodeList prevCode;
  153. lock (_CodeCreateAndUpdateDelegateLock) {
  154. Debug.Assert(context._allCodes != _CodeCreateAndUpdateDelegateLock);
  155. // we do an interlocked operation here because this can run in parallel w/ the CodeCleanup thread. The
  156. // lock only prevents us from running in parallel w/ an update to all of the functions in the world which
  157. // needs to be synchronized.
  158. do {
  159. prevCode = context._allCodes;
  160. } while (Interlocked.CompareExchange(ref context._allCodes, new CodeList(codeRef, prevCode), prevCode) != prevCode);
  161. if (context._codeCount++ == context._nextCodeCleanup) {
  162. // run cleanup of codes on another thread
  163. CleanFunctionCodes(context, false);
  164. }
  165. }
  166. }
  167. internal static void CleanFunctionCodes(PythonContext context, bool synchronous) {
  168. if (synchronous) {
  169. CodeCleanup(context);
  170. } else {
  171. ThreadPool.QueueUserWorkItem(CodeCleanup, context);
  172. }
  173. }
  174. internal void SetTarget(Delegate target) {
  175. Target = LightThrowTarget = target;
  176. }
  177. internal void LightThrowCompile(CodeContext/*!*/ context) {
  178. if (++_exceptionCount > 20) {
  179. if (!_compilingLight && (object)Target == (object)LightThrowTarget) {
  180. _compilingLight = true;
  181. if (!IsOnDiskCode) {
  182. ThreadPool.QueueUserWorkItem(x => {
  183. // on mono if the runtime is shutting down, this can throw an InvalidOperationException
  184. #if FEATURE_UNIX
  185. try {
  186. #endif
  187. var pyCtx = context.LanguageContext;
  188. bool enableTracing;
  189. lock (pyCtx._codeUpdateLock) {
  190. enableTracing = context.LanguageContext.EnableTracing;
  191. }
  192. Delegate target;
  193. if (enableTracing) {
  194. target = ((LambdaExpression)LightExceptions.Rewrite(GetGeneratorOrNormalLambdaTracing(pyCtx).Reduce())).Compile();
  195. } else {
  196. target = ((LambdaExpression)LightExceptions.Rewrite(GetGeneratorOrNormalLambda().Reduce())).Compile();
  197. }
  198. lock (pyCtx._codeUpdateLock) {
  199. if (context.LanguageContext.EnableTracing == enableTracing) {
  200. LightThrowTarget = target;
  201. }
  202. }
  203. #if FEATURE_UNIX
  204. } catch(InvalidOperationException ex) {
  205. // The InvalidOperationException is thrown with an empty message
  206. if(!string.IsNullOrEmpty(ex.Message)) {
  207. throw ex;
  208. }
  209. }
  210. #endif
  211. });
  212. }
  213. }
  214. }
  215. }
  216. private bool IsOnDiskCode {
  217. get {
  218. if (_lambda is Compiler.Ast.SerializedScopeStatement) {
  219. return true;
  220. } else if (_lambda is Compiler.Ast.PythonAst) {
  221. return ((Compiler.Ast.PythonAst)_lambda).OnDiskProxy;
  222. }
  223. return false;
  224. }
  225. }
  226. /// <summary>
  227. /// Enumerates all function codes for updating the current type of targets we generate.
  228. ///
  229. /// While enumerating we hold a lock so that users cannot change sys.settrace/sys.setprofile
  230. /// until the lock is released.
  231. /// </summary>
  232. private static IEnumerable<FunctionCode> GetAllCode(PythonContext context) {
  233. // only a single thread can enumerate the current FunctionCodes at a time.
  234. lock (_CodeCreateAndUpdateDelegateLock) {
  235. CodeList curCodeList = Interlocked.Exchange(ref context._allCodes, _CodeCreateAndUpdateDelegateLock);
  236. Debug.Assert(curCodeList != _CodeCreateAndUpdateDelegateLock);
  237. CodeList initialCode = curCodeList;
  238. try {
  239. while (curCodeList != null) {
  240. WeakReference codeRef = curCodeList.Code;
  241. FunctionCode target = (FunctionCode)codeRef.Target;
  242. if (target != null) {
  243. yield return target;
  244. }
  245. curCodeList = curCodeList.Next;
  246. }
  247. } finally {
  248. Interlocked.Exchange(ref context._allCodes, initialCode);
  249. }
  250. }
  251. }
  252. internal static void UpdateAllCode(PythonContext context) {
  253. foreach (FunctionCode fc in GetAllCode(context)) {
  254. fc.UpdateDelegate(context, false);
  255. }
  256. }
  257. private static void CodeCleanup(object state) {
  258. PythonContext context = (PythonContext)state;
  259. // only allow 1 thread at a time to do cleanup (other threads can continue adding)
  260. lock (context._codeCleanupLock) {
  261. // the bulk of the work is in scanning the list, this proceeeds lock free
  262. int removed = 0, kept = 0;
  263. CodeList prev = null;
  264. CodeList cur = GetRootCodeNoUpdating(context);
  265. while (cur != null) {
  266. if (!cur.Code.IsAlive) {
  267. if (prev == null) {
  268. if (Interlocked.CompareExchange(ref context._allCodes, cur.Next, cur) != cur) {
  269. // someone else snuck in and added a new code entry, spin and try again.
  270. cur = GetRootCodeNoUpdating(context);
  271. continue;
  272. }
  273. cur = cur.Next;
  274. removed++;
  275. continue;
  276. } else {
  277. // remove from the linked list, we're the only one who can change this.
  278. Debug.Assert(prev.Next == cur);
  279. removed++;
  280. cur = prev.Next = cur.Next;
  281. continue;
  282. }
  283. } else {
  284. kept++;
  285. }
  286. prev = cur;
  287. cur = cur.Next;
  288. }
  289. // finally update our bookkeeping statistics which requires locking but is fast.
  290. lock (_CodeCreateAndUpdateDelegateLock) {
  291. // calculate the next cleanup, we want each pass to remove ~50% of all entries
  292. const double removalGoal = .50;
  293. if (context._codeCount == 0) {
  294. // somehow we would have had to queue a bunch of function codes, have 1 thread
  295. // clean them up all up, and a 2nd queued thread waiting to clean them up as well.
  296. // At the same time there would have to be no live functions defined which means
  297. // we're executing top-level code which is causing this to happen.
  298. context._nextCodeCleanup = 200;
  299. return;
  300. }
  301. //Console.WriteLine("Removed {0} entries, {1} remain", removed, context._codeCount);
  302. Debug.Assert(removed <= context._codeCount);
  303. double pctRemoved = (double)removed / (double)context._codeCount; // % of code removed
  304. double targetRatio = pctRemoved / removalGoal; // how we need to adjust our last goal
  305. // update the total and next node cleanup
  306. int newCount = Interlocked.Add(ref context._codeCount, -removed);
  307. Debug.Assert(newCount >= 0);
  308. // set the new target for cleanup
  309. int nextCleanup = targetRatio != 0 ? newCount + (int)(context._nextCodeCleanup / targetRatio) : -1;
  310. if (nextCleanup > 0) {
  311. // we're making good progress, use the newly calculated next cleanup point
  312. context._nextCodeCleanup = nextCleanup;
  313. } else {
  314. // none or very small amount cleaned up, just schedule a cleanup for sometime in the future.
  315. context._nextCodeCleanup += 500;
  316. }
  317. Debug.Assert(context._nextCodeCleanup >= context._codeCount, String.Format("{0} vs {1} ({2})", context._nextCodeCleanup, context._codeCount, targetRatio));
  318. }
  319. }
  320. }
  321. private static CodeList GetRootCodeNoUpdating(PythonContext context) {
  322. CodeList cur = context._allCodes;
  323. if (cur == _CodeCreateAndUpdateDelegateLock) {
  324. lock (_CodeCreateAndUpdateDelegateLock) {
  325. // wait until enumerating thread is done, but it's alright
  326. // if we got cur and then an enumeration started (because we'll
  327. // just clear entries out)
  328. cur = context._allCodes;
  329. Debug.Assert(cur != _CodeCreateAndUpdateDelegateLock);
  330. }
  331. }
  332. return cur;
  333. }
  334. public SourceSpan Span {
  335. [PythonHidden]
  336. get {
  337. return _lambda.Span;
  338. }
  339. }
  340. internal string[] ArgNames {
  341. get {
  342. return _lambda.ParameterNames;
  343. }
  344. }
  345. internal FunctionAttributes Flags {
  346. get {
  347. return _lambda.Flags;
  348. }
  349. }
  350. internal bool IsModule {
  351. get {
  352. return _lambda is Compiler.Ast.PythonAst;
  353. }
  354. }
  355. #region Public constructors
  356. /*
  357. /// <summary>
  358. /// Standard python siganture
  359. /// </summary>
  360. /// <param name="argcount"></param>
  361. /// <param name="nlocals"></param>
  362. /// <param name="stacksize"></param>
  363. /// <param name="flags"></param>
  364. /// <param name="codestring"></param>
  365. /// <param name="constants"></param>
  366. /// <param name="names"></param>
  367. /// <param name="varnames"></param>
  368. /// <param name="filename"></param>
  369. /// <param name="name"></param>
  370. /// <param name="firstlineno"></param>
  371. /// <param name="nlotab"></param>
  372. /// <param name="freevars"></param>
  373. /// <param name="callvars"></param>
  374. public FunctionCode(int argcount, int nlocals, int stacksize, int flags, string codestring, object constants, Tuple names, Tuple varnames, string filename, string name, int firstlineno, object nlotab, [DefaultParameterValue(null)]object freevars, [DefaultParameterValue(null)]object callvars) {
  375. }*/
  376. #endregion
  377. #region Public Python API Surface
  378. public PythonTuple co_varnames {
  379. get {
  380. return SymbolListToTuple(_lambda.GetVarNames());
  381. }
  382. }
  383. public int co_argcount {
  384. get {
  385. return _argCount;
  386. }
  387. }
  388. private int CalculateArgumentCount() {
  389. int argCnt = _lambda.ArgCount;
  390. FunctionAttributes flags = Flags;
  391. if ((flags & FunctionAttributes.ArgumentList) != 0) argCnt--;
  392. if ((flags & FunctionAttributes.KeywordDictionary) != 0) argCnt--;
  393. return argCnt;
  394. }
  395. /// <summary>
  396. /// Returns a list of variable names which are accessed from nested functions.
  397. /// </summary>
  398. public PythonTuple co_cellvars {
  399. get {
  400. return SymbolListToTuple(_lambda.CellVariables != null ? ArrayUtils.ToArray(_lambda.CellVariables) : null);
  401. }
  402. }
  403. /// <summary>
  404. /// Returns the byte code. IronPython does not implement this and always
  405. /// returns an empty string for byte code.
  406. /// </summary>
  407. [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic")]
  408. public object co_code {
  409. get {
  410. return String.Empty;
  411. }
  412. }
  413. /// <summary>
  414. /// Returns a list of constants used by the function.
  415. ///
  416. /// The first constant is the doc string, or None if no doc string is provided.
  417. ///
  418. /// IronPython currently does not include any other constants than the doc string.
  419. /// </summary>
  420. public PythonTuple co_consts {
  421. get {
  422. if (_initialDoc != null) {
  423. return PythonTuple.MakeTuple(_initialDoc, null);
  424. }
  425. return PythonTuple.MakeTuple((object)null);
  426. }
  427. }
  428. /// <summary>
  429. /// Returns the filename that the code object was defined in.
  430. /// </summary>
  431. public string co_filename {
  432. get {
  433. return _lambda.Filename;
  434. }
  435. }
  436. /// <summary>
  437. /// Returns the 1st line number of the code object.
  438. /// </summary>
  439. public int co_firstlineno {
  440. get {
  441. return Span.Start.Line;
  442. }
  443. }
  444. /// <summary>
  445. /// Returns a set of flags for the function.
  446. ///
  447. /// 0x04 is set if the function used *args
  448. /// 0x08 is set if the function used **args
  449. /// 0x20 is set if the function is a generator
  450. /// </summary>
  451. public int co_flags {
  452. get {
  453. return (int)Flags;
  454. }
  455. }
  456. /// <summary>
  457. /// Returns a list of free variables (variables accessed
  458. /// from an outer scope). This does not include variables
  459. /// accessed in the global scope.
  460. /// </summary>
  461. public PythonTuple co_freevars {
  462. get {
  463. return SymbolListToTuple(_lambda.FreeVariables != null ? CollectionUtils.ConvertAll(_lambda.FreeVariables, x => x.Name) : null);
  464. }
  465. }
  466. /// <summary>
  467. /// Returns a mapping between byte code and line numbers. IronPython does
  468. /// not implement this because byte code is not available.
  469. /// </summary>
  470. [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic")]
  471. public object co_lnotab {
  472. get {
  473. throw PythonOps.NotImplementedError("");
  474. }
  475. }
  476. /// <summary>
  477. /// Returns the name of the code (function name, class name, or &lt;module&gt;).
  478. /// </summary>
  479. public string co_name {
  480. get {
  481. return _lambda.Name;
  482. }
  483. }
  484. /// <summary>
  485. /// Returns a list of global variable names accessed by the code.
  486. /// </summary>
  487. public PythonTuple co_names {
  488. get {
  489. return SymbolListToTuple(_lambda.GlobalVariables);
  490. }
  491. }
  492. /// <summary>
  493. /// Returns the number of local varaibles defined in the function.
  494. /// </summary>
  495. public object co_nlocals {
  496. get {
  497. return _localCount;
  498. }
  499. }
  500. /// <summary>
  501. /// Returns the stack size. IronPython does not implement this
  502. /// because byte code is not supported.
  503. /// </summary>
  504. [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic")]
  505. public object co_stacksize {
  506. get {
  507. throw PythonOps.NotImplementedError("");
  508. }
  509. }
  510. #endregion
  511. #region ICodeFormattable Members
  512. public string/*!*/ __repr__(CodeContext/*!*/ context) {
  513. return string.Format(
  514. "<code object {0} at {1}, file {2}, line {3}>",
  515. co_name,
  516. PythonOps.HexId(this),
  517. !string.IsNullOrEmpty(co_filename) ? string.Format("\"{0}\"", co_filename) : "???",
  518. co_firstlineno != 0 ? co_firstlineno : -1);
  519. }
  520. #endregion
  521. #region Internal API Surface
  522. internal LightLambdaExpression Code {
  523. get {
  524. return _lambda.GetLambda();
  525. }
  526. }
  527. internal Compiler.Ast.ScopeStatement PythonCode {
  528. get {
  529. return (Compiler.Ast.ScopeStatement)_lambda;
  530. }
  531. }
  532. internal object Call(CodeContext/*!*/ context) {
  533. if (co_freevars != PythonTuple.EMPTY) {
  534. throw PythonOps.TypeError("cannot exec code object that contains free variables: {0}", co_freevars.__repr__(context));
  535. }
  536. if (Target == null || (Target.GetMethod() != null && Target.GetMethod().DeclaringType == typeof(PythonCallTargets))) {
  537. UpdateDelegate(context.LanguageContext, true);
  538. }
  539. Func<CodeContext, CodeContext> classTarget = Target as Func<CodeContext, CodeContext>;
  540. if (classTarget != null) {
  541. return classTarget(context);
  542. }
  543. LookupCompilationDelegate moduleCode = Target as LookupCompilationDelegate;
  544. if (moduleCode != null) {
  545. return moduleCode(context, this);
  546. }
  547. Func<FunctionCode, object> optimizedModuleCode = Target as Func<FunctionCode, object>;
  548. if (optimizedModuleCode != null) {
  549. return optimizedModuleCode(this);
  550. }
  551. var func = new PythonFunction(context, this, null, ArrayUtils.EmptyObjects, new MutableTuple<object>());
  552. CallSite<Func<CallSite, CodeContext, PythonFunction, object>> site = PythonContext.GetContext(context).FunctionCallSite;
  553. return site.Target(site, context, func);
  554. }
  555. /// <summary>
  556. /// Creates a FunctionCode object for exec/eval/execfile'd/compile'd code.
  557. ///
  558. /// The code is then executed in a specific CodeContext by calling the .Call method.
  559. ///
  560. /// If the code is being used for compile (vs. exec/eval/execfile) then it needs to be
  561. /// registered in case our tracing mode changes.
  562. /// </summary>
  563. internal static FunctionCode FromSourceUnit(SourceUnit sourceUnit, PythonCompilerOptions options, bool register) {
  564. var code = PythonContext.CompilePythonCode(sourceUnit, options, ThrowingErrorSink.Default);
  565. if (sourceUnit.CodeProperties == ScriptCodeParseResult.Empty) {
  566. // source span is made up
  567. throw new SyntaxErrorException("unexpected EOF while parsing",
  568. sourceUnit, new SourceSpan(new SourceLocation(0, 1, 1), new SourceLocation(0, 1, 1)), 0, Severity.Error);
  569. }
  570. return ((RunnableScriptCode)code).GetFunctionCode(register);
  571. }
  572. #endregion
  573. #region Private helper functions
  574. private void ExpandArgsTuple(List<string> names, PythonTuple toExpand) {
  575. for (int i = 0; i < toExpand.__len__(); i++) {
  576. if (toExpand[i] is PythonTuple) {
  577. ExpandArgsTuple(names, toExpand[i] as PythonTuple);
  578. } else {
  579. names.Add(toExpand[i] as string);
  580. }
  581. }
  582. }
  583. #endregion
  584. public override bool Equals(object obj) {
  585. // overridden because CPython defines this on code objects
  586. return base.Equals(obj);
  587. }
  588. public override int GetHashCode() {
  589. // overridden because CPython defines this on code objects
  590. return base.GetHashCode();
  591. }
  592. public int __cmp__(CodeContext/*!*/ context, [NotNull]FunctionCode/*!*/ other) {
  593. if (other == this) {
  594. return 0;
  595. }
  596. long lres = IdDispenser.GetId(this) - IdDispenser.GetId(other);
  597. return lres > 0 ? 1 : -1;
  598. }
  599. // these are present in CPython but always return NotImplemented.
  600. [return: MaybeNotImplemented]
  601. [Python3Warning("code inequality comparisons not supported in 3.x")]
  602. public static NotImplementedType operator >(FunctionCode self, FunctionCode other) {
  603. return PythonOps.NotImplemented;
  604. }
  605. [return: MaybeNotImplemented]
  606. [Python3Warning("code inequality comparisons not supported in 3.x")]
  607. public static NotImplementedType operator <(FunctionCode self, FunctionCode other) {
  608. return PythonOps.NotImplemented;
  609. }
  610. [return: MaybeNotImplemented]
  611. [Python3Warning("code inequality comparisons not supported in 3.x")]
  612. public static NotImplementedType operator >=(FunctionCode self, FunctionCode other) {
  613. return PythonOps.NotImplemented;
  614. }
  615. [return: MaybeNotImplemented]
  616. [Python3Warning("code inequality comparisons not supported in 3.x")]
  617. public static NotImplementedType operator <=(FunctionCode self, FunctionCode other) {
  618. return PythonOps.NotImplemented;
  619. }
  620. /// <summary>
  621. /// Called the 1st time a function is invoked by our OriginalCallTarget* methods
  622. /// over in PythonCallTargets. This computes the real delegate which needs to be
  623. /// created for the function. Usually this means starting off interpretering. It
  624. /// also involves adding the wrapper function for recursion enforcement.
  625. ///
  626. /// Because this can race against sys.settrace/setprofile we need to take our
  627. /// _ThreadIsEnumeratingAndAccountingLock to ensure no one is actively changing all
  628. /// of the live functions.
  629. /// </summary>
  630. internal void LazyCompileFirstTarget(PythonFunction function) {
  631. lock (_CodeCreateAndUpdateDelegateLock) {
  632. UpdateDelegate(PythonContext.GetContext(function.Context), true);
  633. }
  634. }
  635. /// <summary>
  636. /// Updates the delegate based upon current Python context settings for recursion enforcement
  637. /// and for tracing.
  638. /// </summary>
  639. internal void UpdateDelegate(PythonContext context, bool forceCreation) {
  640. Delegate finalTarget;
  641. if (context.EnableTracing && _lambda != null) {
  642. if (_tracingLambda == null) {
  643. if (!forceCreation) {
  644. // the user just called sys.settrace(), don't force re-compilation of every method in the system. Instead
  645. // we'll just re-compile them as they're invoked.
  646. PythonCallTargets.GetPythonTargetType(_lambda.ParameterNames.Length > PythonCallTargets.MaxArgs, _lambda.ParameterNames.Length, out Target);
  647. LightThrowTarget = Target;
  648. return;
  649. }
  650. _tracingLambda = GetGeneratorOrNormalLambdaTracing(context);
  651. }
  652. if (_tracingDelegate == null) {
  653. _tracingDelegate = CompileLambda(_tracingLambda, new TargetUpdaterForCompilation(context, this).SetCompiledTargetTracing);
  654. }
  655. finalTarget = _tracingDelegate;
  656. } else {
  657. if (_normalDelegate == null) {
  658. if (!forceCreation) {
  659. // we cannot create the delegate when forceCreation is false because we hold the
  660. // _CodeCreateAndUpdateDelegateLock and compiling the delegate may create a FunctionCode
  661. // object which requires the lock.
  662. PythonCallTargets.GetPythonTargetType(_lambda.ParameterNames.Length > PythonCallTargets.MaxArgs, _lambda.ParameterNames.Length, out Target);
  663. LightThrowTarget = Target;
  664. return;
  665. }
  666. _normalDelegate = CompileLambda(GetGeneratorOrNormalLambda(), new TargetUpdaterForCompilation(context, this).SetCompiledTarget);
  667. }
  668. finalTarget = _normalDelegate;
  669. }
  670. finalTarget = AddRecursionCheck(context, finalTarget);
  671. SetTarget(finalTarget);
  672. }
  673. /// <summary>
  674. /// Called to set the initial target delegate when the user has passed -X:Debug to enable
  675. /// .NET style debugging.
  676. /// </summary>
  677. internal void SetDebugTarget(PythonContext context, Delegate target) {
  678. _normalDelegate = target;
  679. SetTarget(AddRecursionCheck(context, target));
  680. }
  681. /// <summary>
  682. /// Gets the LambdaExpression for tracing.
  683. ///
  684. /// If this is a generator function code then the lambda gets tranformed into the correct generator code.
  685. /// </summary>
  686. private LambdaExpression GetGeneratorOrNormalLambdaTracing(PythonContext context) {
  687. var debugProperties = new PythonDebuggingPayload(this);
  688. var debugInfo = new Microsoft.Scripting.Debugging.CompilerServices.DebugLambdaInfo(
  689. null, // IDebugCompilerSupport
  690. null, // lambda alias
  691. false, // optimize for leaf frames
  692. null, // hidden variables
  693. null, // variable aliases
  694. debugProperties // custom payload
  695. );
  696. if ((Flags & FunctionAttributes.Generator) == 0) {
  697. return context.DebugContext.TransformLambda((LambdaExpression)Compiler.Ast.Node.RemoveFrame(_lambda.GetLambda()), debugInfo);
  698. }
  699. return Expression.Lambda(
  700. Code.Type,
  701. new GeneratorRewriter(
  702. _lambda.Name,
  703. Compiler.Ast.Node.RemoveFrame(Code.Body)
  704. ).Reduce(
  705. _lambda.ShouldInterpret,
  706. _lambda.EmitDebugSymbols,
  707. context.Options.CompilationThreshold,
  708. Code.Parameters,
  709. x => (Expression<Func<MutableTuple, object>>)context.DebugContext.TransformLambda(x, debugInfo)
  710. ),
  711. Code.Name,
  712. Code.Parameters
  713. );
  714. }
  715. /// <summary>
  716. /// Gets the correct final LambdaExpression for this piece of code.
  717. ///
  718. /// This is either just _lambda or _lambda re-written to be a generator expression.
  719. /// </summary>
  720. private LightLambdaExpression GetGeneratorOrNormalLambda() {
  721. LightLambdaExpression finalCode;
  722. if ((Flags & FunctionAttributes.Generator) == 0) {
  723. finalCode = Code;
  724. } else {
  725. finalCode = Code.ToGenerator(
  726. _lambda.ShouldInterpret,
  727. _lambda.EmitDebugSymbols,
  728. _lambda.GlobalParent.PyContext.Options.CompilationThreshold
  729. );
  730. }
  731. return finalCode;
  732. }
  733. private Delegate CompileLambda(LightLambdaExpression code, EventHandler<LightLambdaCompileEventArgs> handler) {
  734. #if EMIT_PDB
  735. if (_lambda.EmitDebugSymbols) {
  736. return CompilerHelpers.CompileToMethod((LambdaExpression)code.Reduce(), DebugInfoGenerator.CreatePdbGenerator(), true);
  737. }
  738. #endif
  739. if (_lambda.ShouldInterpret) {
  740. Delegate result = code.Compile(_lambda.GlobalParent.PyContext.Options.CompilationThreshold);
  741. // If the adaptive compiler decides to compile this function, we
  742. // want to store the new compiled target. This saves us from going
  743. // through the interpreter stub every call.
  744. var lightLambda = result.Target as LightLambda;
  745. if (lightLambda != null) {
  746. lightLambda.Compile += handler;
  747. }
  748. return result;
  749. }
  750. return code.Compile();
  751. }
  752. private Delegate CompileLambda(LambdaExpression code, EventHandler<LightLambdaCompileEventArgs> handler) {
  753. #if EMIT_PDB
  754. if (_lambda.EmitDebugSymbols) {
  755. return CompilerHelpers.CompileToMethod(code, DebugInfoGenerator.CreatePdbGenerator(), true);
  756. }
  757. #endif
  758. if (_lambda.ShouldInterpret) {
  759. Delegate result = CompilerHelpers.LightCompile(code, _lambda.GlobalParent.PyContext.Options.CompilationThreshold);
  760. // If the adaptive compiler decides to compile this function, we
  761. // want to store the new compiled target. This saves us from going
  762. // through the interpreter stub every call.
  763. var lightLambda = result.Target as LightLambda;
  764. if (lightLambda != null) {
  765. lightLambda.Compile += handler;
  766. }
  767. return result;
  768. }
  769. return code.Compile();
  770. }
  771. internal Delegate AddRecursionCheck(PythonContext context, Delegate finalTarget) {
  772. if (context.RecursionLimit != Int32.MaxValue) {
  773. if (finalTarget is Func<CodeContext, CodeContext> ||
  774. finalTarget is Func<FunctionCode, object> ||
  775. finalTarget is LookupCompilationDelegate) {
  776. // no recursion enforcement on classes or modules
  777. return finalTarget;
  778. }
  779. switch (_lambda.ParameterNames.Length) {
  780. #region Generated Python Recursion Delegate Switch
  781. // *** BEGIN GENERATED CODE ***
  782. // generated by function: gen_recursion_delegate_switch from: generate_calls.py
  783. case 0:
  784. finalTarget = new Func<PythonFunction, object>(new PythonFunctionRecursionCheck0((Func<PythonFunction, object>)finalTarget).CallTarget);
  785. break;
  786. case 1:
  787. finalTarget = new Func<PythonFunction, object, object>(new PythonFunctionRecursionCheck1((Func<PythonFunction, object, object>)finalTarget).CallTarget);
  788. break;
  789. case 2:
  790. finalTarget = new Func<PythonFunction, object, object, object>(new PythonFunctionRecursionCheck2((Func<PythonFunction, object, object, object>)finalTarget).CallTarget);
  791. break;
  792. case 3:
  793. finalTarget = new Func<PythonFunction, object, object, object, object>(new PythonFunctionRecursionCheck3((Func<PythonFunction, object, object, object, object>)finalTarget).CallTarget);
  794. break;
  795. case 4:
  796. finalTarget = new Func<PythonFunction, object, object, object, object, object>(new PythonFunctionRecursionCheck4((Func<PythonFunction, object, object, object, object, object>)finalTarget).CallTarget);
  797. break;
  798. case 5:
  799. finalTarget = new Func<PythonFunction, object, object, object, object, object, object>(new PythonFunctionRecursionCheck5((Func<PythonFunction, object, object, object, object, object, object>)finalTarget).CallTarget);
  800. break;
  801. case 6:
  802. finalTarget = new Func<PythonFunction, object, object, object, object, object, object, object>(new PythonFunctionRecursionCheck6((Func<PythonFunction, object, object, object, object, object, object, object>)finalTarget).CallTarget);
  803. break;
  804. case 7:
  805. finalTarget = new Func<PythonFunction, object, object, object, object, object, object, object, object>(new PythonFunctionRecursionCheck7((Func<PythonFunction, object, object, object, object, object, object, object, object>)finalTarget).CallTarget);
  806. break;
  807. case 8:
  808. finalTarget = new Func<PythonFunction, object, object, object, object, object, object, object, object, object>(new PythonFunctionRecursionCheck8((Func<PythonFunction, object, object, object, object, object, object, object, object, object>)finalTarget).CallTarget);
  809. break;
  810. case 9:
  811. finalTarget = new Func<PythonFunction, object, object, object, object, object, object, object, object, object, object>(new PythonFunctionRecursionCheck9((Func<PythonFunction, object, object, object, object, object, object, object, object, object, object>)finalTarget).CallTarget);
  812. break;
  813. case 10:
  814. finalTarget = new Func<PythonFunction, object, object, object, object, object, object, object, object, object, object, object>(new PythonFunctionRecursionCheck10((Func<PythonFunction, object, object, object, object, object, object, object, object, object, object, object>)finalTarget).CallTarget);
  815. break;
  816. case 11:
  817. finalTarget = new Func<PythonFunction, object, object, object, object, object, object, object, object, object, object, object, object>(new PythonFunctionRecursionCheck11((Func<PythonFunction, object, object, object, object, object, object, object, object, object, object, object, object>)finalTarget).CallTarget);
  818. break;
  819. case 12:
  820. finalTarget = new Func<PythonFunction, object, object, object, object, object, object, object, object, object, object, object, object, object>(new PythonFunctionRecursionCheck12((Func<PythonFunction, object, object, object, object, object, object, object, object, object, object, object, object, object>)finalTarget).CallTarget);
  821. break;
  822. case 13:
  823. finalTarget = new Func<PythonFunction, object, object, object, object, object, object, object, object, object, object, object, object, object, object>(new PythonFunctionRecursionCheck13((Func<PythonFunction, object, object, object, object, object, object, object, object, object, object, object, object, object, object>)finalTarget).CallTarget);
  824. break;
  825. case 14:
  826. finalTarget = new Func<PythonFunction, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object>(new PythonFunctionRecursionCheck14((Func<PythonFunction, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object>)finalTarget).CallTarget);
  827. break;
  828. case 15:
  829. finalTarget = new Func<PythonFunction, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object>(new PythonFunctionRecursionCheck15((Func<PythonFunction, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object, object>)finalTarget).CallTarget);
  830. break;
  831. // *** END GENERATED CODE ***
  832. #endregion
  833. default:
  834. finalTarget = new Func<PythonFunction, object[], object>(new PythonFunctionRecursionCheckN((Func<PythonFunction, object[], object>)finalTarget).CallTarget);
  835. break;
  836. }
  837. }
  838. return finalTarget;
  839. }
  840. class TargetUpdaterForCompilation {
  841. private readonly PythonContext _context;
  842. private readonly FunctionCode _code;
  843. public TargetUpdaterForCompilation(PythonContext context, FunctionCode code) {
  844. _code = code;
  845. _context = context;
  846. }
  847. public void SetCompiledTarget(object sender, Microsoft.Scripting.Interpreter.LightLambdaCompileEventArgs e) {
  848. _code.SetTarget(_code.AddRecursionCheck(_context, _code._normalDelegate = e.Compiled));
  849. }
  850. public void SetCompiledTargetTracing(object sender, Microsoft.Scripting.Interpreter.LightLambdaCompileEventArgs e) {
  851. _code.SetTarget(_code.AddRecursionCheck(_context, _code._tracingDelegate = e.Compiled));
  852. }
  853. }
  854. #region IExpressionSerializable Members
  855. Expression IExpressionSerializable.CreateExpression() {
  856. return Expression.Call(
  857. typeof(PythonOps).GetMethod("MakeFunctionCode"),
  858. Compiler.Ast.PythonAst._globalContext,
  859. Expression.Constant(_lambda.Name),
  860. Expression.Constant(_initialDoc, typeof(string)),
  861. Expression.NewArrayInit(
  862. typeof(string),
  863. ArrayUtils.ConvertAll(_lambda.ParameterNames, (x) => Expression.Constant(x))
  864. ),
  865. Expression.Constant(Flags),
  866. Expression.Constant(_lambda.IndexSpan.Start),
  867. Expression.Constant(_lambda.IndexSpan.End),
  868. Expression.Constant(_lambda.Filename),
  869. GetGeneratorOrNormalLambda(),
  870. TupleToStringArray(co_freevars),
  871. TupleToStringArray(co_names),
  872. TupleToStringArray(co_cellvars),
  873. TupleToStringArray(co_varnames),
  874. Expression.Constant(_localCount)
  875. );
  876. }
  877. private static Expression TupleToStringArray(PythonTuple tuple) {
  878. return tuple.Count > 0 ?
  879. (Expression)Expression.NewArrayInit(
  880. typeof(string),
  881. ArrayUtils.ConvertAll(tuple._data, (x) => Expression.Constant(x))
  882. ) :
  883. (Expression)Expression.Constant(null, typeof(string[]));
  884. }
  885. #endregion
  886. /// <summary>
  887. /// Extremely light weight linked list of weak references used for tracking
  888. /// all of the FunctionCode objects which get created and need to be updated
  889. /// for purposes of recursion enforcement or tracing.
  890. /// </summary>
  891. internal class CodeList {
  892. public readonly WeakReference Code;
  893. public CodeList Next;
  894. public CodeList() { }
  895. public CodeList(WeakReference code, CodeList next) {
  896. Code = code;
  897. Next = next;
  898. }
  899. }
  900. }
  901. internal class PythonDebuggingPayload {
  902. public FunctionCode Code;
  903. private Dictionary<int, bool> _handlerLocations;
  904. private Dictionary<int, Dictionary<int, bool>> _loopAndFinallyLocations;
  905. public PythonDebuggingPayload(FunctionCode code) {
  906. Code = code;
  907. }
  908. public Dictionary<int, bool> HandlerLocations {
  909. get {
  910. if (_handlerLocations == null) {
  911. GatherLocations();
  912. }
  913. return _handlerLocations;
  914. }
  915. }
  916. public Dictionary<int, Dictionary<int, bool>> LoopAndFinallyLocations {
  917. get {
  918. if (_loopAndFinallyLocations == null) {
  919. GatherLocations();
  920. }
  921. return _loopAndFinallyLocations;
  922. }
  923. }
  924. private void GatherLocations() {
  925. var walker = new TracingWalker();
  926. Code.PythonCode.Walk(walker);
  927. _loopAndFinallyLocations = walker.LoopAndFinallyLocations;
  928. _handlerLocations = walker.HandlerLocations;
  929. }
  930. class TracingWalker : Compiler.Ast.PythonWalker {
  931. private bool _inLoop, _inFinally;
  932. private int _loopId;
  933. public Dictionary<int, bool> HandlerLocations = new Dictionary<int, bool>();
  934. public Dictionary<int, Dictionary<int, bool>> LoopAndFinallyLocations = new Dictionary<int, Dictionary<int, bool>>();
  935. private Dictionary<int, bool> _loopIds = new Dictionary<int, bool>();
  936. public override bool Walk(Compiler.Ast.ForStatement node) {
  937. UpdateLoops(node);
  938. WalkLoopBody(node.Body, false);
  939. if (node.Else != null) {
  940. node.Else.Walk(this);
  941. }
  942. return false;
  943. }
  944. private void WalkLoopBody(IronPython.Compiler.Ast.Statement body, bool isFinally) {
  945. bool inLoop = _inLoop;
  946. bool inFinally = _inFinally;
  947. int loopId = ++_loopId;
  948. _inFinally = false;
  949. _inLoop = true;
  950. _loopIds.Add(loopId, isFinally);
  951. body.Walk(this);
  952. _inLoop = inLoop;
  953. _inFinally = inFinally;
  954. LoopOrFinallyIds.Remove(loopId);
  955. }
  956. public override bool Walk(Compiler.Ast.WhileStatement node) {
  957. UpdateLoops(node);
  958. WalkLoopBody(node.Body, false);
  959. if (node.ElseStatement != null) {
  960. node.ElseStatement.Walk(this);
  961. }
  962. return false;
  963. }

Large files files are truncated, but you can click here to view the full file