/DLR_Main/Languages/IronPython/IronPython/Runtime/PythonContext.cs

/* ****************************************************************************

 *

 * Copyright (c) Microsoft Corporation. 

 *

 * This source code is subject to terms and conditions of the Apache License, Version 2.0. A 

 * copy of the license can be found in the License.html file at the root of this distribution. If 

 * you cannot locate the  Apache License, Version 2.0, please send an email to 

 * dlr@microsoft.com. By using this source code in any fashion, you are agreeing to be bound 

 * by the terms of the Apache License, Version 2.0.

 *

 * You must not remove this notice, or any other, from this software.

 *

 *

 * ***************************************************************************/



#if !CLR2

using System.Linq.Expressions;

#else

using dynamic = System.Object;

using Microsoft.Scripting.Ast;

#endif



using System;

using System.Collections;

using System.Collections.Generic;

using System.Diagnostics;

using System.Dynamic;

using System.Globalization;

using System.IO;

using System.Reflection;

using System.Runtime.CompilerServices;

using System.Security;

using System.Text;

using System.Text.RegularExpressions;

using System.Threading;



using Microsoft.Scripting;

using Microsoft.Scripting.Actions;

using Microsoft.Scripting.Generation;

using Microsoft.Scripting.Runtime;

using Microsoft.Scripting.Utils;



using IronPython.Compiler;

using IronPython.Hosting;

using IronPython.Modules;

using IronPython.Runtime.Binding;

using IronPython.Runtime.Exceptions;

using IronPython.Runtime.Operations;

using IronPython.Runtime.Types;



using Debugging = Microsoft.Scripting.Debugging;

using PyAst = IronPython.Compiler.Ast;





namespace IronPython.Runtime {

    public delegate int HashDelegate(object o, ref HashDelegate dlg);



    public sealed partial class PythonContext : LanguageContext {

        internal const string/*!*/ IronPythonDisplayName = "IronPython 2.7 Alpha 1";

        internal const string/*!*/ IronPythonNames = "IronPython;Python;py";

        internal const string/*!*/ IronPythonFileExtensions = ".py";



        private static readonly Guid PythonLanguageGuid = new Guid("03ed4b80-d10b-442f-ad9a-47dae85b2051");

        private static readonly Guid LanguageVendor_Microsoft = new Guid(-1723120188, -6423, 0x11d2, 0x90, 0x3f, 0, 0xc0, 0x4f, 0xa3, 2, 0xa1);



        // fields used during startup

        private readonly IDictionary<object, object>/*!*/ _modulesDict = new PythonDictionary();

        private readonly Dictionary<string, ModuleGlobalCache>/*!*/ _builtinCache = new Dictionary<string, ModuleGlobalCache>(StringComparer.Ordinal);

        private readonly Dictionary<Type, string>/*!*/ _builtinModuleNames = new Dictionary<Type, string>();

        private readonly PythonOptions/*!*/ _options;

        private readonly PythonModule/*!*/ _systemState;

        private readonly Dictionary<string, Type>/*!*/ _builtinModulesDict;

        private readonly PythonOverloadResolverFactory _sharedOverloadResolverFactory;

        private readonly PythonBinder _binder;

        private readonly SysModuleDictionaryStorage _sysDict = new SysModuleDictionaryStorage();

#if !SILVERLIGHT

        private readonly AssemblyResolveHolder _resolveHolder;

#if !CLR2

        private readonly HashSet<Assembly> _loadedAssemblies = new HashSet<Assembly>();

#endif

#endif

        private Encoding _defaultEncoding = PythonAsciiEncoding.Instance;



        // conditional variables for silverlight/desktop CLR features

        private Hosting.PythonService _pythonService;

        private string _initialExecutable, _initialPrefix = GetInitialPrefix();



        // other fields which might only be conditionally used

        private string _initialVersionString;

        private PythonModule _clrModule;

        private PythonDictionary _builtinDict;

        private PythonModule _builtins;



        private PythonFileManager _fileManager;

        private Dictionary<string, object> _errorHandlers;

        private List<object> _searchFunctions;

        private Dictionary<object, object> _moduleState;

        /// <summary> stored for copy_reg module, used for reduce protocol </summary>

        internal BuiltinFunction NewObject;

        /// <summary> stored for copy_reg module, used for reduce protocol </summary>

        internal BuiltinFunction PythonReconstructor;

        private Dictionary<Type, object> _genericSiteStorage;



        private CallSite<Func<CallSite, CodeContext, object, object>>[] _newUnarySites;

        private CallSite<Func<CallSite, CodeContext, object, object, object, object>>[] _newTernarySites;



        private CallSite<Func<CallSite, object, object, int>> _compareSite;

        private Dictionary<AttrKey, CallSite<Func<CallSite, object, object, object>>> _setAttrSites;

        private Dictionary<AttrKey, CallSite<Action<CallSite, object>>> _deleteAttrSites;

        private CallSite<Func<CallSite, CodeContext, object, string, PythonTuple, PythonDictionary, object>> _metaClassSite;

        private CallSite<Func<CallSite, CodeContext, object, string, object>> _writeSite;

        private CallSite<Func<CallSite, object, object, object>> _getIndexSite, _equalSite;

        private CallSite<Action<CallSite, object, object>> _delIndexSite;

        private CallSite<Func<CallSite, CodeContext, object, object>> _finalizerSite;

        private CallSite<Func<CallSite, CodeContext, PythonFunction, object>> _functionCallSite;

        private CallSite<Func<CallSite, object, object, bool>> _greaterThanSite, _lessThanSite, _greaterThanEqualSite, _lessThanEqualSite, _containsSite;

        private CallSite<Func<CallSite, CodeContext, object, object[], object>> _callSplatSite;

        private CallSite<Func<CallSite, CodeContext, object, object[], IDictionary<object, object>, object>> _callDictSite;

        private CallSite<Func<CallSite, CodeContext, object, object, object, object>> _callDictSiteLooselyTyped;

        private CallSite<Func<CallSite, CodeContext, object, string, PythonDictionary, PythonDictionary, PythonTuple, int, object>> _importSite;

        private CallSite<Func<CallSite, CodeContext, object, string, PythonDictionary, PythonDictionary, PythonTuple, object>> _oldImportSite;

        private CallSite<Func<CallSite, object, bool>> _isCallableSite;

        private CallSite<Func<CallSite, object, IList<string>>> _getSignaturesSite;

        private CallSite<Func<CallSite, object, object, object>> _addSite, _divModSite, _rdivModSite;

        private CallSite<Func<CallSite, object, object, object, object>> _setIndexSite, _delSliceSite;

        private CallSite<Func<CallSite, object, object, object, object, object>> _setSliceSite;

        private CallSite<Func<CallSite, object, string>> _docSite;



        // conversion sites

        private CallSite<Func<CallSite, object, int>> _intSite;

        private CallSite<Func<CallSite, object, string>> _tryStringSite;

        private CallSite<Func<CallSite, object, object>> _tryIntSite;

        private CallSite<Func<CallSite, object, IEnumerable>> _tryIEnumerableSite;

        private Dictionary<Type, CallSite<Func<CallSite, object, object>>> _implicitConvertSites;

        private Dictionary<PythonOperationKind, CallSite<Func<CallSite, object, object, object>>> _binarySites;

        private Dictionary<Type, DefaultPythonComparer> _defaultComparer;

        private CallSite<Func<CallSite, CodeContext, object, object, object, int>> _sharedFunctionCompareSite;

        private CallSite<Func<CallSite, CodeContext, PythonFunction, object, object, int>> _sharedPythonFunctionCompareSite;

        private CallSite<Func<CallSite, CodeContext, BuiltinFunction, object, object, int>> _sharedBuiltinFunctionCompareSite;

        private CallSite<Func<CallSite, CodeContext, object, int, object>> _getItemCallSite;



        private CallSite<Func<CallSite, CodeContext, object, object, object>> _propGetSite, _propDelSite;

        private CallSite<Func<CallSite, CodeContext, object, object, object, object>> _propSetSite;

        private CompiledLoader _compiledLoader;

        internal bool _importWarningThrows;

        private bool _importedEncodings;

        private Action<Action> _commandDispatcher; // can be null

        private ClrModule.ReferencesList _referencesList;

        private FloatFormat _floatFormat, _doubleFormat;

        private CultureInfo _collateCulture, _ctypeCulture, _timeCulture, _monetaryCulture, _numericCulture;

        private CodeContext _defaultContext, _defaultClsContext;

        private readonly TopNamespaceTracker _topNamespace;

        private readonly IEqualityComparer<object> _equalityComparer;

        private readonly IEqualityComparer _equalityComparerNonGeneric;



        private Dictionary<Type, CallSite<Func<CallSite, object, object, bool>>> _equalSites;



        private Dictionary<Type, PythonSiteCache> _systemSiteCache;

        internal static object _syntaxErrorNoCaret = new object();



        // atomized binders

        private PythonInvokeBinder _invokeNoArgs, _invokeOneArg;

        private Dictionary<CallSignature, PythonInvokeBinder/*!*/> _invokeBinders;

        private Dictionary<string/*!*/, PythonGetMemberBinder/*!*/> _getMemberBinders;

        private Dictionary<string/*!*/, PythonGetMemberBinder/*!*/> _tryGetMemberBinders;

        private Dictionary<string/*!*/, PythonSetMemberBinder/*!*/> _setMemberBinders;

        private Dictionary<string/*!*/, PythonDeleteMemberBinder/*!*/> _deleteMemberBinders;

        private Dictionary<string/*!*/, CompatibilityGetMember/*!*/> _compatGetMember;

        private Dictionary<string/*!*/, CompatibilityGetMember/*!*/> _compatGetMemberNoThrow;

        private Dictionary<PythonOperationKind, PythonOperationBinder/*!*/> _operationBinders;

        private Dictionary<ExpressionType, PythonUnaryOperationBinder/*!*/> _unaryBinders;

        private PythonBinaryOperationBinder[] _binaryBinders;

        private Dictionary<OperationRetTypeKey<ExpressionType>, BinaryRetTypeBinder/*!*/> _binaryRetTypeBinders;

        private Dictionary<OperationRetTypeKey<PythonOperationKind>, BinaryRetTypeBinder/*!*/> _operationRetTypeBinders;

        private Dictionary<Type/*!*/, PythonConversionBinder/*!*/>[] _conversionBinders;

        private Dictionary<Type/*!*/, DynamicMetaObjectBinder/*!*/>[] _convertRetObjectBinders;

        private Dictionary<CallSignature, CreateFallback/*!*/> _createBinders;

        private Dictionary<CallSignature, CompatibilityInvokeBinder/*!*/> _compatInvokeBinders;

        private PythonGetSliceBinder _getSlice;

        private PythonSetSliceBinder _setSlice;

        private PythonDeleteSliceBinder _deleteSlice;

        private PythonGetIndexBinder[] _getIndexBinders;

        private PythonSetIndexBinder[] _setIndexBinders;

        private PythonDeleteIndexBinder[] _deleteIndexBinders;

        private DynamicMetaObjectBinder _invokeTwoConvertToInt;

        private static CultureInfo _CCulture;

        private DynamicDelegateCreator _delegateCreator;

        // tracing / in-proc debugging support

        private Debugging.CompilerServices.DebugContext _debugContext;

        private Debugging.ITracePipeline _tracePipeline;

        

        [ThreadStatic]

        private static Stack<PythonTracebackListener> _tracebackListeners;

        private static int _tracingThreads;



        internal FunctionCode.CodeList _allCodes;

        internal readonly object _codeCleanupLock = new object(), _codeUpdateLock = new object();

        internal int _codeCount, _nextCodeCleanup = 200;

        private int _recursionLimit;

        [ThreadStatic]

        private static bool _enableTracing;



        internal readonly List<FunctionStack> _mainThreadFunctionStack;

        private CallSite<Func<CallSite, CodeContext, object, object>> _callSite0LightEh;



        #region Generated Python Shared Call Sites Storage



        // *** BEGIN GENERATED CODE ***

        // generated by function: gen_shared_call_sites_storage from: generate_calls.py



        private CallSite<Func<CallSite, CodeContext, object, object>> _callSite0;

        private CallSite<Func<CallSite, CodeContext, object, object, object>> _callSite1;

        private CallSite<Func<CallSite, CodeContext, object, object, object, object>> _callSite2;

        private CallSite<Func<CallSite, CodeContext, object, object, object, object, object>> _callSite3;

        private CallSite<Func<CallSite, CodeContext, object, object, object, object, object, object>> _callSite4;

        private CallSite<Func<CallSite, CodeContext, object, object, object, object, object, object, object>> _callSite5;

        private CallSite<Func<CallSite, CodeContext, object, object, object, object, object, object, object, object>> _callSite6;



        // *** END GENERATED CODE ***



        #endregion



        /// <summary>

        /// Creates a new PythonContext not bound to Engine.

        /// </summary>

        public PythonContext(ScriptDomainManager/*!*/ manager, IDictionary<string, object> options)

            : base(manager) {

            _options = new PythonOptions(options);

            _builtinModulesDict = CreateBuiltinTable();



            PythonDictionary defaultScope = new PythonDictionary();

            ModuleContext modContext = new ModuleContext(defaultScope, this);

            _defaultContext = modContext.GlobalContext;

            

            PythonDictionary sysDict = new PythonDictionary(_sysDict);

            _systemState = new PythonModule(sysDict);

            _systemState.__dict__["__name__"] = "sys";

            _systemState.__dict__["__package__"] = null;



            PythonBinder binder = new PythonBinder(this, _defaultContext);

            _sharedOverloadResolverFactory = new PythonOverloadResolverFactory(binder, Expression.Constant(_defaultContext));

            _binder = binder;



            CodeContext defaultClsContext = DefaultContext.CreateDefaultCLSContext(this);

            _defaultClsContext = defaultClsContext;



            if (DefaultContext._default == null) {

                DefaultContext.InitializeDefaults(_defaultContext, defaultClsContext);

            }



            InitializeBuiltins();



            InitializeSystemState();

#if SILVERLIGHT

            AddToPath("");

#endif



            // sys.argv always includes at least one empty string.

            SetSystemStateValue("argv", (_options.Arguments.Count == 0) ?

                new List(new object[] { String.Empty }) :

                new List(_options.Arguments)

            );



            if (_options.WarningFilters.Count > 0) {

                _systemState.__dict__["warnoptions"] = new List(_options.WarningFilters);

            }



            if (_options.Frames) {

                var getFrame = BuiltinFunction.MakeFunction(

                    "_getframe", 

                    ArrayUtils.ConvertAll(typeof(SysModule).GetMember("_getframeImpl"), (x) => (MethodBase)x), 

                    typeof(SysModule)

                );

                _systemState.__dict__["_getframe"] = getFrame;

            }



            if (_options.Tracing) {

                EnsureDebugContext();

                RegisterTracebackHandler();

            }



            List path = new List(_options.SearchPaths);

#if !SILVERLIGHT

            _resolveHolder = new AssemblyResolveHolder(this);

            try {

                Assembly entryAssembly = Assembly.GetEntryAssembly();

                // Can be null if called from unmanaged code (VS integration scenario)

                if (entryAssembly != null) {

                    string entry = Path.GetDirectoryName(entryAssembly.Location);

                    string lib = Path.Combine(entry, "Lib");

                    path.append(lib);



                    // add DLLs directory for user-defined extention modules

                    path.append(Path.Combine(entry, "DLLs"));

                }

            } catch (SecurityException) {

            }

#endif



            _systemState.__dict__["path"] = path;



            RecursionLimit = _options.RecursionLimit;



#if !SILVERLIGHT

            object asmResolve;

            if (options == null ||

                !options.TryGetValue("NoAssemblyResolveHook", out asmResolve) ||

                !System.Convert.ToBoolean(asmResolve)) {

                try {

                    HookAssemblyResolve();

                } catch (System.Security.SecurityException) {

                    // We may not have SecurityPermissionFlag.ControlAppDomain. 

                    // If so, we will not look up sys.path for module loads

                }

            }

#endif



            _equalityComparer = new PythonEqualityComparer(this);

            _equalityComparerNonGeneric = (IEqualityComparer)_equalityComparer;



            InitialHasher = InitialHasherImpl;

            IntHasher = IntHasherImpl;

            DoubleHasher = DoubleHasherImpl;

            StringHasher = StringHasherImpl;

            FallbackHasher = FallbackHasherImpl;



            _topNamespace = new TopNamespaceTracker(manager);

            foreach (Assembly asm in manager.GetLoadedAssemblyList()) {

                _topNamespace.LoadAssembly(asm);

            }

            manager.AssemblyLoaded += new EventHandler<AssemblyLoadedEventArgs>(ManagerAssemblyLoaded);



            _mainThreadFunctionStack = Runtime.Operations.PythonOps.GetFunctionStack();

        }



        void ManagerAssemblyLoaded(object sender, AssemblyLoadedEventArgs e) {

            _topNamespace.LoadAssembly(e.Assembly);

        }



        /// <summary>

        /// Gets or sets the maximum depth of function calls.  Equivalent to sys.getrecursionlimit

        /// and sys.setrecursionlimit.

        /// </summary>

        public int RecursionLimit {

            get {

                return _recursionLimit;

            }

            set {

                if (value < 0) {

                    throw PythonOps.ValueError("recursion limit must be positive");

                }



                lock (_codeUpdateLock) {

                    int oldRecLimit = _recursionLimit;

                    _recursionLimit = value;



                    if ((_recursionLimit == Int32.MaxValue) != (value == Int32.MaxValue)) {

                        // recursion setting has changed, we need to update all of our

                        // function codes to enforce or un-enforce recursion.

                        FunctionCode.UpdateAllCode(this);

                    }

                }

            }

        }



        internal bool EnableTracing {

            get {

                return _tracingThreads > 0 || PythonOptions.Tracing;

            }

            set {

                lock (_codeUpdateLock) {

                    bool oldEnableTracing = _enableTracing;

                    _enableTracing = value;



                    bool flip = false;

                    if (value && !oldEnableTracing) {

                        flip = _tracingThreads == 0;

                        _tracingThreads++;

                    } else if (!value && oldEnableTracing) {

                        _tracingThreads--;

                        flip = _tracingThreads == 0;

                        if (flip) {

                            _tracePipeline.TraceCallback = null;

                        }

                    }



                    if (flip) {

                        // recursion setting has changed, we need to update all of our

                        // function codes to enforce or un-enforce recursion.

                        FunctionCode.UpdateAllCode(this);

                    }

                }

            }

        }



        internal TopNamespaceTracker TopNamespace {

            get {

                return _topNamespace;

            }

        }



        public IEqualityComparer<object>/*!*/ EqualityComparer {

            get { return _equalityComparer; }

        }



        public IEqualityComparer/*!*/ EqualityComparerNonGeneric {

            get { return _equalityComparerNonGeneric; }

        }



        internal sealed class PythonEqualityComparer : IEqualityComparer, IEqualityComparer<object> {

            public readonly PythonContext/*!*/ Context;



            public PythonEqualityComparer(PythonContext/*!*/ context) {

                Assert.NotNull(context);

                Context = context;

            }



            bool IEqualityComparer.Equals(object x, object y) {

                return PythonOps.EqualRetBool(Context._defaultContext, x, y);

            }



            bool IEqualityComparer<object>.Equals(object x, object y) {

                return PythonOps.EqualRetBool(Context._defaultContext, x, y);

            }



            int IEqualityComparer.GetHashCode(object obj) {

                return PythonContext.Hash(obj);

            }



            int IEqualityComparer<object>.GetHashCode(object obj) {

                return PythonContext.Hash(obj);

            }

        }



        #region Specialized Hashers



        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes")]

        internal readonly HashDelegate InitialHasher;

        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes")]

        internal readonly HashDelegate IntHasher;

        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes")]

        internal readonly HashDelegate DoubleHasher;

        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes")]

        internal readonly HashDelegate StringHasher;

        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes")]

        internal readonly HashDelegate FallbackHasher;



        private int InitialHasherImpl(object o, ref HashDelegate dlg) {

            if (o == null) {

                return NoneTypeOps.NoneHashCode;

            }



            switch (Type.GetTypeCode(o.GetType())) {

                case TypeCode.String:

                    dlg = StringHasher;

                    return StringHasher(o, ref dlg);

                case TypeCode.Int32:

                    dlg = IntHasher;

                    return IntHasher(o, ref dlg);

                case TypeCode.Double:

                    dlg = DoubleHasher;

                    return DoubleHasher(o, ref dlg);

                default:

                    if (o is IPythonObject) {

                        dlg = new OptimizedUserHasher(this, ((IPythonObject)o).PythonType).Hasher;

                    } else {

                        dlg = new OptimizedBuiltinHasher(this, o.GetType()).Hasher;

                    }



                    return dlg(o, ref dlg);

            }

        }



        private int IntHasherImpl(object o, ref HashDelegate dlg) {

            if (o != null && o.GetType() == typeof(int)) {

                return o.GetHashCode();

            }



            dlg = FallbackHasher;

            return FallbackHasher(o, ref dlg);

        }



        private int DoubleHasherImpl(object o, ref HashDelegate dlg) {

            if (o != null && o.GetType() == typeof(double)) {

                return DoubleOps.__hash__((double)o);

            }



            dlg = FallbackHasher;

            return FallbackHasher(o, ref dlg);

        }



        private int StringHasherImpl(object o, ref HashDelegate dlg) {

            if (o != null && o.GetType() == typeof(string)) {

                return o.GetHashCode();

            }



            dlg = FallbackHasher;

            return FallbackHasher(o, ref dlg);

        }



        private int FallbackHasherImpl(object o, ref HashDelegate dlg) {

            return PythonOps.Hash(this.SharedContext, o);

        }



        private sealed class OptimizedUserHasher {

            private readonly PythonContext _context;

            private readonly PythonType _pt;



            public OptimizedUserHasher(PythonContext context, PythonType pt) {

                _context = context;

                _pt = pt;

            }



            public int Hasher(object o, ref HashDelegate dlg) {

                IPythonObject ipo = o as IPythonObject;

                if (ipo != null && ipo.PythonType == _pt) {

                    return _pt.Hash(o);

                }



                dlg = _context.FallbackHasher;

                return _context.FallbackHasher(o, ref dlg);

            }

        }



        private sealed class OptimizedBuiltinHasher {

            private readonly PythonContext _context;

            private readonly Type _type;

            private readonly PythonType _pt;



            public OptimizedBuiltinHasher(PythonContext context, Type type) {

                _context = context;

                _type = type;

                _pt = DynamicHelpers.GetPythonTypeFromType(type);

            }



            public int Hasher(object o, ref HashDelegate dlg) {

                if (o != null && o.GetType() == _type) {

                    return _pt.Hash(o);

                }



                dlg = _context.FallbackHasher;

                return _context.FallbackHasher(o, ref dlg);

            }

        }



        #endregion



        public override LanguageOptions/*!*/ Options {

            get { return PythonOptions; }

        }



        /// <summary>

        /// Checks to see if module state has the current value stored already.

        /// </summary>

        public bool HasModuleState(object key) {

            EnsureModuleState();



            lock (_moduleState) {

                return _moduleState.ContainsKey(key);

            }

        }



        private void EnsureModuleState() {

            if (_moduleState == null) {

                Interlocked.CompareExchange(ref _moduleState, new Dictionary<object, object>(), null);

            }

        }



        /// <summary>

        /// Gets per-runtime state used by a module.  The module should have a unique key for

        /// each piece of state it needs to store.

        /// </summary>

        public object GetModuleState(object key) {

            EnsureModuleState();



            lock (_moduleState) {

                Debug.Assert(_moduleState.ContainsKey(key));



                return _moduleState[key];

            }

        }



        /// <summary>

        /// Sets per-runtime state used by a module.  The module should have a unique key for

        /// each piece of state it needs to store.

        /// </summary>

        public void SetModuleState(object key, object value) {

            EnsureModuleState();



            lock (_moduleState) {

                _moduleState[key] = value;

            }

        }



        /// <summary>

        /// Sets per-runtime state used by a module and returns the previous value.  The module

        /// should have a unique key for each piece of state it needs to store.

        /// </summary>

        public object GetSetModuleState(object key, object value) {

            EnsureModuleState();



            lock (_moduleState) {

                object result;

                _moduleState.TryGetValue(key, out result);

                _moduleState[key] = value;

                return result;

            }

        }



        /// <summary>

        /// Sets per-runtime state used by a module and returns the previous value.  The module

        /// should have a unique key for each piece of state it needs to store.

        /// </summary>

        public T GetOrCreateModuleState<T>(object key, Func<T> value) where T : class {

            EnsureModuleState();



            lock (_moduleState) {

                object result;

                if (!_moduleState.TryGetValue(key, out result)) {

                    _moduleState[key] = result = value();

                }

                return (result as T);

            }

        }



        public PythonType EnsureModuleException(object key, PythonDictionary dict, string name, string module) {

            return (PythonType)(dict[name] = GetOrCreateModuleState(

                key,

                () => PythonExceptions.CreateSubType(this, PythonExceptions.Exception, name, module, "", PythonType.DefaultMakeException)

            ));

        }



        public PythonType EnsureModuleException(object key, PythonType baseType, PythonDictionary dict, string name, string module) {

            return (PythonType)(dict[name] = GetOrCreateModuleState(

                key,

                () => PythonExceptions.CreateSubType(this, baseType, name, module, "", PythonType.DefaultMakeException)

            ));

        }



        public PythonType EnsureModuleException(object key, PythonType baseType, Type underlyingType, PythonDictionary dict, string name, string module, Func<string, Exception> exceptionMaker) {

            return (PythonType)(dict[name] = GetOrCreateModuleState(

                key,

                () => PythonExceptions.CreateSubType(this, baseType, underlyingType, name, module, "", exceptionMaker)

            ));

        }



        public PythonType EnsureModuleException(object key, PythonType[] baseTypes, Type underlyingType, PythonDictionary dict, string name, string module) {

            return (PythonType)(dict[name] = GetOrCreateModuleState(

                key,

                () => PythonExceptions.CreateSubType(this, baseTypes, underlyingType, name, module, "", PythonType.DefaultMakeException)

            ));

        }



        internal PythonOptions/*!*/ PythonOptions {

            get {

                return _options;

            }

        }



        public override Guid VendorGuid {

            get {

                return LanguageVendor_Microsoft;

            }

        }



        public override Guid LanguageGuid {

            get {

                return PythonLanguageGuid;

            }

        }



        public PythonModule/*!*/ SystemState {

            get {

                return _systemState;

            }

        }



        public PythonModule/*!*/ ClrModule {

            get {

                if (_clrModule == null) {

                    Interlocked.CompareExchange(ref _clrModule, CreateBuiltinModule("clr"), null);

                }



                return _clrModule;

            }

        }



        internal bool TryGetSystemPath(out List path) {

            object val;

            if (SystemState.__dict__.TryGetValue("path", out val)) {

                path = val as List;

            } else {

                path = null;

            }



            return path != null;

        }



        internal object SystemStandardOut {

            get {

                return GetSystemStateValue("stdout");

            }

        }



        internal object SystemStandardIn {

            get {

                return GetSystemStateValue("stdin");

            }

        }



        internal object SystemStandardError {

            get {

                return GetSystemStateValue("stderr");

            }

        }



        internal IDictionary<object, object> SystemStateModules {

            get {

                return _modulesDict;

            }

        }



        internal void UpdateExceptionInfo(object type, object value, object traceback) {

            _sysDict.UpdateExceptionInfo(type, value, traceback);

        }



        internal void UpdateExceptionInfo(Exception clrException, object type, object value, List<DynamicStackFrame> traceback) {

            _sysDict.UpdateExceptionInfo(clrException, type, value, traceback);            

        }



        internal void ExceptionHandled() {

            _sysDict.ExceptionHandled();

        }



        internal PythonModule GetModuleByName(string/*!*/ name) {

            Assert.NotNull(name);

            object scopeObj;

            PythonModule module;

            if (SystemStateModules.TryGetValue(name, out scopeObj) && (module = scopeObj as PythonModule) != null) {

                return module;

            }

            return null;

        }



        internal PythonModule GetModuleByPath(string/*!*/ path) {

            Assert.NotNull(path);

            foreach (object moduleObj in SystemStateModules.Values) {

                PythonModule module = moduleObj as PythonModule;

                if (module != null) {

                    if (DomainManager.Platform.PathComparer.Compare(module.GetFile(), path) == 0) {

                        return module;

                    }

                }

            }

            return null;

        }



        public override Version LanguageVersion {

            get {

                // Assembly.GetName() can't be called in Silverlight...

                return GetPythonVersion();

            }

        }



        internal static Version GetPythonVersion() {

            return new AssemblyName(typeof(PythonContext).Assembly.FullName).Version;

        }



        internal FloatFormat FloatFormat {

            get {

                return _floatFormat;

            }

            set {

                _floatFormat = value;

            }

        }



        internal FloatFormat DoubleFormat {

            get {

                return _doubleFormat;

            }

            set {

                _doubleFormat = value;

            }

        }



        /// <summary>

        /// Initializes the sys module on startup.  Called both to load and reload sys

        /// </summary>

        private void InitializeSystemState() {

            // These fields do not get reset on "reload(sys)", we populate them once on startup

            SetSystemStateValue("argv", List.FromArrayNoCopy(new object[] { String.Empty }));

            SetSystemStateValue("modules", _modulesDict);

            InitializeSysFlags();



            _modulesDict["sys"] = _systemState;



            SetSystemStateValue("path", new List(3));



            SetStandardIO();



            SysModule.PerformModuleReload(this, _systemState.__dict__);

        }



        internal bool EmitDebugSymbols(SourceUnit sourceUnit) {

            return sourceUnit.EmitDebugSymbols && (PythonOptions.NoDebug == null || !PythonOptions.NoDebug.IsMatch(sourceUnit.Path));

        }



        private void InitializeSysFlags() {

            // sys.flags

            SysModule.SysFlags flags = new SysModule.SysFlags();

            SetSystemStateValue("flags", flags);

            flags.debug = _options.Debug ? 1 : 0;

            flags.py3k_warning = _options.WarnPython30 ? 1 : 0;

            SetSystemStateValue("py3kwarning", _options.WarnPython30);

            switch (_options.DivisionOptions) {

                case PythonDivisionOptions.Old:

                    break;

                case PythonDivisionOptions.New:

                    flags.division_new = 1;

                    break;

                case PythonDivisionOptions.Warn:

                    flags.division_warning = 1;

                    break;

                case PythonDivisionOptions.WarnAll:

                    flags.division_warning = 2;

                    break;

            }

            flags.inspect = flags.interactive = _options.Inspect ? 1 : 0;

            if (_options.StripDocStrings) {

                flags.optimize = 2;

            } else if (_options.Optimize) {

                flags.optimize = 1;

            }

            flags.dont_write_bytecode = 1;

            SetSystemStateValue("dont_write_bytecode", true);

            flags.no_user_site = _options.NoUserSite ? 1 : 0;

            flags.no_site = _options.NoSite ? 1 : 0;

            flags.ignore_environment = _options.IgnoreEnvironment ? 1 : 0;

            switch (_options.IndentationInconsistencySeverity) {

                case Severity.Warning:

                    flags.tabcheck = 1;

                    break;

                case Severity.Error:

                    flags.tabcheck = 2;

                    break;

            }

            flags.verbose = _options.Verbose ? 1 : 0;

            flags.unicode = 1;

            flags.bytes_warning = _options.BytesWarning ? 1 : 0;

        }



        internal bool ShouldInterpret(PythonCompilerOptions options, SourceUnit source) {

            // We have to turn off adaptive compilation in debug mode to

            // support mangaged debuggers. Also turn off in optimized mode.

            bool adaptiveCompilation = !_options.NoAdaptiveCompilation && !EmitDebugSymbols(source);



            return options.Interpreted || adaptiveCompilation;

        }



        private static PyAst.PythonAst ParseAndBindAst(CompilerContext context) {

            ScriptCodeParseResult properties = ScriptCodeParseResult.Complete;

            bool propertiesSet = false;

            int errorCode = 0;



            PyAst.PythonAst ast;

            using (Parser parser = Parser.CreateParser(context, PythonContext.GetPythonOptions(null))) {

                switch (context.SourceUnit.Kind) {

                    case SourceCodeKind.InteractiveCode:

                        ast = parser.ParseInteractiveCode(out properties);

                        propertiesSet = true;

                        break;



                    case SourceCodeKind.Expression:

                        ast = parser.ParseTopExpression();

                        break;



                    case SourceCodeKind.SingleStatement:

                        ast = parser.ParseSingleStatement();

                        break;



                    case SourceCodeKind.File:

                        ast = parser.ParseFile(true, false);

                        break;



                    case SourceCodeKind.Statements:

                        ast = parser.ParseFile(false, false);

                        break;



                    default:

                    case SourceCodeKind.AutoDetect:

                        ast = parser.ParseFile(true, true);

                        break;

                }



                errorCode = parser.ErrorCode;

            }



            if (!propertiesSet && errorCode != 0) {

                properties = ScriptCodeParseResult.Invalid;

            }



            context.SourceUnit.CodeProperties = properties;



            if (errorCode != 0 || properties == ScriptCodeParseResult.Empty) {

                return null;

            }



            ast.Bind();

            return ast;

        }



        internal static ScriptCode CompilePythonCode(SourceUnit/*!*/ sourceUnit, CompilerOptions/*!*/ options, ErrorSink/*!*/ errorSink) {

            var pythonOptions = (PythonCompilerOptions)options;



            if (sourceUnit.Kind == SourceCodeKind.File) {

                pythonOptions.Module |= ModuleOptions.Initialize;

            }



            CompilerContext context = new CompilerContext(sourceUnit, options, errorSink);



            PyAst.PythonAst ast = ParseAndBindAst(context);

            if (ast == null) {

                return null;

            }

            

            return ast.ToScriptCode();

        }



        public override ScriptCode CompileSourceCode(SourceUnit/*!*/ sourceUnit, CompilerOptions/*!*/ options, ErrorSink/*!*/ errorSink) {            

            ScriptCode res = CompilePythonCode(sourceUnit, options, errorSink);

            if (res != null) {

                Scope scope = res.CreateScope();



                // if this is an optimized module we need to initialize the optimized scope.

                // Optimized scopes come w/ extensions already attached so we use that to know

                // if we're optimized or not.

                PythonScopeExtension scopeExtension = (PythonScopeExtension)scope.GetExtension(ContextId);

                if (scopeExtension != null) {

                    InitializeModule(sourceUnit.Path, scopeExtension.ModuleContext, res, ModuleOptions.None);

                }

            }



            return res;

        }



        public override ScriptCode/*!*/ LoadCompiledCode(Delegate/*!*/ method, string path, string customData) {

            SourceUnit su = new SourceUnit(this, NullTextContentProvider.Null, path, SourceCodeKind.File);

            return new OnDiskScriptCode((LookupCompilationDelegate)method, su, customData);

        }



        public override SourceCodeReader/*!*/ GetSourceReader(Stream/*!*/ stream, Encoding/*!*/ defaultEncoding, string path) {

            ContractUtils.RequiresNotNull(stream, "stream");

            ContractUtils.RequiresNotNull(defaultEncoding, "defaultEncoding");

            ContractUtils.Requires(stream.CanSeek && stream.CanRead, "stream", "The stream must support seeking and reading");



            // we choose ASCII by default, if the file has a Unicode pheader though

            // we'll automatically get it as unicode.

            Encoding encoding = PythonAsciiEncoding.SourceEncoding;



            long startPosition = stream.Position;



            StreamReader sr = new StreamReader(stream, PythonAsciiEncoding.SourceEncoding);

            byte[] bomBuffer = new byte[3];

            int bomRead = stream.Read(bomBuffer, 0, 3);

            int bytesRead = 0;

            bool isUtf8 = false;

            if (bomRead == 3 && (bomBuffer[0] == 0xef && bomBuffer[1] == 0xbb && bomBuffer[2] == 0xbf)) {

                isUtf8 = true;

                bytesRead = 3;

            } else {

                stream.Seek(0, SeekOrigin.Begin);

            }



            string line;

            try {

                line = ReadOneLine(sr, ref bytesRead);

            } catch (BadSourceException) {

                throw ReportEncodingError(stream, path);                

            }



            bool gotEncoding = false;

            string encodingName = null;

            // magic encoding must be on line 1 or 2

            if (line != null && !(gotEncoding = Tokenizer.TryGetEncoding(defaultEncoding, line, ref encoding, out encodingName))) {

                try {

                    line = ReadOneLine(sr, ref bytesRead);

                } catch (BadSourceException) {

                    throw ReportEncodingError(stream, path);

                }



                if (line != null) {

                    gotEncoding = Tokenizer.TryGetEncoding(defaultEncoding, line, ref encoding, out encodingName);

                }

            }



            if (gotEncoding && isUtf8 && encodingName != "utf-8") {

                // we have both a BOM & an encoding type, throw an error

                throw new IOException("file has both Unicode marker and PEP-263 file encoding.  You can only use \"utf-8\" as the encoding name when a BOM is present.");

            } else if (encoding == null) {

                throw new IOException("unknown encoding type");

            }



            // if we didn't get an encoding seek back to the beginning...

            if (!gotEncoding || stream.Position != stream.Length) {

                stream.Seek(startPosition, SeekOrigin.Begin);

            }



            // re-read w/ the correct encoding type...

            return new SourceCodeReader(new StreamReader(stream, encoding), encoding);

        }



        internal static Exception ReportEncodingError(Stream stream, string path) {

            stream.Seek(0, SeekOrigin.Begin);

            byte[] buffer = new byte[1024];

            int bytesRead = 0;

            int curLine = 1, curOffset = 1, index = 0;

            while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) != -1) {

                for (int i = 0; i < bytesRead; i++) {

                    if (buffer[i] > 0x7f) {

                        return PythonOps.BadSourceError(

                            buffer[i], 

                            new SourceSpan(

                                new SourceLocation(index, curLine, curOffset),

                                new SourceLocation(index, curLine, curOffset)

                            ), 

                            path

                        );

                    } else if (buffer[i] == '\n') {

                        curLine++;

                        curOffset = 1;

                    } else {

                        curOffset++;

                    }



                    index++;

                }

            }



            return new InvalidOperationException();

        }



        /// <summary>

        /// Reads one line keeping track of the # of bytes read

        /// </summary>

        private static string ReadOneLine(StreamReader reader, ref int totalRead) {

            Stream sr = reader.BaseStream;

            byte[] buffer = new byte[256];

            StringBuilder builder = null;

            

            int bytesRead = sr.Read(buffer, 0, buffer.Length);



            while (bytesRead > 0) {

                totalRead += bytesRead;



                bool foundEnd = false;

                for (int i = 0; i < bytesRead; i++) {

                    if (buffer[i] == '\r') {

                        if (i + 1 < bytesRead) {

                            if (buffer[i + 1] == '\n') {

                                totalRead -= (bytesRead - (i + 2));   // skip cr/lf

                                sr.Seek(i + 2, SeekOrigin.Begin);

                                reader.DiscardBufferedData();

                                foundEnd = true;

                            }

                        } else {

                            totalRead -= (bytesRead - (i + 1)); // skip cr

                            sr.Seek(i + 1, SeekOrigin.Begin);

                            reader.DiscardBufferedData();

                            foundEnd = true;

                        }

                    } else if (buffer[i] == '\n') {

                        totalRead -= (bytesRead - (i + 1)); // skip lf

                        sr.Seek(i + 1, SeekOrigin.Begin);

                        reader.DiscardBufferedData();

                        foundEnd = true;

                    }



                    if (foundEnd) {

                        if (builder != null) {

                            builder.Append(buffer.MakeString(), 0, i);

                            return builder.ToString();

                        }

                        return buffer.MakeString().Substring(0, i);

                    }

                }



                if (builder == null) builder = new StringBuilder();

                builder.Append(buffer.MakeString(), 0, bytesRead);

                bytesRead = sr.Read(buffer, 0, buffer.Length);

            }



            // no string

            if (builder == null) {

                return null;

            }



            // no new-line

            return builder.ToString();

        }



#if !SILVERLIGHT

        // Convert a CodeDom to source code, and output the generated code and the line number mappings (if any)

        public override SourceUnit/*!*/ GenerateSourceCode(System.CodeDom.CodeObject codeDom, string path, SourceCodeKind kind) {

            return new IronPython.Hosting.PythonCodeDomCodeGen().GenerateCode((System.CodeDom.CodeMemberMethod)codeDom, this, path, kind);

        }

#endif



        #region Scopes



        public override Scope GetScope(string/*!*/ path) {

            PythonModule module = GetModuleByPath(path);

            return (module != null) ? module.Scope : null;

        }



        public PythonModule/*!*/ InitializeModule(string fileName, ModuleContext moduleContext, ScriptCode scriptCode, ModuleOptions options) {

            if ((options & ModuleOptions.NoBuiltins) == 0) {

                moduleContext.InitializeBuiltins((options & ModuleOptions.ModuleBuiltins) != 0);

            }



            // If the filename is __init__.py then this is the initialization code

            // for a package and we need to set the __path__ variable appropriately

            if (fileName != null && Path.GetFileName(fileName) == "__init__.py") {

                string dirname = Path.GetDirectoryName(fileName);

                string dir_path = DomainManager.Platform.GetFullPath(dirname);

                moduleContext.Globals["__path__"] = PythonOps.MakeList(dir_path);

            }



            moduleContext.ShowCls = (options & ModuleOptions.ShowClsMethods) != 0;

            moduleContext.Features = options;



            if ((options & ModuleOptions.Initialize) != 0) {

                scriptCode.Run(moduleContext.GlobalScope);



                if (!moduleContext.Globals.ContainsKey("__package__")) {

                    moduleContext.Globals["__package__"] = null;

                }

            }



            return moduleContext.Module;

        }



        public override ScopeExtension CreateScopeExtension(Scope scope) {

            var ret = new PythonScopeExtension(this, scope);

            ret.ModuleContext.InitializeBuiltins(false);



            return ret;

        }



        internal PythonModule/*!*/ CompileModule(string fileName, string moduleName, SourceUnit sourceCode, ModuleOptions options) {

            ScriptCode compiledCode;

            return CompileModule(fileName, moduleName, sourceCode, options, out compiledCode);

        }



        internal PythonModule/*!*/ CompileModule(string fileName, string moduleName, SourceUnit so