/Languages/IronPython/IronPython/Runtime/PythonContext.cs

Large files files are truncated, but you can click here to view the full file

/* ****************************************************************************
 *
 * Copyright (c) Microsoft Corporation. 
 *
 * This source code is subject to terms and conditions of the Apache License, Version 2.0. A 
 * copy of the license can be found in the License.html file at the root of this distribution. If 
 * you cannot locate the  Apache License, Version 2.0, please send an email to 
 * dlr@microsoft.com. By using this source code in any fashion, you are agreeing to be bound 
 * by the terms of the Apache License, Version 2.0.
 *
 * You must not remove this notice, or any other, from this software.
 *
 *
 * ***************************************************************************/

#if FEATURE_CORE_DLR
using System.Linq.Expressions;
#else
using dynamic = System.Object;
using Microsoft.Scripting.Ast;
#endif

using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Dynamic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Security;
using System.Text;
using System.Threading;

using Microsoft.Scripting;
using Microsoft.Scripting.Actions;
using Microsoft.Scripting.Debugging.CompilerServices;
using Microsoft.Scripting.Generation;
using Microsoft.Scripting.Runtime;
using Microsoft.Scripting.Utils;

using IronPython.Compiler;
using IronPython.Hosting;
using IronPython.Modules;
using IronPython.Runtime.Binding;
using IronPython.Runtime.Exceptions;
using IronPython.Runtime.Operations;
using IronPython.Runtime.Types;

using Debugging = Microsoft.Scripting.Debugging;
using PyAst = IronPython.Compiler.Ast;

#if NETCOREAPP1_0
using Environment = System.FakeEnvironment;
#endif

namespace IronPython.Runtime
{
    public delegate int HashDelegate(object o, ref HashDelegate dlg);

    public sealed partial class PythonContext : LanguageContext {
        internal const string/*!*/ IronPythonDisplayName = CurrentVersion.DisplayName;
        internal const string/*!*/ IronPythonNames = "IronPython;Python;py";
        internal const string/*!*/ IronPythonFileExtensions = ".py";

        private static readonly Guid PythonLanguageGuid = new Guid("03ed4b80-d10b-442f-ad9a-47dae85b2051");
        private static readonly Guid LanguageVendor_Microsoft = new Guid(-1723120188, -6423, 0x11d2, 0x90, 0x3f, 0, 0xc0, 0x4f, 0xa3, 2, 0xa1);

        // fields used during startup
        private readonly IDictionary<object, object>/*!*/ _modulesDict = new PythonDictionary();
        private readonly Dictionary<string, ModuleGlobalCache>/*!*/ _builtinCache = new Dictionary<string, ModuleGlobalCache>(StringComparer.Ordinal);
        private readonly Dictionary<Type, string>/*!*/ _builtinModuleNames = new Dictionary<Type, string>();
        private readonly PythonOptions/*!*/ _options;
        private readonly PythonModule/*!*/ _systemState;
        private readonly Dictionary<string, Type>/*!*/ _builtinModulesDict;
        private readonly PythonOverloadResolverFactory _sharedOverloadResolverFactory;
        private readonly PythonBinder _binder;
        private readonly SysModuleDictionaryStorage _sysDict = new SysModuleDictionaryStorage();
#if FEATURE_FILESYSTEM
#if FEATURE_ASSEMBLY_RESOLVE
        private readonly AssemblyResolveHolder _resolveHolder;
#endif
#if !CLR2
        private readonly HashSet<Assembly> _loadedAssemblies = new HashSet<Assembly>();
#endif
#endif
        private Encoding _defaultEncoding = PythonAsciiEncoding.Instance;

        // conditional variables for silverlight/desktop CLR features
        private Hosting.PythonService _pythonService;
        private string _initialExecutable, _initialPrefix = GetInitialPrefix();

        // other fields which might only be conditionally used
        private string _initialVersionString;
        private PythonModule _clrModule;
        private PythonDictionary _builtinDict;
        private PythonModule _builtins;

        private PythonFileManager _fileManager;
        private Dictionary<string, object> _errorHandlers;
        private List<object> _searchFunctions;
        private Dictionary<object, object> _moduleState;
        /// <summary> stored for copy_reg module, used for reduce protocol </summary>
        internal BuiltinFunction NewObject;
        /// <summary> stored for copy_reg module, used for reduce protocol </summary>
        internal BuiltinFunction PythonReconstructor;
        private Dictionary<Type, object> _genericSiteStorage;

        private CallSite<Func<CallSite, CodeContext, object, object>>[] _newUnarySites;
        private CallSite<Func<CallSite, CodeContext, object, object, object, object>>[] _newTernarySites;

        private CallSite<Func<CallSite, object, object, int>> _compareSite;
        private Dictionary<AttrKey, CallSite<Func<CallSite, object, object, object>>> _setAttrSites;
        private Dictionary<AttrKey, CallSite<Action<CallSite, object>>> _deleteAttrSites;
        private CallSite<Func<CallSite, CodeContext, object, string, PythonTuple, PythonDictionary, object>> _metaClassSite;
        private CallSite<Func<CallSite, CodeContext, object, string, object>> _writeSite;
        private CallSite<Func<CallSite, object, object, object>> _getIndexSite, _equalSite;
        private CallSite<Action<CallSite, object, object>> _delIndexSite;
        private CallSite<Func<CallSite, CodeContext, object, object>> _finalizerSite;
        private CallSite<Func<CallSite, CodeContext, PythonFunction, object>> _functionCallSite;
        private CallSite<Func<CallSite, object, object, bool>> _greaterThanSite, _lessThanSite, _greaterThanEqualSite, _lessThanEqualSite, _containsSite;
        private CallSite<Func<CallSite, CodeContext, object, object[], object>> _callSplatSite;
        private CallSite<Func<CallSite, CodeContext, object, object[], IDictionary<object, object>, object>> _callDictSite;
        private CallSite<Func<CallSite, CodeContext, object, object, object, object>> _callDictSiteLooselyTyped;
        private CallSite<Func<CallSite, CodeContext, object, string, PythonDictionary, PythonDictionary, PythonTuple, int, object>> _importSite;
        private CallSite<Func<CallSite, CodeContext, object, string, PythonDictionary, PythonDictionary, PythonTuple, object>> _oldImportSite;
        private CallSite<Func<CallSite, object, bool>> _isCallableSite;
        private CallSite<Func<CallSite, object, IList<string>>> _getSignaturesSite;
        private CallSite<Func<CallSite, object, object, object>> _addSite, _divModSite, _rdivModSite;
        private CallSite<Func<CallSite, object, object, object, object>> _setIndexSite, _delSliceSite;
        private CallSite<Func<CallSite, object, object, object, object, object>> _setSliceSite;
        private CallSite<Func<CallSite, object, string>> _docSite;

        // conversion sites
        private CallSite<Func<CallSite, object, int>> _intSite;
        private CallSite<Func<CallSite, object, string>> _tryStringSite;
        private CallSite<Func<CallSite, object, object>> _tryIntSite;
        private CallSite<Func<CallSite, object, IEnumerable>> _tryIEnumerableSite;
        private Dictionary<Type, CallSite<Func<CallSite, object, object>>> _implicitConvertSites;
        private Dictionary<PythonOperationKind, CallSite<Func<CallSite, object, object, object>>> _binarySites;
        private Dictionary<Type, DefaultPythonComparer> _defaultComparer;
        private CallSite<Func<CallSite, CodeContext, object, object, object, int>> _sharedFunctionCompareSite;
        private CallSite<Func<CallSite, CodeContext, PythonFunction, object, object, int>> _sharedPythonFunctionCompareSite;
        private CallSite<Func<CallSite, CodeContext, BuiltinFunction, object, object, int>> _sharedBuiltinFunctionCompareSite;
        private CallSite<Func<CallSite, CodeContext, object, int, object>> _getItemCallSite;

        private CallSite<Func<CallSite, CodeContext, object, object, object>> _propGetSite, _propDelSite;
        private CallSite<Func<CallSite, CodeContext, object, object, object, object>> _propSetSite;
        private CompiledLoader _compiledLoader;
        internal bool _importWarningThrows;
        private bool _importedEncodings;
        private Action<Action> _commandDispatcher; // can be null
        private ClrModule.ReferencesList _referencesList;
        private FloatFormat _floatFormat, _doubleFormat;
        private CultureInfo _collateCulture, _ctypeCulture, _timeCulture, _monetaryCulture, _numericCulture;
        private CodeContext _defaultContext, _defaultClsContext;
        private readonly TopNamespaceTracker _topNamespace;
        private readonly IEqualityComparer<object> _equalityComparer;
        private readonly IEqualityComparer _equalityComparerNonGeneric;

        private Dictionary<Type, CallSite<Func<CallSite, object, object, bool>>> _equalSites;

        private Dictionary<Type, PythonSiteCache> _systemSiteCache;
        internal static object _syntaxErrorNoCaret = new object();

        // atomized binders
        private PythonInvokeBinder _invokeNoArgs, _invokeOneArg;
        private Dictionary<CallSignature, PythonInvokeBinder/*!*/> _invokeBinders;
        private Dictionary<string/*!*/, PythonGetMemberBinder/*!*/> _getMemberBinders;
        private Dictionary<string/*!*/, PythonGetMemberBinder/*!*/> _tryGetMemberBinders;
        private Dictionary<string/*!*/, PythonSetMemberBinder/*!*/> _setMemberBinders;
        private Dictionary<string/*!*/, PythonDeleteMemberBinder/*!*/> _deleteMemberBinders;
        private Dictionary<string/*!*/, CompatibilityGetMember/*!*/> _compatGetMember;
        private Dictionary<string/*!*/, CompatibilityGetMember/*!*/> _compatGetMemberNoThrow;
        private Dictionary<PythonOperationKind, PythonOperationBinder/*!*/> _operationBinders;
        private Dictionary<ExpressionType, PythonUnaryOperationBinder/*!*/> _unaryBinders;
        private PythonBinaryOperationBinder[] _binaryBinders;
        private Dictionary<OperationRetTypeKey<ExpressionType>, BinaryRetTypeBinder/*!*/> _binaryRetTypeBinders;
        private Dictionary<OperationRetTypeKey<PythonOperationKind>, BinaryRetTypeBinder/*!*/> _operationRetTypeBinders;
        private Dictionary<Type/*!*/, PythonConversionBinder/*!*/>[] _conversionBinders;
        private Dictionary<Type/*!*/, DynamicMetaObjectBinder/*!*/>[] _convertRetObjectBinders;
        private Dictionary<CallSignature, CreateFallback/*!*/> _createBinders;
        private Dictionary<CallSignature, CompatibilityInvokeBinder/*!*/> _compatInvokeBinders;
        private PythonGetSliceBinder _getSlice;
        private PythonSetSliceBinder _setSlice;
        private PythonDeleteSliceBinder _deleteSlice;
        private PythonGetIndexBinder[] _getIndexBinders;
        private PythonSetIndexBinder[] _setIndexBinders;
        private PythonDeleteIndexBinder[] _deleteIndexBinders;
        private DynamicMetaObjectBinder _invokeTwoConvertToInt;
        private static CultureInfo _CCulture;
        private DynamicDelegateCreator _delegateCreator;

        // tracing / in-proc debugging support
        private DebugContext _debugContext;
        private Debugging.TracePipeline _tracePipeline;
        private readonly Microsoft.Scripting.Utils.ThreadLocal<PythonTracebackListener> _tracebackListeners = new Microsoft.Scripting.Utils.ThreadLocal<PythonTracebackListener>();
        private int _tracebackListenersCount;

        internal FunctionCode.CodeList _allCodes;
        internal readonly object _codeCleanupLock = new object(), _codeUpdateLock = new object();
        internal int _codeCount, _nextCodeCleanup = 200;
        private int _recursionLimit;

        internal readonly List<FunctionStack> _mainThreadFunctionStack;
        private CallSite<Func<CallSite, CodeContext, object, object>> _callSite0LightEh;
        private List<WeakReference> _weakExtensionMethodSets;

        // store the Python types mapping to each .NET type
        private CommonDictionaryStorage _systemPythonTypesWeakRefs = new CommonDictionaryStorage();

#region Generated Python Shared Call Sites Storage

        // *** BEGIN GENERATED CODE ***
        // generated by function: gen_shared_call_sites_storage from: generate_calls.py

        private CallSite<Func<CallSite, CodeContext, object, object>> _callSite0;
        private CallSite<Func<CallSite, CodeContext, object, object, object>> _callSite1;
        private CallSite<Func<CallSite, CodeContext, object, object, object, object>> _callSite2;
        private CallSite<Func<CallSite, CodeContext, object, object, object, object, object>> _callSite3;
        private CallSite<Func<CallSite, CodeContext, object, object, object, object, object, object>> _callSite4;
        private CallSite<Func<CallSite, CodeContext, object, object, object, object, object, object, object>> _callSite5;
        private CallSite<Func<CallSite, CodeContext, object, object, object, object, object, object, object, object>> _callSite6;

        // *** END GENERATED CODE ***

#endregion

        /// <summary>
        /// Creates a new PythonContext not bound to Engine.
        /// </summary>
        public PythonContext(ScriptDomainManager/*!*/ manager, IDictionary<string, object> options)
            : base(manager) {
            _options = new PythonOptions(options);
            _builtinModulesDict = CreateBuiltinTable();

            PythonDictionary defaultScope = new PythonDictionary();
            ModuleContext modContext = new ModuleContext(defaultScope, this);
            _defaultContext = modContext.GlobalContext;
            
            PythonDictionary sysDict = new PythonDictionary(_sysDict);
            _systemState = new PythonModule(sysDict);
            _systemState.__dict__["__name__"] = "sys";
            _systemState.__dict__["__package__"] = null;

            PythonBinder binder = new PythonBinder(this, _defaultContext);
            _sharedOverloadResolverFactory = new PythonOverloadResolverFactory(binder, Expression.Constant(_defaultContext));
            _binder = binder;

            CodeContext defaultClsContext = DefaultContext.CreateDefaultCLSContext(this);
            _defaultClsContext = defaultClsContext;

            if (DefaultContext._default == null) {
                DefaultContext.InitializeDefaults(_defaultContext, defaultClsContext);
            }

            InitializeBuiltins();

            InitializeSystemState();
#if SILVERLIGHT
            AddToPath("");
#endif

            // sys.argv always includes at least one empty string.
            SetSystemStateValue("argv", (_options.Arguments.Count == 0) ?
                new List(new object[] { String.Empty }) :
                new List(_options.Arguments)
            );

            if (_options.WarningFilters.Count > 0) {
                _systemState.__dict__["warnoptions"] = new List(_options.WarningFilters);
            }

            if (_options.Frames) {
                var getFrame = BuiltinFunction.MakeFunction(
                    "_getframe", 
                    ArrayUtils.ConvertAll(typeof(SysModule).GetMember("_getframeImpl"), (x) => (MethodBase)x), 
                    typeof(SysModule)
                );
                _systemState.__dict__["_getframe"] = getFrame;
            }

            if (_options.Tracing) {
                EnsureDebugContext();
            }

            List path = new List(_options.SearchPaths);
#if FEATURE_ASSEMBLY_RESOLVE && FEATURE_FILESYSTEM
            _resolveHolder = new AssemblyResolveHolder(this);
            try {
                Assembly entryAssembly = Assembly.GetEntryAssembly();
                // Can be null if called from unmanaged code (VS integration scenario)
                if (entryAssembly != null) {
                    string entry = Path.GetDirectoryName(entryAssembly.Location);
                    string lib = Path.Combine(entry, "Lib");
                    path.append(lib);

                    // add DLLs directory for user-defined extention modules
                    path.append(Path.Combine(entry, "DLLs"));

#if DEBUG
                    // For developer use, add External.LCA_RESTRICTED/Languages/IronPython/27/Lib
                    string devStdLib = Path.GetFullPath(Path.Combine(entry, @"../../External.LCA_RESTRICTED/Languages/IronPython/27/Lib"));
                    if (Directory.Exists(devStdLib))
                        path.append(devStdLib);
#endif
                }
            } catch (SecurityException) {
            }
#endif

            _systemState.__dict__["path"] = path;

            RecursionLimit = _options.RecursionLimit;

#if FEATURE_ASSEMBLY_RESOLVE && FEATURE_FILESYSTEM
            object asmResolve;
            if (options == null ||
                !options.TryGetValue("NoAssemblyResolveHook", out asmResolve) ||
                !System.Convert.ToBoolean(asmResolve)) {
                try {
                    HookAssemblyResolve();
                } catch (System.Security.SecurityException) {
                    // We may not have SecurityPermissionFlag.ControlAppDomain. 
                    // If so, we will not look up sys.path for module loads
                }
            }
#endif
            _equalityComparer = new PythonEqualityComparer(this);
            _equalityComparerNonGeneric = (IEqualityComparer)_equalityComparer;

            InitialHasher = InitialHasherImpl;
            IntHasher = IntHasherImpl;
            DoubleHasher = DoubleHasherImpl;
            StringHasher = StringHasherImpl;
            FallbackHasher = FallbackHasherImpl;

            _topNamespace = new TopNamespaceTracker(manager);
            foreach (Assembly asm in manager.GetLoadedAssemblyList()) {
                _topNamespace.LoadAssembly(asm);
            }
            manager.AssemblyLoaded += new EventHandler<AssemblyLoadedEventArgs>(ManagerAssemblyLoaded);

            _mainThreadFunctionStack = PythonOps.GetFunctionStack();
        }

        void ManagerAssemblyLoaded(object sender, AssemblyLoadedEventArgs e) {
            _topNamespace.LoadAssembly(e.Assembly);
        }

        /// <summary>
        /// Gets or sets the maximum depth of function calls.  Equivalent to sys.getrecursionlimit
        /// and sys.setrecursionlimit.
        /// </summary>
        public int RecursionLimit {
            get {
                return _recursionLimit;
            }
            set {
                if (value < 0) {
                    throw PythonOps.ValueError("recursion limit must be positive");
                }

                lock (_codeUpdateLock) {
                    _recursionLimit = value;

                    if ((_recursionLimit == Int32.MaxValue) != (value == Int32.MaxValue)) {
                        // recursion setting has changed, we need to update all of our
                        // function codes to enforce or un-enforce recursion.
                        FunctionCode.UpdateAllCode(this);
                    }
                }
            }
        }

        internal bool EnableTracing {
            get {
                return PythonOptions.Tracing || _tracebackListenersCount > 0;
            }
        }

        internal TopNamespaceTracker TopNamespace {
            get {
                return _topNamespace;
            }
        }

#if FEATURE_THREAD
        /// <summary>
        /// Gets or sets the main thread which should be interupted by thread.interrupt_main
        /// </summary>
        public Thread MainThread {
            get {
                return _mainThread;
            }
            set {
                _mainThread = value;
            }
        }

        private Thread _mainThread;
#endif

        public IEqualityComparer<object>/*!*/ EqualityComparer {
            get { return _equalityComparer; }
        }

        public IEqualityComparer/*!*/ EqualityComparerNonGeneric {
            get { return _equalityComparerNonGeneric; }
        }

        internal sealed class PythonEqualityComparer : IEqualityComparer, IEqualityComparer<object> {
            public readonly PythonContext/*!*/ Context;

            public PythonEqualityComparer(PythonContext/*!*/ context) {
                Assert.NotNull(context);
                Context = context;
            }

            bool IEqualityComparer.Equals(object x, object y) {
                return PythonOps.EqualRetBool(Context._defaultContext, x, y);
            }

            bool IEqualityComparer<object>.Equals(object x, object y) {
                return PythonOps.EqualRetBool(Context._defaultContext, x, y);
            }

            int IEqualityComparer.GetHashCode(object obj) {
                return PythonContext.Hash(obj);
            }

            int IEqualityComparer<object>.GetHashCode(object obj) {
                return PythonContext.Hash(obj);
            }
        }

#region Specialized Hashers

        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes")]
        internal readonly HashDelegate InitialHasher;
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes")]
        internal readonly HashDelegate IntHasher;
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes")]
        internal readonly HashDelegate DoubleHasher;
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes")]
        internal readonly HashDelegate StringHasher;
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes")]
        internal readonly HashDelegate FallbackHasher;

        private int InitialHasherImpl(object o, ref HashDelegate dlg) {
            if (o == null) {
                return NoneTypeOps.NoneHashCode;
            }

            switch (o.GetType().GetTypeCode()) {
                case TypeCode.String:
                    dlg = StringHasher;
                    return StringHasher(o, ref dlg);
                case TypeCode.Int32:
                    dlg = IntHasher;
                    return IntHasher(o, ref dlg);
                case TypeCode.Double:
                    dlg = DoubleHasher;
                    return DoubleHasher(o, ref dlg);
                default:
                    if (o is IPythonObject) {
                        dlg = new OptimizedUserHasher(this, ((IPythonObject)o).PythonType).Hasher;
                    } else {
                        dlg = new OptimizedBuiltinHasher(this, o.GetType()).Hasher;
                    }

                    return dlg(o, ref dlg);
            }
        }

        private int IntHasherImpl(object o, ref HashDelegate dlg) {
            if (o != null && o.GetType() == typeof(int)) {
                return o.GetHashCode();
            }

            dlg = FallbackHasher;
            return FallbackHasher(o, ref dlg);
        }

        private int DoubleHasherImpl(object o, ref HashDelegate dlg) {
            if (o != null && o.GetType() == typeof(double)) {
                return DoubleOps.__hash__((double)o);
            }

            dlg = FallbackHasher;
            return FallbackHasher(o, ref dlg);
        }

        private int StringHasherImpl(object o, ref HashDelegate dlg) {
            if (o != null && o.GetType() == typeof(string)) {
                return o.GetHashCode();
            }

            dlg = FallbackHasher;
            return FallbackHasher(o, ref dlg);
        }

        private int FallbackHasherImpl(object o, ref HashDelegate dlg) {
            return PythonOps.Hash(this.SharedContext, o);
        }

        private sealed class OptimizedUserHasher {
            private readonly PythonContext _context;
            private readonly PythonType _pt;

            public OptimizedUserHasher(PythonContext context, PythonType pt) {
                _context = context;
                _pt = pt;
            }

            public int Hasher(object o, ref HashDelegate dlg) {
                IPythonObject ipo = o as IPythonObject;
                if (ipo != null && ipo.PythonType == _pt) {
                    return _pt.Hash(o);
                }

                dlg = _context.FallbackHasher;
                return _context.FallbackHasher(o, ref dlg);
            }
        }

        private sealed class OptimizedBuiltinHasher {
            private readonly PythonContext _context;
            private readonly Type _type;
            private readonly PythonType _pt;

            public OptimizedBuiltinHasher(PythonContext context, Type type) {
                _context = context;
                _type = type;
                _pt = DynamicHelpers.GetPythonTypeFromType(type);
            }

            public int Hasher(object o, ref HashDelegate dlg) {
                if (o != null && o.GetType() == _type) {
                    return _pt.Hash(o);
                }

                dlg = _context.FallbackHasher;
                return _context.FallbackHasher(o, ref dlg);
            }
        }

#endregion

        public override LanguageOptions/*!*/ Options {
            get { return PythonOptions; }
        }

        /// <summary>
        /// Checks to see if module state has the current value stored already.
        /// </summary>
        public bool HasModuleState(object key) {
            EnsureModuleState();

            lock (_moduleState) {
                return _moduleState.ContainsKey(key);
            }
        }

        private void EnsureModuleState() {
            if (_moduleState == null) {
                Interlocked.CompareExchange(ref _moduleState, new Dictionary<object, object>(), null);
            }
        }

        /// <summary>
        /// Gets per-runtime state used by a module.  The module should have a unique key for
        /// each piece of state it needs to store.
        /// </summary>
        public object GetModuleState(object key) {
            EnsureModuleState();

            lock (_moduleState) {
                Debug.Assert(_moduleState.ContainsKey(key));

                return _moduleState[key];
            }
        }

        /// <summary>
        /// Sets per-runtime state used by a module.  The module should have a unique key for
        /// each piece of state it needs to store.
        /// </summary>
        public void SetModuleState(object key, object value) {
            EnsureModuleState();

            lock (_moduleState) {
                _moduleState[key] = value;
            }
        }

        /// <summary>
        /// Sets per-runtime state used by a module and returns the previous value.  The module
        /// should have a unique key for each piece of state it needs to store.
        /// </summary>
        public object GetSetModuleState(object key, object value) {
            EnsureModuleState();

            lock (_moduleState) {
                object result;
                _moduleState.TryGetValue(key, out result);
                _moduleState[key] = value;
                return result;
            }
        }

        /// <summary>
        /// Sets per-runtime state used by a module and returns the previous value.  The module
        /// should have a unique key for each piece of state it needs to store.
        /// </summary>
        public T GetOrCreateModuleState<T>(object key, Func<T> value) where T : class {
            EnsureModuleState();

            lock (_moduleState) {
                object result;
                if (!_moduleState.TryGetValue(key, out result)) {
                    _moduleState[key] = result = value();
                }
                return (result as T);
            }
        }

        public PythonType EnsureModuleException(object key, PythonDictionary dict, string name, string module) {
            return (PythonType)(dict[name] = GetOrCreateModuleState(
                key,
                () => PythonExceptions.CreateSubType(this, PythonExceptions.Exception, name, module, null, PythonType.DefaultMakeException)
            ));
        }

        public PythonType EnsureModuleException(object key, PythonType baseType, PythonDictionary dict, string name, string module) {
            return (PythonType)(dict[name] = GetOrCreateModuleState(
                key,
                () => PythonExceptions.CreateSubType(this, baseType, name, module, null, PythonType.DefaultMakeException)
            ));
        }

        public PythonType EnsureModuleException(object key, PythonType baseType, Type underlyingType, PythonDictionary dict, string name, string module, Func<string, Exception> exceptionMaker) {
            return (PythonType)(dict[name] = GetOrCreateModuleState(
                key,
                () => PythonExceptions.CreateSubType(this, baseType, underlyingType, name, module, null, exceptionMaker)
            ));
        }

        public PythonType EnsureModuleException(object key, PythonType[] baseTypes, Type underlyingType, PythonDictionary dict, string name, string module) {
            return (PythonType)(dict[name] = GetOrCreateModuleState(
                key,
                () => PythonExceptions.CreateSubType(this, baseTypes, underlyingType, name, module, null, PythonType.DefaultMakeException)
            ));
        }

        internal PythonOptions/*!*/ PythonOptions {
            get {
                return _options;
            }
        }

        public override Guid VendorGuid {
            get {
                return LanguageVendor_Microsoft;
            }
        }

        public override Guid LanguageGuid {
            get {
                return PythonLanguageGuid;
            }
        }

        public PythonModule/*!*/ SystemState {
            get {
                return _systemState;
            }
        }

        public PythonModule/*!*/ ClrModule {
            get {
                if (_clrModule == null) {
                    Interlocked.CompareExchange(ref _clrModule, CreateBuiltinModule("clr"), null);
                }

                return _clrModule;
            }
        }

        internal bool TryGetSystemPath(out List path) {
            object val;
            if (SystemState.__dict__.TryGetValue("path", out val)) {
                path = val as List;
            } else {
                path = null;
            }

            return path != null;
        }

        internal object SystemStandardOut {
            get {
                return GetSystemStateValue("stdout");
            }
        }

        internal object SystemStandardIn {
            get {
                return GetSystemStateValue("stdin");
            }
        }

        internal object SystemStandardError {
            get {
                return GetSystemStateValue("stderr");
            }
        }

        internal IDictionary<object, object> SystemStateModules {
            get {
                return _modulesDict;
            }
        }

        internal void UpdateExceptionInfo(object type, object value, object traceback) {
            _sysDict.UpdateExceptionInfo(type, value, traceback);
        }

        internal void UpdateExceptionInfo(Exception clrException, object type, object value, List<DynamicStackFrame> traceback) {
            _sysDict.UpdateExceptionInfo(clrException, type, value, traceback);            
        }

        internal void ExceptionHandled() {
            _sysDict.ExceptionHandled();
        }

        internal PythonModule GetModuleByName(string/*!*/ name) {
            Assert.NotNull(name);
            object scopeObj;
            PythonModule module;
            if (SystemStateModules.TryGetValue(name, out scopeObj) && (module = scopeObj as PythonModule) != null) {
                return module;
            }
            return null;
        }

        internal PythonModule GetModuleByPath(string/*!*/ path) {
            Assert.NotNull(path);
            foreach (object moduleObj in SystemStateModules.Values) {
                PythonModule module = moduleObj as PythonModule;
                if (module != null) {
                    if (DomainManager.Platform.PathComparer.Compare(module.GetFile(), path) == 0) {
                        return module;
                    }
                }
            }
            return null;
        }

        public override Version LanguageVersion {
            get {
                // Assembly.GetName() can't be called in Silverlight...
                return GetPythonVersion();
            }
        }

        internal static Version GetPythonVersion() {
            return new AssemblyName(typeof(PythonContext).GetTypeInfo().Assembly.FullName).Version;
        }

        internal FloatFormat FloatFormat {
            get {
                return _floatFormat;
            }
            set {
                _floatFormat = value;
            }
        }

        internal FloatFormat DoubleFormat {
            get {
                return _doubleFormat;
            }
            set {
                _doubleFormat = value;
            }
        }

        /// <summary>
        /// Initializes the sys module on startup.  Called both to load and reload sys
        /// </summary>
        private void InitializeSystemState() {
            // These fields do not get reset on "reload(sys)", we populate them once on startup
            SetSystemStateValue("argv", List.FromArrayNoCopy(new object[] { String.Empty }));
            SetSystemStateValue("modules", _modulesDict);
            InitializeSysFlags();

            _modulesDict["sys"] = _systemState;

            SetSystemStateValue("path", new List(3));

            SetStandardIO();

            SysModule.PerformModuleReload(this, _systemState.__dict__);
        }

        internal bool EmitDebugSymbols(SourceUnit sourceUnit) {
            return sourceUnit.EmitDebugSymbols && (PythonOptions.NoDebug == null || !PythonOptions.NoDebug.IsMatch(sourceUnit.Path));
        }

        private void InitializeSysFlags() {
            // sys.flags
            SysModule.SysFlags flags = new SysModule.SysFlags();
            SetSystemStateValue("flags", flags);
            flags.debug = _options.Debug ? 1 : 0;
            flags.py3k_warning = _options.WarnPython30 ? 1 : 0;
            SetSystemStateValue("py3kwarning", _options.WarnPython30);
            switch (_options.DivisionOptions) {
                case PythonDivisionOptions.Old:
                    break;
                case PythonDivisionOptions.New:
                    flags.division_new = 1;
                    break;
                case PythonDivisionOptions.Warn:
                    flags.division_warning = 1;
                    break;
                case PythonDivisionOptions.WarnAll:
                    flags.division_warning = 2;
                    break;
            }
            flags.inspect = flags.interactive = _options.Inspect ? 1 : 0;
            if (_options.StripDocStrings) {
                flags.optimize = 2;
            } else if (_options.Optimize) {
                flags.optimize = 1;
            }
            flags.dont_write_bytecode = 1;
            SetSystemStateValue("dont_write_bytecode", true);
            flags.no_user_site = _options.NoUserSite ? 1 : 0;
            flags.no_site = _options.NoSite ? 1 : 0;
            flags.ignore_environment = _options.IgnoreEnvironment ? 1 : 0;
            switch (_options.IndentationInconsistencySeverity) {
                case Severity.Warning:
                    flags.tabcheck = 1;
                    break;
                case Severity.Error:
                    flags.tabcheck = 2;
                    break;
            }
            flags.verbose = _options.Verbose ? 1 : 0;
            flags.unicode = 1;
            flags.bytes_warning = _options.BytesWarning ? 1 : 0;
        }

        internal bool ShouldInterpret(PythonCompilerOptions options, SourceUnit source) {
            // We have to turn off adaptive compilation in debug mode to
            // support mangaged debuggers. Also turn off in optimized mode.
            bool adaptiveCompilation = !_options.NoAdaptiveCompilation && !EmitDebugSymbols(source);

            return options.Interpreted || adaptiveCompilation;
        }

        private static PyAst.PythonAst ParseAndBindAst(CompilerContext context) {
            ScriptCodeParseResult properties = ScriptCodeParseResult.Complete;
            bool propertiesSet = false;
            int errorCode = 0;

            PyAst.PythonAst ast;
            using (Parser parser = Parser.CreateParser(context, PythonContext.GetPythonOptions(null))) {
                switch (context.SourceUnit.Kind) {
                    case SourceCodeKind.InteractiveCode:
                        ast = parser.ParseInteractiveCode(out properties);
                        propertiesSet = true;
                        break;

                    case SourceCodeKind.Expression:
                        ast = parser.ParseTopExpression();
                        break;

                    case SourceCodeKind.SingleStatement:
                        ast = parser.ParseSingleStatement();
                        break;

                    case SourceCodeKind.File:
                        ast = parser.ParseFile(true, false);
                        break;

                    case SourceCodeKind.Statements:
                        ast = parser.ParseFile(false, false);
                        break;

                    default:
                    case SourceCodeKind.AutoDetect:
                        ast = parser.ParseFile(true, true);
                        break;
                }

                errorCode = parser.ErrorCode;
            }

            if (!propertiesSet && errorCode != 0) {
                properties = ScriptCodeParseResult.Invalid;
            }

            context.SourceUnit.CodeProperties = properties;

            if (errorCode != 0 || properties == ScriptCodeParseResult.Empty) {
                return null;
            }

            ast.Bind();
            return ast;
        }

        internal static ScriptCode CompilePythonCode(SourceUnit/*!*/ sourceUnit, CompilerOptions/*!*/ options, ErrorSink/*!*/ errorSink) {
            var pythonOptions = (PythonCompilerOptions)options;

            if (sourceUnit.Kind == SourceCodeKind.File) {
                pythonOptions.Module |= ModuleOptions.Initialize;
            }

            CompilerContext context = new CompilerContext(sourceUnit, options, errorSink);

            PyAst.PythonAst ast = ParseAndBindAst(context);
            if (ast == null) {
                return null;
            }
            
            return ast.ToScriptCode();
        }

        public override ScriptCode CompileSourceCode(SourceUnit/*!*/ sourceUnit, CompilerOptions/*!*/ options, ErrorSink/*!*/ errorSink) {            
            ScriptCode res = CompilePythonCode(sourceUnit, options, errorSink);
            if (res != null) {
                Scope scope = res.CreateScope();

                // if this is an optimized module we need to initialize the optimized scope.
                // Optimized scopes come w/ extensions already attached so we use that to know
                // if we're optimized or not.
                PythonScopeExtension scopeExtension = (PythonScopeExtension)scope.GetExtension(ContextId);
                if (scopeExtension != null) {
                    InitializeModule(sourceUnit.Path, scopeExtension.ModuleContext, res, ModuleOptions.None);
                }
            }

            return res;
        }

        public override ScriptCode/*!*/ LoadCompiledCode(Delegate/*!*/ method, string path, string customData) {
            SourceUnit su = new SourceUnit(this, NullTextContentProvider.Null, path, SourceCodeKind.File);
            return new OnDiskScriptCode((LookupCompilationDelegate)method, su, customData);
        }

        public override SourceCodeReader/*!*/ GetSourceReader(Stream/*!*/ stream, Encoding/*!*/ defaultEncoding, string path) {
            ContractUtils.RequiresNotNull(stream, "stream");
            ContractUtils.RequiresNotNull(defaultEncoding, "defaultEncoding");
            ContractUtils.Requires(stream.CanSeek && stream.CanRead, "stream", "The stream must support seeking and reading");

            // we choose ASCII by default, if the file has a Unicode header though
            // we'll automatically get it as unicode.
            Encoding encoding = PythonAsciiEncoding.SourceEncoding;

            long startPosition = stream.Position;

            StreamReader sr = new StreamReader(stream, PythonAsciiEncoding.SourceEncoding);
            byte[] bomBuffer = new byte[3];
            int bomRead = stream.Read(bomBuffer, 0, 3);
            int bytesRead = 0;
            bool isUtf8 = false;
            if (bomRead == 3 && (bomBuffer[0] == 0xef && bomBuffer[1] == 0xbb && bomBuffer[2] == 0xbf)) {
                isUtf8 = true;
                bytesRead = 3;
            } else {
                stream.Seek(0, SeekOrigin.Begin);
            }

            string line;
            try {
                line = ReadOneLine(sr, ref bytesRead);
            } catch (BadSourceException) {
                throw ReportEncodingError(stream, path);                
            }

            bool gotEncoding = false;
            string encodingName = null;
            // magic encoding must be on line 1 or 2
            if (line != null && !(gotEncoding = Tokenizer.TryGetEncoding(defaultEncoding, line, ref encoding, out encodingName))) {
                try {
                    line = ReadOneLine(sr, ref bytesRead);
                } catch (BadSourceException) {
                    throw ReportEncodingError(stream, path);
                }

                if (line != null) {
                    gotEncoding = Tokenizer.TryGetEncoding(defaultEncoding, line, ref encoding, out encodingName);
                }
            }

            if (gotEncoding && isUtf8 && encodingName != "utf-8") {
                // we have both a BOM & an encoding type, throw an error
                throw new IOException("file has both Unicode marker and PEP-263 file encoding.  You can only use \"utf-8\" as the encoding name when a BOM is present.");
            } else if (encoding == null) {
                throw new IOException("unknown encoding type");
            }

            // if we didn't get an encoding seek back to the beginning...
            if (!gotEncoding || stream.Position != stream.Length) {
                stream.Seek(startPosition, SeekOrigin.Begin);
            }

            // re-read w/ the correct encoding type...
            return new SourceCodeReader(new StreamReader(stream, encoding), encoding);
        }

        internal static Exception ReportEncodingError(Stream stream, string path) {
            stream.Seek(0, SeekOrigin.Begin);
            byte[] buffer = new byte[1024];
            int bytesRead = 0;
            int curLine = 1, curOffset = 1, index = 0;
            while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) != -1) {
                for (int i = 0; i < bytesRead; i++) {
                    if (buffer[i] > 0x7f) {
                        return PythonOps.BadSourceError(
                            buffer[i], 
                            new SourceSpan(
                                new SourceLocation(index, curLine, curOffset),
                                new SourceLocation(index, curLine, curOffset)
                            ), 
                            path
                        );
                    } else if (buffer[i] == '\n') {
                        curLine++;
                        curOffset = 1;
                    } else {
                        curOffset++;
                    }

                    index++;
                }
            }

            return new InvalidOperationException();
        }

        /// <summary>
        /// Reads one line keeping track of the # of bytes read
        /// </summary>
        private static string ReadOneLine(StreamReader reader, ref int totalRead) {
            Stream sr = reader.BaseStream;
            byte[] buffer = new byte[256];
            StringBuilder builder = null;
            
            int bytesRead = sr.Read(buffer, 0, buffer.Length);

            while (bytesRead > 0) {
                totalRead += bytesRead;

                bool foundEnd = false;
                for (int i = 0; i < bytesRead; i++) {
                    if (buffer[i] == '\r') {
                        if (i + 1 < bytesRead) {
                            if (buffer[i + 1] == '\n') {
                                totalRead -= (bytesRead - (i + 2));   // skip cr/lf
                                sr.Seek(i + 2, SeekOrigin.Begin);
                                reader.DiscardBufferedData();
                                foundEnd = true;
                            }
                        } else {
                            totalRead -= (bytesRead - (i + 1)); // skip cr
                            sr.Seek(i + 1, SeekOrigin.Begin);
                            reader.DiscardBufferedData();
                            foundEnd = true;
                        }
                    } else if (buffer[i] == '\n') {
                        totalRead -= (bytesRead - (i + 1)); // skip lf
                        sr.Seek(i + 1, SeekOrigin.Begin);
                        reader.DiscardBufferedData();
                        foundEnd = true;
                    }

                    if (foundEnd) {
                        if (builder != null) {
                            builder.Append(buffer.MakeString(), 0, i);
                            return builder.ToString();
                        }
                        return buffer.MakeString().Substring(0, i);
                    }
                }

                if (builder == null) builder = new StringBuilder();
                builder.Append(buffer.MakeString(), 0, bytesRead);
                bytesRead = sr.Read(buffer, 0, buffer.Length);
            }

            // no string
            if (builder == null) {
                return null;
            }

            // no new-line
            return builder.ToString();
        }

#if FEATURE_CODEDOM
        // Convert a CodeDom to source code, and output the generated code and the line number mappings (if any)
        public override SourceUnit/*!*/ GenerateSourceCode(System.CodeDom.CodeObject codeDom, string path, SourceCodeKind kind) {
            return new IronPython.Hosting.PythonCodeDomCodeGen().GenerateCode((System.CodeDom.CodeMemberMethod)codeDom, this, path, kind);
        }
#endif

#region Scopes

        public override Scope GetScope(string/*!*/ path) {
            PythonModule module = GetModuleByPath(path);
            return (module != null) ? module.Scope : null;
        }

        public PythonModule/*!*/ InitializeModule(string fileName, ModuleContext moduleContext, ScriptCode scriptCode, ModuleOptions options) {
            if ((options & ModuleOptions.NoBuiltins) == 0) {
                moduleContext.InitializeBuiltins((options & ModuleOptions.ModuleBuiltins) != 0);
            }

            // If the filename is __init__.py then this is the initialization code
            // for a package and we need to set the __path__ variable appropriately
            if (fileName != null && Path.GetFileName(fileName) == "__init__.py") {
                string dirname = Path.GetDirectoryName(fileName);
                string dir_path = DomainManager.Platform.GetFullPath(dirname);
                moduleContext.Globals["__path__"] = PythonOps.MakeList(dir_path);
            }

            moduleContext.ShowCls = (options & ModuleOptions.ShowClsMethods) != 0;
            moduleContext.Features = options;

            if ((options & ModuleOptions.Initialize) != 0) {
                scriptCode.Run(moduleContext.GlobalScope);

                if (!moduleContext.Globals.ContainsKey("__package__")) {
                    moduleContext.Globals["__package__"] = null;
                }
            }

            return moduleContext.Module;
        }

        public override ScopeExtension CreateScopeExtension(Scope scope) {
            var ret = new PythonScopeExtension(this, scope);
            ret.ModuleContext.InitializeBuiltins(false);

            return ret;
        }

        public PythonModule/*!*/ CompileModule(string fileName, string moduleName, SourceUnit sourceCode, ModuleOptions options) {
            ScriptCode compiledCode;
            return CompileModule(fileName, moduleName, sourceCode, options, out compiledCode);
        }

        public PythonModule/*!*/ CompileModule(string fileName, string moduleName, SourceUnit sourceCode, ModuleOptions options, out ScriptCode scriptCode) {
            ContractUtils.RequiresNotNull(fileName, "fileName");
            ContractUtils.RequiresNotNull(moduleName, "moduleName");
            ContractUtils.RequiresNotNull(sourceCode, "sourceCode");

            scriptCode = GetScriptCode(sourceCode, moduleName, options);
            Scope scope = scriptCode.CreateScope();
            return InitializeModule(fileName, ((PythonScopeExtension)scope.GetExtension(ContextId)).ModuleContext, scriptCode, options);
        }

        internal ScriptCode GetScriptCode(SourceUnit sourceCode, string moduleName, ModuleOptions options) {
            return GetScriptCode(sourceCode, moduleName, options, null);
        }

        internal ScriptCode GetScriptCode(SourceUnit sourceCode, string moduleName, ModuleOptions options, Compiler.CompilationMode mode) {
            PythonCompilerOptions compilerOptions = GetPythonCompilerOptions();

            compilerOptions.SkipFirstLine = (options & ModuleOptions.SkipFirstLine) != 0;
            compilerOptions.ModuleName = m…

Large files files are truncated, but you can click here to view the full file