/DICK.B1/IronPython.Modules/cPickle.cs
C# | 1919 lines | 1733 code | 115 blank | 71 comment | 157 complexity | f67e1e744406b8aea75d1cef552d26ef MD5 | raw file
- /* ****************************************************************************
- *
- * Copyright (c) Microsoft Corporation.
- *
- * This source code is subject to terms and conditions of the Microsoft Public License. A
- * copy of the license can be found in the License.html file at the root of this distribution. If
- * you cannot locate the Microsoft Public License, please send an email to
- * ironpy@microsoft.com. By using this source code in any fashion, you are agreeing to be bound
- * by the terms of the Microsoft Public License.
- *
- * You must not remove this notice, or any other, from this software.
- *
- *
- * ***************************************************************************/
-
- using System;
- using System.Collections;
- using System.Collections.Generic;
- using System.Diagnostics;
- using System.IO;
- using System.Runtime.InteropServices;
- using System.Text;
- using IronPython.Runtime;
- using IronPython.Runtime.Exceptions;
- using IronPython.Runtime.Operations;
- using IronPython.Runtime.Types;
- using Microsoft.Scripting;
- using Microsoft.Scripting.Runtime;
- using Microsoft.Scripting.Utils;
-
- #if CLR2
- using Microsoft.Scripting.Math;
- #else
- using System.Numerics;
- #endif
-
- [assembly: PythonModule("cPickle", typeof(IronPython.Modules.PythonPickle))]
- namespace IronPython.Modules {
- public static class PythonPickle {
- public const string __doc__ = "Fast object serialization/deserialization.\n\n"
- + "Differences from CPython:\n"
- + " - does not implement the undocumented fast mode\n";
- [System.Runtime.CompilerServices.SpecialName]
- public static void PerformModuleReload(PythonContext/*!*/ context, PythonDictionary/*!*/ dict) {
- context.EnsureModuleException("PickleError", dict, "PickleError", "cPickle");
- context.EnsureModuleException("PicklingError", dict, "PicklingError", "cPickle");
- context.EnsureModuleException("UnpicklingError", dict, "UnpicklingError", "cPickle");
- context.EnsureModuleException("UnpickleableError", dict, "UnpickleableError", "cPickle");
- context.EnsureModuleException("BadPickleGet", dict, "BadPickleGet", "cPickle");
- dict["__builtins__"] = context.BuiltinModuleInstance;
- dict["compatible_formats"] = PythonOps.MakeList("1.0", "1.1", "1.2", "1.3", "2.0");
- }
-
- private static readonly PythonStruct.Struct _float64 = PythonStruct.Struct.Create(">d");
- private static readonly PythonStruct.Struct _uint8 = PythonStruct.Struct.Create("B");
- private static readonly PythonStruct.Struct _uint16 = PythonStruct.Struct.Create("<H");
- private static readonly PythonStruct.Struct _uint32 = PythonStruct.Struct.Create("<i");
-
- private const int highestProtocol = 2;
-
- public const string __version__ = "1.71";
- public const string format_version = "2.0";
-
- public static int HIGHEST_PROTOCOL {
- get { return highestProtocol; }
- }
-
- private const string Newline = "\n";
-
- #region Public module-level functions
-
- [Documentation("dump(obj, file, protocol=0) -> None\n\n"
- + "Pickle obj and write the result to file.\n"
- + "\n"
- + "See documentation for Pickler() for a description the file, protocol, and\n"
- + "(deprecated) bin parameters."
- )]
- public static void dump(CodeContext/*!*/ context, object obj, object file, [DefaultParameterValue(null)] object protocol, [DefaultParameterValue(null)] object bin) {
- PicklerObject pickler = new PicklerObject(context, file, protocol, bin);
- pickler.dump(context, obj);
- }
-
- [Documentation("dumps(obj, protocol=0) -> pickle string\n\n"
- + "Pickle obj and return the result as a string.\n"
- + "\n"
- + "See the documentation for Pickler() for a description of the protocol and\n"
- + "(deprecated) bin parameters."
- )]
- public static string dumps(CodeContext/*!*/ context, object obj, [DefaultParameterValue(null)] object protocol, [DefaultParameterValue(null)] object bin) {
- //??? possible perf enhancement: use a C# TextWriter-backed IFileOutput and
- // thus avoid Python call overhead. Also do similar thing for LoadFromString.
- object stringIO = PythonOps.Invoke(context, DynamicHelpers.GetPythonTypeFromType(typeof(PythonStringIO)), "StringIO");
- PicklerObject pickler = new PicklerObject(context, stringIO, protocol, bin);
- pickler.dump(context, obj);
- return Converter.ConvertToString(PythonOps.Invoke(context, stringIO, "getvalue"));
- }
-
- [Documentation("load(file) -> unpickled object\n\n"
- + "Read pickle data from the open file object and return the corresponding\n"
- + "unpickled object. Data after the first pickle found is ignored, but the file\n"
- + "cursor is not reset, so if a file objects contains multiple pickles, then\n"
- + "load() may be called multiple times to unpickle them.\n"
- + "\n"
- + "file: an object (such as an open file or a StringIO) with read(num_chars) and\n"
- + " readline() methods that return strings\n"
- + "\n"
- + "load() automatically determines if the pickle data was written in binary or\n"
- + "text mode."
- )]
- public static object load(CodeContext/*!*/ context, object file) {
- return new UnpicklerObject(context, file).load(context);
- }
-
- [Documentation("loads(string) -> unpickled object\n\n"
- + "Read a pickle object from a string, unpickle it, and return the resulting\n"
- + "reconstructed object. Characters in the string beyond the end of the first\n"
- + "pickle are ignored."
- )]
- public static object loads(CodeContext/*!*/ context, string @string) {
- PythonFile pf = PythonFile.Create(
- context,
- new MemoryStream(@string.MakeByteArray()),
- "loads",
- "b"
- );
-
- return new UnpicklerObject(context, pf).load(context);
- }
-
- #endregion
-
- #region File I/O wrappers
-
- /// <summary>
- /// Interface for "file-like objects" that implement the protocol needed by load() and friends.
- /// This enables the creation of thin wrappers that make fast .NET types and slow Python types look the same.
- /// </summary>
- internal interface IFileInput {
- string Read(CodeContext/*!*/ context, int size);
- string ReadLine(CodeContext/*!*/ context);
- }
-
- /// <summary>
- /// Interface for "file-like objects" that implement the protocol needed by dump() and friends.
- /// This enables the creation of thin wrappers that make fast .NET types and slow Python types look the same.
- /// </summary>
- internal interface IFileOutput {
- void Write(CodeContext/*!*/ context, string data);
- }
-
- private class PythonFileInput : IFileInput {
- private object _readMethod;
- private object _readLineMethod;
-
- public PythonFileInput(CodeContext/*!*/ context, object file) {
- if (!PythonOps.TryGetBoundAttr(context, file, "read", out _readMethod) ||
- !PythonOps.IsCallable(context, _readMethod) ||
- !PythonOps.TryGetBoundAttr(context, file, "readline", out _readLineMethod) ||
- !PythonOps.IsCallable(context, _readLineMethod)
- ) {
- throw PythonOps.TypeError("argument must have callable 'read' and 'readline' attributes");
- }
- }
-
- public string Read(CodeContext/*!*/ context, int size) {
- return Converter.ConvertToString(PythonCalls.Call(context, _readMethod, size));
- }
-
- public string ReadLine(CodeContext/*!*/ context) {
- return Converter.ConvertToString(PythonCalls.Call(context, _readLineMethod));
- }
- }
-
- private class PythonFileOutput : IFileOutput {
- private object _writeMethod;
-
- public PythonFileOutput(CodeContext/*!*/ context, object file) {
- if (!PythonOps.TryGetBoundAttr(context, file, "write", out _writeMethod) ||
- !PythonOps.IsCallable(context, this._writeMethod)
- ) {
- throw PythonOps.TypeError("argument must have callable 'write' attribute");
- }
- }
-
- public void Write(CodeContext/*!*/ context, string data) {
- PythonCalls.Call(context, _writeMethod, data);
- }
- }
-
- private class PythonReadableFileOutput : PythonFileOutput {
- private object _getValueMethod;
-
- public PythonReadableFileOutput(CodeContext/*!*/ context, object file)
- : base(context, file) {
- if (!PythonOps.TryGetBoundAttr(context, file, "getvalue", out _getValueMethod) ||
- !PythonOps.IsCallable(context, _getValueMethod)
- ) {
- throw PythonOps.TypeError("argument must have callable 'getvalue' attribute");
- }
- }
-
- public object GetValue(CodeContext/*!*/ context) {
- return PythonCalls.Call(context, _getValueMethod);
- }
- }
-
- #endregion
-
- #region Opcode constants
-
- internal static class Opcode {
- public const string Append = "a";
- public const string Appends = "e";
- public const string BinFloat = "G";
- public const string BinGet = "h";
- public const string BinInt = "J";
- public const string BinInt1 = "K";
- public const string BinInt2 = "M";
- public const string BinPersid = "Q";
- public const string BinPut = "q";
- public const string BinString = "T";
- public const string BinUnicode = "X";
- public const string Build = "b";
- public const string Dict = "d";
- public const string Dup = "2";
- public const string EmptyDict = "}";
- public const string EmptyList = "]";
- public const string EmptyTuple = ")";
- public const string Ext1 = "\x82";
- public const string Ext2 = "\x83";
- public const string Ext4 = "\x84";
- public const string Float = "F";
- public const string Get = "g";
- public const string Global = "c";
- public const string Inst = "i";
- public const string Int = "I";
- public const string List = "l";
- public const string Long = "L";
- public const string Long1 = "\x8a";
- public const string Long4 = "\x8b";
- public const string LongBinGet = "j";
- public const string LongBinPut = "r";
- public const string Mark = "(";
- public const string NewFalse = "\x89";
- public const string NewObj = "\x81";
- public const string NewTrue = "\x88";
- public const string NoneValue = "N";
- public const string Obj = "o";
- public const string PersId = "P";
- public const string Pop = "0";
- public const string PopMark = "1";
- public const string Proto = "\x80";
- public const string Put = "p";
- public const string Reduce = "R";
- public const string SetItem = "s";
- public const string SetItems = "u";
- public const string ShortBinstring = "U";
- public const string Stop = ".";
- public const string String = "S";
- public const string Tuple = "t";
- public const string Tuple1 = "\x85";
- public const string Tuple2 = "\x86";
- public const string Tuple3 = "\x87";
- public const string Unicode = "V";
- }
-
- #endregion
-
- #region Pickler object
-
- public static PicklerObject Pickler(CodeContext/*!*/ context, [DefaultParameterValue(null)]object file, [DefaultParameterValue(null)]object protocol, [DefaultParameterValue(null)]object bin) {
- return new PicklerObject(context, file, protocol, bin);
- }
-
- [Documentation("Pickler(file, protocol=0) -> Pickler object\n\n"
- + "A Pickler object serializes Python objects to a pickle bytecode stream, which\n"
- + "can then be converted back into equivalent objects using an Unpickler.\n"
- + "\n"
- + "file: an object (such as an open file) that has a write(string) method.\n"
- + "protocol: if omitted, protocol 0 is used. If HIGHEST_PROTOCOL or a negative\n"
- + " number, the highest available protocol is used.\n"
- + "bin: (deprecated; use protocol instead) for backwards compability, a 'bin'\n"
- + " keyword parameter is supported. When protocol is specified it is ignored.\n"
- + " If protocol is not specified, then protocol 0 is used if bin is false, and\n"
- + " protocol 1 is used if bin is true."
- )]
- [PythonType("Pickler"), PythonHidden]
- public class PicklerObject {
-
- private const char LowestPrintableChar = (char)32;
- private const char HighestPrintableChar = (char)126;
- // max elements that can be set/appended at a time using SETITEMS/APPENDS
-
- private delegate void PickleFunction(CodeContext/*!*/ context, object value);
- private readonly Dictionary<PythonType, PickleFunction> dispatchTable;
-
- private int _batchSize = 1000;
- private IFileOutput _file;
- private int _protocol;
- private IDictionary _memo;
- private object _persist_id;
-
- #region Public API
-
- [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
- public IDictionary memo {
- get { return _memo; }
- set { _memo = value; }
- }
-
- public int proto {
- get { return _protocol; }
- set { _protocol = value; }
- }
-
- public int _BATCHSIZE {
- get { return _batchSize; }
- set { _batchSize = value; }
- }
-
- public object persistent_id {
- get {
- return _persist_id;
- }
- set {
- _persist_id = value;
- }
- }
-
- public int binary {
- get { return _protocol == 0 ? 1 : 0; }
- set { _protocol = value; }
- }
-
- public int fast {
- // We don't implement fast, but we silently ignore it when it's set so that test_cpickle works.
- // For a description of fast, see http://mail.python.org/pipermail/python-bugs-list/2001-October/007695.html
- get { return 0; }
- set { /* ignore */ }
- }
-
- public PicklerObject(CodeContext/*!*/ context, object file, object protocol, object bin) {
- dispatchTable = new Dictionary<PythonType, PickleFunction>();
- dispatchTable[TypeCache.Boolean] = SaveBoolean;
- dispatchTable[TypeCache.Int32] = SaveInteger;
- dispatchTable[TypeCache.Null] = SaveNone;
- dispatchTable[TypeCache.Dict] = SaveDict;
- dispatchTable[TypeCache.BigInteger] = SaveLong;
- dispatchTable[TypeCache.Double] = SaveFloat;
- dispatchTable[TypeCache.String] = SaveUnicode;
- dispatchTable[TypeCache.PythonTuple] = SaveTuple;
- dispatchTable[TypeCache.List] = SaveList;
- dispatchTable[TypeCache.OldClass] = SaveGlobal;
- dispatchTable[TypeCache.Function] = SaveGlobal;
- dispatchTable[TypeCache.BuiltinFunction] = SaveGlobal;
- dispatchTable[TypeCache.PythonType] = SaveGlobal;
- dispatchTable[TypeCache.OldInstance] = SaveInstance;
-
- int intProtocol;
- if (file == null) {
- _file = new PythonReadableFileOutput(context, new PythonStringIO.StringO());
- } else if (Converter.TryConvertToInt32(file, out intProtocol)) {
- // For undocumented (yet tested in official CPython tests) list-based pickler, the
- // user could do something like Pickler(1), which would create a protocol-1 pickler
- // with an internal string output buffer (retrievable using GetValue()). For a little
- // more info, see
- // https://sourceforge.net/tracker/?func=detail&atid=105470&aid=939395&group_id=5470
- _file = new PythonReadableFileOutput(context, new PythonStringIO.StringO());
- protocol = file;
- } else if (file is IFileOutput) {
- _file = (IFileOutput)file;
- } else {
- _file = new PythonFileOutput(context, file);
- }
-
- this._memo = new PythonDictionary();
-
- if (protocol == null) protocol = PythonOps.IsTrue(bin) ? 1 : 0;
-
- intProtocol = PythonContext.GetContext(context).ConvertToInt32(protocol);
- if (intProtocol > highestProtocol) {
- throw PythonOps.ValueError("pickle protocol {0} asked for; the highest available protocol is {1}", intProtocol, highestProtocol);
- } else if (intProtocol < 0) {
- this._protocol = highestProtocol;
- } else {
- this._protocol = intProtocol;
- }
- }
-
- [Documentation("dump(obj) -> None\n\n"
- + "Pickle obj and write the result to the file object that was passed to the\n"
- + "constructor\n."
- + "\n"
- + "Note that you may call dump() multiple times to pickle multiple objects. To\n"
- + "unpickle the stream, you will need to call Unpickler's load() method a\n"
- + "corresponding number of times.\n"
- + "\n"
- + "The first time a particular object is encountered, it will be pickled normally.\n"
- + "If the object is encountered again (in the same or a later dump() call), a\n"
- + "reference to the previously generated value will be pickled. Unpickling will\n"
- + "then create multiple references to a single object."
- )]
- public void dump(CodeContext/*!*/ context, object obj) {
- if (_protocol >= 2) WriteProto(context);
- Save(context, obj);
- Write(context, Opcode.Stop);
- }
-
- [Documentation("clear_memo() -> None\n\n"
- + "Clear the memo, which is used internally by the pickler to keep track of which\n"
- + "objects have already been pickled (so that shared or recursive objects are\n"
- + "pickled only once)."
- )]
- public void clear_memo() {
- _memo.Clear();
- }
-
- [Documentation("getvalue() -> string\n\n"
- + "Return the value of the internal string. Raises PicklingError if a file object\n"
- + "was passed to this pickler's constructor."
- )]
- public object getvalue(CodeContext/*!*/ context) {
- if (_file is PythonReadableFileOutput) {
- return ((PythonReadableFileOutput)_file).GetValue(context);
- }
- throw PythonExceptions.CreateThrowable(PicklingError(context), "Attempt to getvalue() a non-list-based pickler");
- }
-
- #endregion
-
- #region Save functions
-
- private void Save(CodeContext/*!*/ context, object obj) {
- if (_persist_id != null) {
- string res = Converter.ConvertToString(PythonContext.GetContext(context).CallSplat(_persist_id, obj));
- if (res != null) {
- SavePersId(context, res);
- return;
- }
- }
-
- if (_memo.Contains(PythonOps.Id(obj))) {
- WriteGet(context, obj);
- } else {
- PickleFunction pickleFunction;
- PythonType objType = DynamicHelpers.GetPythonType(obj);
- if (!dispatchTable.TryGetValue(objType, out pickleFunction)) {
- if (objType.IsSubclassOf(TypeCache.PythonType)) {
- // treat classes with metaclasses like regular classes
- pickleFunction = SaveGlobal;
- } else {
- pickleFunction = SaveObject;
- }
- }
- pickleFunction(context, obj);
- }
- }
-
- private void SavePersId(CodeContext/*!*/ context, string res) {
- if (this.binary != 0) {
- Save(context, res);
- Write(context, Opcode.BinPersid);
- } else {
- Write(context, Opcode.PersId);
- Write(context, res);
- Write(context, "\n");
- }
- }
-
- private void SaveBoolean(CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Boolean), "arg must be bool");
- if (_protocol < 2) {
- Write(context, Opcode.Int);
- Write(context, String.Format("0{0}", ((bool)obj) ? 1 : 0));
- Write(context, Newline);
- } else {
- if ((bool)obj) {
- Write(context, Opcode.NewTrue);
- } else {
- Write(context, Opcode.NewFalse);
- }
- }
- }
-
- private void SaveDict(CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Dict), "arg must be dict");
- Debug.Assert(!_memo.Contains(PythonOps.Id(obj)));
- Memoize(obj);
-
- if (_protocol < 1) {
- Write(context, Opcode.Mark);
- Write(context, Opcode.Dict);
- } else {
- Write(context, Opcode.EmptyDict);
- }
-
- WritePut(context, obj);
- BatchSetItems(context, (DictionaryOps.iteritems((IDictionary<object, object>)obj)));
- }
-
- private void SaveFloat(CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Double), "arg must be float");
-
- if (_protocol < 1) {
- Write(context, Opcode.Float);
- WriteFloatAsString(context, obj);
- } else {
- Write(context, Opcode.BinFloat);
- WriteFloat64(context, obj);
- }
- }
-
- private void SaveGlobal(CodeContext/*!*/ context, object obj) {
- Debug.Assert(
- DynamicHelpers.GetPythonType(obj).Equals(TypeCache.OldClass) ||
- DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Function) ||
- DynamicHelpers.GetPythonType(obj).Equals(TypeCache.BuiltinFunction) ||
- DynamicHelpers.GetPythonType(obj).Equals(TypeCache.PythonType) ||
- DynamicHelpers.GetPythonType(obj).IsSubclassOf(TypeCache.PythonType),
- "arg must be classic class, function, built-in function or method, or new-style type"
- );
-
- object name;
- if (PythonOps.TryGetBoundAttr(context, obj, "__name__", out name)) {
- SaveGlobalByName(context, obj, name);
- } else {
- throw CannotPickle(context, obj, "could not determine its __name__");
- }
- }
-
- private void SaveGlobalByName(CodeContext/*!*/ context, object obj, object name) {
- Debug.Assert(!_memo.Contains(PythonOps.Id(obj)));
-
- object moduleName = FindModuleForGlobal(context, obj, name);
-
- if (_protocol >= 2) {
- object code;
- if (((IDictionary<object, object>)PythonCopyReg.GetExtensionRegistry(context)).TryGetValue(PythonTuple.MakeTuple(moduleName, name), out code)) {
- if (IsUInt8(context, code)) {
- Write(context, Opcode.Ext1);
- WriteUInt8(context, code);
- } else if (IsUInt16(context, code)) {
- Write(context, Opcode.Ext2);
- WriteUInt16(context, code);
- } else if (IsInt32(context, code)) {
- Write(context, Opcode.Ext4);
- WriteInt32(context, code);
- } else {
- throw PythonOps.RuntimeError("unrecognized integer format");
- }
- return;
- }
- }
-
- Memoize(obj);
-
- Write(context, Opcode.Global);
- WriteStringPair(context, moduleName, name);
- WritePut(context, obj);
- }
-
- private void SaveInstance(CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.OldInstance), "arg must be old-class instance");
- Debug.Assert(!_memo.Contains(PythonOps.Id(obj)));
-
- Write(context, Opcode.Mark);
-
- // Memoize() call isn't in the usual spot to allow class to be memoized before
- // instance (when using proto other than 0) to match CPython's bytecode output
-
- object objClass;
- if (!PythonOps.TryGetBoundAttr(context, obj, "__class__", out objClass)) {
- throw CannotPickle(context, obj, "could not determine its __class__");
- }
-
- if (_protocol < 1) {
- object className, classModuleName;
- if (!PythonOps.TryGetBoundAttr(context, objClass, "__name__", out className)) {
- throw CannotPickle(context, obj, "its __class__ has no __name__");
- }
- classModuleName = FindModuleForGlobal(context, objClass, className);
-
- Memoize(obj);
- WriteInitArgs(context, obj);
- Write(context, Opcode.Inst);
- WriteStringPair(context, classModuleName, className);
- } else {
- Save(context, objClass);
- Memoize(obj);
- WriteInitArgs(context, obj);
- Write(context, Opcode.Obj);
- }
-
- WritePut(context, obj);
-
- object getStateCallable;
- if (PythonOps.TryGetBoundAttr(context, obj, "__getstate__", out getStateCallable)) {
- Save(context, PythonCalls.Call(context, getStateCallable));
- } else {
- Save(context, PythonOps.GetBoundAttr(context, obj, "__dict__"));
- }
-
- Write(context, Opcode.Build);
- }
-
- private void SaveInteger(CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Int32), "arg must be int");
- if (_protocol < 1) {
- Write(context, Opcode.Int);
- WriteIntAsString(context, obj);
- } else {
- if (IsUInt8(context, obj)) {
- Write(context, Opcode.BinInt1);
- WriteUInt8(context, obj);
- } else if (IsUInt16(context, obj)) {
- Write(context, Opcode.BinInt2);
- WriteUInt16(context, obj);
- } else if (IsInt32(context, obj)) {
- Write(context, Opcode.BinInt);
- WriteInt32(context, obj);
- } else {
- throw PythonOps.RuntimeError("unrecognized integer format");
- }
- }
- }
-
- private void SaveList(CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.List), "arg must be list");
- Debug.Assert(!_memo.Contains(PythonOps.Id(obj)));
- Memoize(obj);
- if (_protocol < 1) {
- Write(context, Opcode.Mark);
- Write(context, Opcode.List);
- } else {
- Write(context, Opcode.EmptyList);
- }
-
- WritePut(context, obj);
- BatchAppends(context, ((IEnumerable)obj).GetEnumerator());
- }
-
- private void SaveLong(CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.BigInteger), "arg must be long");
-
- if (_protocol < 2) {
- Write(context, Opcode.Long);
- WriteLongAsString(context, obj);
- } else {
- if (((BigInteger)obj).IsZero()) {
- Write(context, Opcode.Long1);
- WriteUInt8(context, 0);
- } else {
- byte[] dataBytes = ((BigInteger)obj).ToByteArray();
- if (dataBytes.Length < 256) {
- Write(context, Opcode.Long1);
- WriteUInt8(context, dataBytes.Length);
- } else {
- Write(context, Opcode.Long4);
- WriteInt32(context, dataBytes.Length);
- }
-
- foreach (byte b in dataBytes) {
- WriteUInt8(context, b);
- }
- }
- }
- }
-
- private void SaveNone(CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Null), "arg must be None");
- Write(context, Opcode.NoneValue);
- }
-
- /// <summary>
- /// Call the appropriate reduce method for obj and pickle the object using
- /// the resulting data. Use the first available of
- /// copy_reg.dispatch_table[type(obj)], obj.__reduce_ex__, and obj.__reduce__.
- /// </summary>
- private void SaveObject(CodeContext/*!*/ context, object obj) {
- Debug.Assert(!_memo.Contains(PythonOps.Id(obj)));
- Memoize(obj);
-
- object reduceCallable, result;
- PythonType objType = DynamicHelpers.GetPythonType(obj);
-
- if (((IDictionary<object, object>)PythonCopyReg.GetDispatchTable(context)).TryGetValue(objType, out reduceCallable)) {
- result = PythonCalls.Call(context, reduceCallable, obj);
- } else if (PythonOps.TryGetBoundAttr(context, obj, "__reduce_ex__", out reduceCallable)) {
- if (obj is PythonType) {
- result = context.LanguageContext.Call(context, reduceCallable, obj, _protocol);
- } else {
- result = context.LanguageContext.Call(context, reduceCallable, _protocol);
- }
- } else if (PythonOps.TryGetBoundAttr(context, obj, "__reduce__", out reduceCallable)) {
- if (obj is PythonType) {
- result = context.LanguageContext.Call(context, reduceCallable, obj);
- } else {
- result = context.LanguageContext.Call(context, reduceCallable);
- }
- } else {
- throw PythonOps.AttributeError("no reduce function found for {0}", obj);
- }
-
- if (objType.Equals(TypeCache.String)) {
- if (_memo.Contains(PythonOps.Id(obj))) {
- WriteGet(context, obj);
- } else {
- SaveGlobalByName(context, obj, result);
- }
- } else if (result is PythonTuple) {
- PythonTuple rt = (PythonTuple)result;
- switch (rt.__len__()) {
- case 2:
- SaveReduce(context, obj, reduceCallable, rt[0], rt[1], null, null, null);
- break;
- case 3:
- SaveReduce(context, obj, reduceCallable, rt[0], rt[1], rt[2], null, null);
- break;
- case 4:
- SaveReduce(context, obj, reduceCallable, rt[0], rt[1], rt[2], rt[3], null);
- break;
- case 5:
- SaveReduce(context, obj, reduceCallable, rt[0], rt[1], rt[2], rt[3], rt[4]);
- break;
- default:
- throw CannotPickle(context, obj, "tuple returned by {0} must have to to five elements", reduceCallable);
- }
- } else {
- throw CannotPickle(context, obj, "{0} must return string or tuple", reduceCallable);
- }
- }
-
- /// <summary>
- /// Pickle the result of a reduce function.
- ///
- /// Only context, obj, func, and reduceCallable are required; all other arguments may be null.
- /// </summary>
- private void SaveReduce(CodeContext/*!*/ context, object obj, object reduceCallable, object func, object args, object state, object listItems, object dictItems) {
- if (!PythonOps.IsCallable(context, func)) {
- throw CannotPickle(context, obj, "func from reduce() should be callable");
- } else if (!(args is PythonTuple) && args != null) {
- throw CannotPickle(context, obj, "args from reduce() should be a tuple");
- } else if (listItems != null && !(listItems is IEnumerator)) {
- throw CannotPickle(context, obj, "listitems from reduce() should be a list iterator");
- } else if (dictItems != null && !(dictItems is IEnumerator)) {
- throw CannotPickle(context, obj, "dictitems from reduce() should be a dict iterator");
- }
-
- object funcName;
- string funcNameString;
- if (!PythonOps.TryGetBoundAttr(context, func, "__name__", out funcName)) {
- throw CannotPickle(context, obj, "func from reduce() ({0}) should have a __name__ attribute");
- } else if (!Converter.TryConvertToString(funcName, out funcNameString) || funcNameString == null) {
- throw CannotPickle(context, obj, "__name__ of func from reduce() must be string");
- }
-
- if (_protocol >= 2 && "__newobj__" == funcNameString) {
- if (args == null) {
- throw CannotPickle(context, obj, "__newobj__ arglist is None");
- }
- PythonTuple argsTuple = (PythonTuple)args;
- if (argsTuple.__len__() == 0) {
- throw CannotPickle(context, obj, "__newobj__ arglist is empty");
- } else if (!DynamicHelpers.GetPythonType(obj).Equals(argsTuple[0])) {
- throw CannotPickle(context, obj, "args[0] from __newobj__ args has the wrong class");
- }
- Save(context, argsTuple[0]);
- Save(context, argsTuple[new Slice(1, null)]);
- Write(context, Opcode.NewObj);
- } else {
- Save(context, func);
- Save(context, args);
- Write(context, Opcode.Reduce);
- }
-
- WritePut(context, obj);
-
- if (state != null) {
- Save(context, state);
- Write(context, Opcode.Build);
- }
-
- if (listItems != null) {
- BatchAppends(context, (IEnumerator)listItems);
- }
-
- if (dictItems != null) {
- BatchSetItems(context, (IEnumerator)dictItems);
- }
- }
-
- private void SaveTuple(CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.PythonTuple), "arg must be tuple");
- Debug.Assert(!_memo.Contains(PythonOps.Id(obj)));
- PythonTuple t = (PythonTuple)obj;
- string opcode;
- bool needMark = false;
- if (_protocol > 0 && t.__len__() == 0) {
- opcode = Opcode.EmptyTuple;
- } else if (_protocol >= 2 && t.__len__() == 1) {
- opcode = Opcode.Tuple1;
- } else if (_protocol >= 2 && t.__len__() == 2) {
- opcode = Opcode.Tuple2;
- } else if (_protocol >= 2 && t.__len__() == 3) {
- opcode = Opcode.Tuple3;
- } else {
- opcode = Opcode.Tuple;
- needMark = true;
- }
-
- if (needMark) Write(context, Opcode.Mark);
- foreach (object o in t) {
- Save(context, o);
- }
-
- if (_memo.Contains(PythonOps.Id(obj))) {
- // recursive tuple
- if (_protocol == 1) {
- Write(context, Opcode.PopMark);
- } else {
- if (_protocol == 0) {
- Write(context, Opcode.Pop);
- }
- for (int i = 0; i < t.__len__(); i++) {
- Write(context, Opcode.Pop);
- }
- }
- WriteGet(context, obj);
- return;
- }
-
- Write(context, opcode);
-
- if (t.__len__() > 0) {
- Memoize(t);
- WritePut(context, t);
- }
- }
-
- private void SaveUnicode(CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.String), "arg must be unicode");
- Debug.Assert(!_memo.Contains(PythonOps.Id(obj)));
- Memoize(obj);
- if (_protocol < 1) {
- Write(context, Opcode.Unicode);
- WriteUnicodeStringRaw(context, obj);
- } else {
- Write(context, Opcode.BinUnicode);
- WriteUnicodeStringUtf8(context, obj);
- }
-
- WritePut(context, obj);
- }
-
- #endregion
-
- #region Output encoding
-
- /// <summary>
- /// Write value in pickle decimalnl_short format.
- /// </summary>
- private void WriteFloatAsString(CodeContext/*!*/ context, object value) {
- Debug.Assert(DynamicHelpers.GetPythonType(value).Equals(TypeCache.Double));
- Write(context, DoubleOps.__repr__(context, (double)value));
- Write(context, Newline);
- }
-
- /// <summary>
- /// Write value in pickle float8 format.
- /// </summary>
- private void WriteFloat64(CodeContext/*!*/ context, object value) {
- Debug.Assert(DynamicHelpers.GetPythonType(value).Equals(TypeCache.Double));
- Write(context, _float64.pack(context, value));
- }
-
- /// <summary>
- /// Write value in pickle uint1 format.
- /// </summary>
- private void WriteUInt8(CodeContext/*!*/ context, object value) {
- Debug.Assert(IsUInt8(context, value));
- Write(context, _uint8.pack(context, value));
- }
-
- /// <summary>
- /// Write value in pickle uint2 format.
- /// </summary>
- private void WriteUInt16(CodeContext/*!*/ context, object value) {
- Debug.Assert(IsUInt16(context, value));
- Write(context, _uint16.pack(context, value));
- }
-
- /// <summary>
- /// Write value in pickle int4 format.
- /// </summary>
- private void WriteInt32(CodeContext/*!*/ context, object value) {
- Debug.Assert(IsInt32(context, value));
- Write(context, _uint32.pack(context, value));
- }
-
- /// <summary>
- /// Write value in pickle decimalnl_short format.
- /// </summary>
- private void WriteIntAsString(CodeContext/*!*/ context, object value) {
- Debug.Assert(IsInt32(context, value));
- Write(context, PythonOps.Repr(context, value));
- Write(context, Newline);
- }
-
- /// <summary>
- /// Write value in pickle decimalnl_long format.
- /// </summary>
- private void WriteLongAsString(CodeContext/*!*/ context, object value) {
- Debug.Assert(DynamicHelpers.GetPythonType(value).Equals(TypeCache.BigInteger));
- Write(context, PythonOps.Repr(context, value));
- Write(context, Newline);
- }
-
- /// <summary>
- /// Write value in pickle unicodestringnl format.
- /// </summary>
- private void WriteUnicodeStringRaw(CodeContext/*!*/ context, object value) {
- Debug.Assert(DynamicHelpers.GetPythonType(value).Equals(TypeCache.String));
- // manually escape backslash and newline
- Write(context, StringOps.RawUnicodeEscapeEncode(((string)value).Replace("\\", "\\u005c").Replace("\n", "\\u000a")));
- Write(context, Newline);
- }
-
- /// <summary>
- /// Write value in pickle unicodestring4 format.
- /// </summary>
- private void WriteUnicodeStringUtf8(CodeContext/*!*/ context, object value) {
- Debug.Assert(DynamicHelpers.GetPythonType(value).Equals(TypeCache.String));
- string encodedString = System.Text.Encoding.UTF8.GetBytes((string)value).MakeString();
- WriteInt32(context, encodedString.Length);
- Write(context, encodedString);
- }
-
- /// <summary>
- /// Write value in pickle stringnl_noescape_pair format.
- /// </summary>
- private void WriteStringPair(CodeContext/*!*/ context, object value1, object value2) {
- Debug.Assert(DynamicHelpers.GetPythonType(value1).Equals(TypeCache.String));
- Debug.Assert(DynamicHelpers.GetPythonType(value2).Equals(TypeCache.String));
- #if DEBUG
- Debug.Assert(IsPrintableAscii(value1));
- Debug.Assert(IsPrintableAscii(value2));
- #endif
- Write(context, (string)value1);
- Write(context, Newline);
- Write(context, (string)value2);
- Write(context, Newline);
- }
-
- #endregion
-
- #region Type checking
-
- /// <summary>
- /// Return true if value is appropriate for formatting in pickle uint1 format.
- /// </summary>
- private bool IsUInt8(CodeContext/*!*/ context, object value) {
- PythonContext pc = PythonContext.GetContext(context);
-
- return pc.LessThanOrEqual(0, value) && pc.LessThan(value, 1 << 8);
- }
-
- /// <summary>
- /// Return true if value is appropriate for formatting in pickle uint2 format.
- /// </summary>
- private bool IsUInt16(CodeContext/*!*/ context, object value) {
- PythonContext pc = PythonContext.GetContext(context);
-
- return pc.LessThanOrEqual(1 << 8, value) && pc.LessThan(value, 1 << 16);
- }
-
- /// <summary>
- /// Return true if value is appropriate for formatting in pickle int4 format.
- /// </summary>
- private bool IsInt32(CodeContext/*!*/ context, object value) {
- PythonContext pc = PythonContext.GetContext(context);
-
- return pc.LessThanOrEqual(Int32.MinValue, value) && pc.LessThanOrEqual(value, Int32.MaxValue);
- }
-
- #if DEBUG
- /// <summary>
- /// Return true if value is a string where each value is in the range of printable ASCII characters.
- /// </summary>
- private bool IsPrintableAscii(object value) {
- Debug.Assert(DynamicHelpers.GetPythonType(value).Equals(TypeCache.String));
- string strValue = (string)value;
- foreach (char c in strValue) {
- if (!(LowestPrintableChar <= c && c <= HighestPrintableChar)) return false;
- }
- return true;
- }
- #endif
-
- #endregion
-
- #region Output generation helpers
-
- private void Write(CodeContext/*!*/ context, string data) {
- _file.Write(context, data);
- }
-
- private void WriteGet(CodeContext/*!*/ context, object obj) {
- Debug.Assert(_memo.Contains(PythonOps.Id(obj)));
- // Memo entries are tuples, and the first element is the memo index
- IList<object> memoEntry = (IList<object>)_memo[PythonOps.Id(obj)];
-
- object index = memoEntry[0];
- Debug.Assert(PythonContext.GetContext(context).GreaterThanOrEqual(index, 0));
- if (_protocol < 1) {
- Write(context, Opcode.Get);
- WriteIntAsString(context, index);
- } else {
- if (IsUInt8(context, index)) {
- Write(context, Opcode.BinGet);
- WriteUInt8(context, index);
- } else {
- Write(context, Opcode.LongBinGet);
- WriteInt32(context, index);
- }
- }
- }
-
- private void WriteInitArgs(CodeContext/*!*/ context, object obj) {
- object getInitArgsCallable;
- if (PythonOps.TryGetBoundAttr(context, obj, "__getinitargs__", out getInitArgsCallable)) {
- object initArgs = PythonCalls.Call(context, getInitArgsCallable);
- if (!(initArgs is PythonTuple)) {
- throw CannotPickle(context, obj, "__getinitargs__() must return tuple");
- }
- foreach (object arg in (PythonTuple)initArgs) {
- Save(context, arg);
- }
- }
- }
-
- private void WritePut(CodeContext/*!*/ context, object obj) {
- Debug.Assert(_memo.Contains(PythonOps.Id(obj)));
- // Memo entries are tuples, and the first element is the memo index
- IList<object> memoEntry = (IList<object>)_memo[PythonOps.Id(obj)];
-
- object index = memoEntry[0];
- Debug.Assert(PythonContext.GetContext(context).GreaterThanOrEqual(index, 0));
-
- if (_protocol < 1) {
- Write(context, Opcode.Put);
- WriteIntAsString(context, index);
- } else {
- if (IsUInt8(context, index)) {
- Write(context, Opcode.BinPut);
- WriteUInt8(context, index);
- } else {
- Write(context, Opcode.LongBinPut);
- WriteInt32(context, index);
- }
- }
- }
-
- private void WriteProto(CodeContext/*!*/ context) {
- Write(context, Opcode.Proto);
- WriteUInt8(context, _protocol);
- }
-
- /// <summary>
- /// Emit a series of opcodes that will set append all items indexed by iter
- /// to the object at the top of the stack. Use APPENDS if possible, but
- /// append no more than BatchSize items at a time.
- /// </summary>
- private void BatchAppends(CodeContext/*!*/ context, IEnumerator enumerator) {
- if (_protocol < 1) {
- while (enumerator.MoveNext()) {
- Save(context, enumerator.Current);
- Write(context, Opcode.Append);
- }
- } else {
- object next;
- if (enumerator.MoveNext()) {
- next = enumerator.Current;
- } else {
- return;
- }
-
- int batchCompleted = 0;
- object current;
-
- // We do a one-item lookahead to avoid emitting an APPENDS for a
- // single remaining item.
- while (enumerator.MoveNext()) {
- current = next;
- next = enumerator.Current;
-
- if (batchCompleted == _BATCHSIZE) {
- Write(context, Opcode.Appends);
- batchCompleted = 0;
- }
-
- if (batchCompleted == 0) {
- Write(context, Opcode.Mark);
- }
-
- Save(context, current);
- batchCompleted++;
- }
-
- if (batchCompleted == _BATCHSIZE) {
- Write(context, Opcode.Appends);
- batchCompleted = 0;
- }
- Save(context, next);
- batchCompleted++;
-
- if (batchCompleted > 1) {
- Write(context, Opcode.Appends);
- } else {
- Write(context, Opcode.Append);
- }
- }
- }
-
- /// <summary>
- /// Emit a series of opcodes that will set all (key, value) pairs indexed by
- /// iter in the object at the top of the stack. Use SETITEMS if possible,
- /// but append no more than BatchSize items at a time.
- /// </summary>
- private void BatchSetItems(CodeContext/*!*/ context, IEnumerator enumerator) {
- PythonTuple kvTuple;
- if (_protocol < 1) {
- while (enumerator.MoveNext()) {
- kvTuple = (PythonTuple)enumerator.Current;
- Save(context, kvTuple[0]);
- Save(context, kvTuple[1]);
- Write(context, Opcode.SetItem);
- }
- } else {
- object nextKey, nextValue;
- if (enumerator.MoveNext()) {
- kvTuple = (PythonTuple)enumerator.Current;
- nextKey = kvTuple[0];
- nextValue = kvTuple[1];
- } else {
- return;
- }
-
- int batchCompleted = 0;
- object curKey, curValue;
-
- // We do a one-item lookahead to avoid emitting a SETITEMS for a
- // single remaining item.
- while (enumerator.MoveNext()) {
- curKey = nextKey;
- curValue = nextValue;
- kvTuple = (PythonTuple)enumerator.Current;
- nextKey = kvTuple[0];
- nextValue = kvTuple[1];
-
- if (batchCompleted == _BATCHSIZE) {
- Write(context, Opcode.SetItems);
- batchCompleted = 0;
- }
-
- if (batchCompleted == 0) {
- Write(context, Opcode.Mark);
- }
-
- Save(context, curKey);
- Save(context, curValue);
- batchCompleted++;
- }
-
- if (batchCompleted == _BATCHSIZE) {
- Write(context, Opcode.SetItems);
- batchCompleted = 0;
- }
- Save(context, nextKey);
- Save(context, nextValue);
- batchCompleted++;
-
- if (batchCompleted > 1) {
- Write(context, Opcode.SetItems);
- } else {
- Write(context, Opcode.SetItem);
- }
- }
- }
-
- #endregion
-
- #region Other private helper methods
-
- private Exception CannotPickle(CodeContext/*!*/ context, object obj, string format, params object[] args) {
- StringBuilder msgBuilder = new StringBuilder();
- msgBuilder.Append("Can't pickle ");
- msgBuilder.Append(PythonOps.ToString(context, obj));
- if (format != null) {
- msgBuilder.Append(": ");
- msgBuilder.Append(String.Format(format, args));
- }
- return PythonExceptions.CreateThrowable(PickleError(context), msgBuilder.ToString());
- }
-
- private void Memoize(object obj) {
- if (!_memo.Contains(PythonOps.Id(obj))) {
- _memo[PythonOps.Id(obj)] = PythonTuple.MakeTuple(_memo.Count, obj);
- }
- }
-
- /// <summary>
- /// Find the module for obj and ensure that obj is reachable in that module by the given name.
- ///
- /// Throw PicklingError if any of the following are true:
- /// - The module couldn't be determined.
- /// - The module couldn't be loaded.
- /// - The given name doesn't exist in the module.
- /// - The given name is a different object than obj.
- ///
- /// Otherwise, return the name of the module.
- ///
- /// To determine which module obj lives in, obj.__module__ is used if available. The
- /// module named by obj.__module__ is loaded if needed. If obj has no __module__
- /// attribute, then each loaded module is searched. If a loaded module has an
- /// attribute with the given name, and that attribute is the same object as obj,
- /// then that module is used.
- /// </summary>
- private object FindModuleForGlobal(CodeContext/*!*/ context, object obj, object name) {
- object module;
- object moduleName;
- if (PythonOps.TryGetBoundAttr(context, obj, "__module__", out moduleName)) {
- // TODO: Global SystemState
- Builtin.__import__(context, Converter.ConvertToString(moduleName));
-
- object foundObj;
- if (Importer.TryGetExistingModule(context, Converter.ConvertToString(moduleName), out module) &&
- PythonOps.TryGetBoundAttr(context, module, Converter.ConvertToString(name), out foundObj)) {
- if (PythonOps.IsRetBool(foundObj, obj)) {
- return moduleName;
- } else {
- throw CannotPickle(context, obj, "it's not the same object as {0}.{1}", moduleName, name);
- }
- } else {
- throw CannotPickle(context, obj, "it's not found as {0}.{1}", moduleName, name);
- }
- } else {
- // No obj.__module__, so crawl through all loaded modules looking for obj
- foreach (KeyValuePair<object, object> modulePair in PythonContext.GetContext(context).SystemStateModules) {
- moduleName = modulePair.Key;
- module = modulePair.Value;
- object foundObj;
- if (PythonOps.TryGetBoundAttr(context, module, Converter.ConvertToString(name), out foundObj) &&
- PythonOps.IsRetBool(foundObj, obj)
- ) {
- return moduleName;
- }
- }
- throw CannotPickle(context, obj, "could not determine its module");
- }
-
- }
-
- #endregion
-
- }
-
- #endregion
-
- #region Unpickler object
-
- public static UnpicklerObject Unpickler(CodeContext/*!*/ context, object file) {
- return new UnpicklerObject(context, file);
- }
-
- [Documentation("Unpickler(file) -> Unpickler object\n\n"
- + "An Unpickler object reads a pickle bytecode stream and creates corresponding\n"
- + "objects."
- + "\n"
- + "file: an object (such as an open file or a StringIO) with read(num_chars) and\n"
- + " readline() methods that return strings"
- )]
- [PythonType("Unpickler"), PythonHidden]
- public class UnpicklerObject {
-
- private readonly object _mark = new object();
-
- private delegate void LoadFunction(CodeContext/*!*/ context);
- private readonly Dictionary<string, LoadFunction> _dispatch;
-
- private IFileInput _file;
- private List _stack;
- private IDictionary<object, object> _memo;
- private object _pers_loader;
-
- public UnpicklerObject(CodeContext context, object file) {
- this._file = file as IFileInput ?? new PythonFileInput(context, file);
- _memo = new PythonDictionary();
-
- _dispatch = new Dictionary<string, LoadFunction>();
- _dispatch[""] = LoadEof;
- _dispatch[Opcode.Append] = LoadAppend;
- _dispatch[Opcode.Appends] = LoadAppends;
- _dispatch[Opcode.BinFloat] = LoadBinFloat;
- _dispatch[Opcode.BinGet] = LoadBinGet;
- _dispatch[Opcode.BinInt] = LoadBinInt;
- _dispatch[Opcode.BinInt1] = LoadBinInt1;
- _dispatch[Opcode.BinInt2] = LoadBinInt2;
- _dispatch[Opcode.BinPersid] = LoadBinPersid;
- _dispatch[Opcode.BinPut] = LoadBinPut;
- _dispatch[Opcode.BinString] = LoadBinString;
- _dispatch[Opcode.BinUnicode] = LoadBinUnicode;
- _dispatch[Opcode.Build] = LoadBuild;
- _dispatch[Opcode.Dict] = LoadDict;
- _dispatch[Opcode.Dup] = LoadDup;
- _dispatch[Opcode.EmptyDict] = LoadEmptyDict;
- _dispatch[Opcode.EmptyList] = LoadEmptyList;
- _dispatch[Opcode.EmptyTuple] = LoadEmptyTuple;
- _dispatch[Opcode.Ext1] = LoadExt1;
- _dispatch[Opcode.Ext2] = LoadExt2;
- _dispatch[Opcode.Ext4] = LoadExt4;
- _dispatch[Opcode.Float] = LoadFloat;
- _dispatch[Opcode.Get] = LoadGet;
- _dispatch[Opcode.Global] = LoadGlobal;
- _dispatch[Opcode.Inst] = LoadInst;
- _dispatch[Opcode.Int] = LoadInt;
- _dispatch[Opcode.List] = LoadList;
- _dispatch[Opcode.Long] = LoadLong;
- _dispatch[Opcode.Long1] = LoadLong1;
- _dispatch[Opcode.Long4] = LoadLong4;
- _dispatch[Opcode.LongBinGet] = LoadLongBinGet;
- _dispatch[Opcode.LongBinPut] = LoadLongBinPut;
- _dispatch[Opcode.Mark] = LoadMark;
- _dispatch[Opcode.NewFalse] = LoadNewFalse;
- _dispatch[Opcode.NewObj] = LoadNewObj;
- _dispatch[Opcode.NewTrue] = LoadNewTrue;
- _dispatch[Opcode.NoneValue] = LoadNoneValue;
- _dispatch[Opcode.Obj] = LoadObj;
- _dispatch[Opcode.PersId] = LoadPersId;
- _dispatch[Opcode.Pop] = LoadPop;
- _dispatch[Opcode.PopMark] = LoadPopMark;
- _dispatch[Opcode.Proto] = LoadProto;
- _dispatch[Opcode.Put] = LoadPut;
- _dispatch[Opcode.Reduce] = LoadReduce;
- _dispatch[Opcode.SetItem] = LoadSetItem;
- _dispatch[Opcode.SetItems] = LoadSetItems;
- _dispatch[Opcode.ShortBinstring] = LoadShortBinstring;
- _dispatch[Opcode.String] = LoadString;
- _dispatch[Opcode.Tuple] = LoadTuple;
- _dispatch[Opcode.Tuple1] = LoadTuple1;
- _dispatch[Opcode.Tuple2] = LoadTuple2;
- _dispatch[Opcode.Tuple3] = LoadTuple3;
- _dispatch[Opcode.Unicode] = LoadUnicode;
- }
-
- [Documentation("load() -> unpickled object\n\n"
- + "Read pickle data from the file object that was passed to the constructor and\n"
- + "return the corresponding unpickled objects."
- )]
- public object load(CodeContext/*!*/ context) {
- _stack = new List();
-
- string opcode = Read(context, 1);
-
- while (opcode != Opcode.Stop) {
- if (!_dispatch.ContainsKey(opcode)) {
- throw CannotUnpickle(context, "invalid opcode: {0}", PythonOps.Repr(context, opcode));
- }
- _dispatch[opcode](context);
- opcode = Read(context, 1);
- }
-
- return _stack.pop();
- }
-
- [Documentation("noload() -> unpickled object\n\n"
- // 1234567890123456789012345678901234567890123456789012345678901234567890123456789
- + "Like load(), but don't import any modules or create create any instances of\n"
- + "user-defined types. (Builtin objects such as ints, tuples, etc. are created as\n"
- + "with load().)\n"
- + "\n"
- + "This is primarily useful for scanning a pickle for persistent ids without\n"
- + "incurring the overhead of completely unpickling an object. See the pickle\n"
- + "module documentation for more information about persistent ids."
- )]
- public void noload(CodeContext/*!*/ context) {
- throw PythonOps.NotImplementedError("noload() is not implemented");
- }
-
- private Exception CannotUnpickle(CodeContext/*!*/ context, string format, params object[] args) {
- return PythonExceptions.CreateThrowable(UnpicklingError(context), String.Format(format, args));
- }
-
- [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
- public IDictionary<object, object> memo {
- get { return _memo; }
- set { _memo = value; }
- }
-
- public object persistent_load {
- get {
- return _pers_loader;
- }
- set {
- _pers_loader = value;
- }
- }
-
- private object MemoGet(CodeContext/*!*/ context, long key) {
- object value;
- if (_memo.TryGetValue(key, out value)) return value;
- throw PythonExceptions.CreateThrowable(BadPickleGet(context), String.Format("memo key {0} not found", key));
- }
-
- private void MemoPut(long key, object value) {
- _memo[key] = value;
- }
-
- [PropertyMethod, System.Runtime.CompilerServices.SpecialName]
- public int GetMarkIndex(CodeContext/*!*/ context) {
- int i = _stack.__len__() - 1;
- while (i > 0 && _stack[i] != _mark) i -= 1;
- if (i == -1) throw CannotUnpickle(context, "mark not found");
- return i;
- }
-
- private string Read(CodeContext/*!*/ context, int size) {
- string res = _file.Read(context, size);
- if (res.Length < size) {
- throw PythonOps.EofError("unexpected EOF while unpickling");
- }
- return res;
- }
-
- private string ReadLineNoNewline(CodeContext/*!*/ context) {
- string raw = _file.ReadLine(context);
- return raw.Substring(0, raw.Length - 1);
- }
-
- private object ReadFloatString(CodeContext/*!*/ context) {
- return DoubleOps.__new__(context, TypeCache.Double, ReadLineNoNewline(context));
- }
-
- private double ReadFloat64(CodeContext/*!*/ context) {
- int index = 0;
- return PythonStruct.CreateDoubleValue(context, ref index, false, Read(context, 8));
- }
-
- private object ReadIntFromString(CodeContext/*!*/ context) {
- string raw = ReadLineNoNewline(context);
- if ("00" == raw) return ScriptingRuntimeHelpers.False;
- else if ("01" == raw) return ScriptingRuntimeHelpers.True;
- return Int32Ops.__new__(context, TypeCache.Int32, raw);
- }
-
- private int ReadInt32(CodeContext/*!*/ context) {
- int index = 0;
- return PythonStruct.CreateIntValue(context, ref index, true, Read(context, 4));
- }
-
- private object ReadLongFromString(CodeContext/*!*/ context) {
- return BigIntegerOps.__new__(context, TypeCache.BigInteger, ReadLineNoNewline(context));
- }
-
- private object ReadLong(CodeContext/*!*/ context, int size) {
- return new BigInteger(Read(context, size).MakeByteArray());
- }
-
- private char ReadUInt8(CodeContext/*!*/ context) {
- int index = 0;
- return PythonStruct.CreateCharValue(context, ref index, Read(context, 1));
- }
-
- private ushort ReadUInt16(CodeContext/*!*/ context) {
- int index = 0;
- return PythonStruct.CreateUShortValue(context, ref index, true, Read(context, 2));
- }
-
- public object find_global(CodeContext/*!*/ context, object module, object attr) {
- object moduleObject;
- if (!Importer.TryGetExistingModule(context, Converter.ConvertToString(module), out moduleObject)) {
- Builtin.__import__(context, Converter.ConvertToString(module));
- moduleObject = PythonContext.GetContext(context).SystemStateModules[module];
- }
- return PythonOps.GetBoundAttr(context, moduleObject, Converter.ConvertToString(attr));
- }
-
- private object MakeInstance(CodeContext/*!*/ context, object cls, object[] args) {
- OldClass oc = cls as OldClass;
- if (oc != null) {
- OldInstance inst = new OldInstance(context, oc);
- if (args.Length != 0 || PythonOps.HasAttr(context, cls, "__getinitargs__")) {
- PythonOps.CallWithContext(context, PythonOps.GetBoundAttr(context, inst, "__init__"), args);
- }
- return inst;
- }
- return PythonOps.CallWithContext(context, cls, args);
- }
-
- private void PopMark(int markIndex) {
- _stack.__delslice__(markIndex, _stack.__len__());
- }
-
- /// <summary>
- /// Interpret everything from markIndex to the top of the stack as a sequence
- /// of key, value, key, value, etc. Set dict[key] = value for each. Pop
- /// everything from markIndex up when done.
- /// </summary>
- private void SetItems(PythonDictionary dict, int markIndex) {
- for (int i = markIndex + 1; i < _stack.__len__(); i += 2) {
- dict[_stack[i]] = _stack[i+1];
- }
- PopMark(markIndex);
- }
-
- private void LoadEof(CodeContext/*!*/ context) {
- throw PythonOps.EofError("unexpected end of opcode stream");
- }
-
- private void LoadAppend(CodeContext/*!*/ context) {
- object item = _stack.pop();
- object seq = _stack[-1];
- if (seq is List) {
- ((List)seq).append(item);
- } else {
- PythonCalls.Call(context, PythonOps.GetBoundAttr(context, seq, "append"), item);
- }
- }
-
- private void LoadAppends(CodeContext/*!*/ context) {
- int markIndex = GetMarkIndex(context);
- object seq = _stack[markIndex - 1];
- object stackSlice = _stack.__getslice__(markIndex + 1, _stack.__len__());
- if (seq is List) {
- ((List)seq).extend(stackSlice);
- } else {
- PythonOps.CallWithContext(context, PythonOps.GetBoundAttr(context, seq, "extend"), stackSlice);
- }
- PopMark(markIndex);
- }
-
- private void LoadBinFloat(CodeContext/*!*/ context) {
- _stack.append(ReadFloat64(context));
- }
-
- private void LoadBinGet(CodeContext/*!*/ context) {
- _stack.append(MemoGet(context, (long)ReadUInt8(context)));
- }
-
- private void LoadBinInt(CodeContext/*!*/ context) {
- _stack.append(ReadInt32(context));
- }
-
- private void LoadBinInt1(CodeContext/*!*/ context) {
- _stack.append((int)ReadUInt8(context));
- }
-
- private void LoadBinInt2(CodeContext/*!*/ context) {
- _stack.append((int)ReadUInt16(context));
- }
-
- private void LoadBinPersid(CodeContext/*!*/ context) {
- if (_pers_loader == null) throw CannotUnpickle(context, "cannot unpickle binary persistent ID w/o persistent_load");
-
- _stack.append(PythonContext.GetContext(context).CallSplat(_pers_loader, _stack.pop()));
- }
-
- private void LoadBinPut(CodeContext/*!*/ context) {
- MemoPut((long)ReadUInt8(context), _stack[-1]);
- }
-
- private void LoadBinString(CodeContext/*!*/ context) {
- _stack.append(Read(context, ReadInt32(context)));
- }
-
- private void LoadBinUnicode(CodeContext/*!*/ context) {
- _stack.append(StringOps.decode(context, Read(context, ReadInt32(context)), "utf-8", "strict"));
- }
-
- private void LoadBuild(CodeContext/*!*/ context) {
- object arg = _stack.pop();
- object inst = _stack[-1];
- object setStateCallable;
- if (PythonOps.TryGetBoundAttr(context, inst, "__setstate__", out setStateCallable)) {
- PythonOps.CallWithContext(context, setStateCallable, arg);
- return;
- }
-
- PythonDictionary dict;
- PythonDictionary slots;
- if (arg == null) {
- dict = null;
- slots = null;
- } else if (arg is PythonDictionary) {
- dict = (PythonDictionary)arg;
- slots = null;
- } else if (arg is PythonTuple) {
- PythonTuple argsTuple = (PythonTuple)arg;
- if (argsTuple.__len__() != 2) {
- throw PythonOps.ValueError("state for object without __setstate__ must be None, dict, or 2-tuple");
- }
- dict = (PythonDictionary)argsTuple[0];
- slots = (PythonDictionary)argsTuple[1];
- } else {
- throw PythonOps.ValueError("state for object without __setstate__ must be None, dict, or 2-tuple");
- }
-
- if (dict != null) {
- object instDict;
- if (PythonOps.TryGetBoundAttr(context, inst, "__dict__", out instDict)) {
- PythonDictionary realDict = instDict as PythonDictionary;
- if (realDict != null) {
- realDict.update(context, dict);
- } else {
- object updateCallable;
- if (PythonOps.TryGetBoundAttr(context, instDict, "update", out updateCallable)) {
- PythonOps.CallWithContext(context, updateCallable, dict);
- } else {
- throw CannotUnpickle(context, "could not update __dict__ {0} when building {1}", dict, inst);
- }
- }
- }
- }
-
- if (slots != null) {
- foreach(object key in (IEnumerable)slots) {
- PythonOps.SetAttr(context, inst, (string)key, slots[key]);
- }
- }
- }
-
- private void LoadDict(CodeContext/*!*/ context) {
- int markIndex = GetMarkIndex(context);
- PythonDictionary dict = new PythonDictionary((_stack.__len__() - 1 - markIndex) / 2);
- SetItems(dict, markIndex);
- _stack.append(dict);
- }
-
- private void LoadDup(CodeContext/*!*/ context) {
- _stack.append(_stack[-1]);
- }
-
- private void LoadEmptyDict(CodeContext/*!*/ context) {
- _stack.append(new PythonDictionary());
- }
-
- private void LoadEmptyList(CodeContext/*!*/ context) {
- _stack.append(PythonOps.MakeList());
- }
-
- private void LoadEmptyTuple(CodeContext/*!*/ context) {
- _stack.append(PythonTuple.MakeTuple());
- }
-
- private void LoadExt1(CodeContext/*!*/ context) {
- PythonTuple global = (PythonTuple)PythonCopyReg.GetInvertedRegistry(context)[(int)ReadUInt8(context)];
- _stack.append(find_global(context, global[0], global[1]));
- }
-
- private void LoadExt2(CodeContext/*!*/ context) {
- PythonTuple global = (PythonTuple)PythonCopyReg.GetInvertedRegistry(context)[(int)ReadUInt16(context)];
- _stack.append(find_global(context, global[0], global[1]));
- }
-
- private void LoadExt4(CodeContext/*!*/ context) {
- PythonTuple global = (PythonTuple)PythonCopyReg.GetInvertedRegistry(context)[ReadInt32(context)];
- _stack.append(find_global(context, global[0], global[1]));
- }
-
- private void LoadFloat(CodeContext/*!*/ context) {
- _stack.append(ReadFloatString(context));
- }
-
- private void LoadGet(CodeContext/*!*/ context) {
- try {
- _stack.append(MemoGet(context, (long)(int)ReadIntFromString(context)));
- } catch (ArgumentException) {
- throw PythonExceptions.CreateThrowable(BadPickleGet(context), "while executing GET: invalid integer value");
- }
- }
-
- private void LoadGlobal(CodeContext/*!*/ context) {
- string module = ReadLineNoNewline(context);
- string attr = ReadLineNoNewline(context);
- _stack.append(find_global(context, module, attr));
- }
-
- private void LoadInst(CodeContext/*!*/ context) {
- LoadGlobal(context);
- object cls = _stack.pop();
- if (cls is OldClass || cls is PythonType) {
- int markIndex = GetMarkIndex(context);
- object[] args = _stack.GetSliceAsArray(markIndex + 1, _stack.__len__());
- PopMark(markIndex);
-
- _stack.append(MakeInstance(context, cls, args));
- } else {
- throw PythonOps.TypeError("expected class or type after INST, got {0}", DynamicHelpers.GetPythonType(cls));
- }
- }
-
- private void LoadInt(CodeContext/*!*/ context) {
- _stack.append(ReadIntFromString(context));
- }
-
- private void LoadList(CodeContext/*!*/ context) {
- int markIndex = GetMarkIndex(context);
- object list = _stack.__getslice__(markIndex + 1, _stack.__len__());
- PopMark(markIndex);
- _stack.append(list);
- }
-
- private void LoadLong(CodeContext/*!*/ context) {
- _stack.append(ReadLongFromString(context));
- }
-
- private void LoadLong1(CodeContext/*!*/ context) {
- _stack.append(ReadLong(context, ReadUInt8(context)));
- }
-
- private void LoadLong4(CodeContext/*!*/ context) {
- _stack.append(ReadLong(context, ReadInt32(context)));
- }
-
- private void LoadLongBinGet(CodeContext/*!*/ context) {
- _stack.append(MemoGet(context, (long)(int)ReadInt32(context)));
- }
-
- private void LoadLongBinPut(CodeContext/*!*/ context) {
- MemoPut((long)(int)ReadInt32(context), _stack[-1]);
- }
-
- private void LoadMark(CodeContext/*!*/ context) {
- _stack.append(_mark);
- }
-
- private void LoadNewFalse(CodeContext/*!*/ context) {
- _stack.append(ScriptingRuntimeHelpers.False);
- }
-
- private void LoadNewObj(CodeContext/*!*/ context) {
- PythonTuple args = _stack.pop() as PythonTuple;
- if (args == null) {
- throw PythonOps.TypeError("expected tuple as second argument to NEWOBJ, got {0}", DynamicHelpers.GetPythonType(args));
- }
-
- PythonType cls = _stack.pop() as PythonType;
- if (args == null) {
- throw PythonOps.TypeError("expected new-style type as first argument to NEWOBJ, got {0}", DynamicHelpers.GetPythonType(args));
- }
-
- PythonTypeSlot dts;
- object value;
- if (cls.TryResolveSlot(context, "__new__", out dts) &&
- dts.TryGetValue(context, null, cls, out value)) {
- object[] newargs = new object[args.__len__() + 1];
- ((ICollection)args).CopyTo(newargs, 1);
- newargs[0] = cls;
-
- _stack.append(PythonOps.CallWithContext(context, value, newargs));
- return;
- }
-
- throw PythonOps.TypeError("didn't find __new__");
- }
-
- private void LoadNewTrue(CodeContext/*!*/ context) {
- _stack.append(ScriptingRuntimeHelpers.True);
- }
-
- private void LoadNoneValue(CodeContext/*!*/ context) {
- _stack.append(null);
- }
-
- private void LoadObj(CodeContext/*!*/ context) {
- int markIndex = GetMarkIndex(context);
- if ((markIndex + 1) >= _stack.Count) {
- throw PythonExceptions.CreateThrowable(UnpicklingError(context), "could not find MARK");
- }
-
- object cls = _stack[markIndex + 1];
- if (cls is OldClass || cls is PythonType) {
- object[] args = _stack.GetSliceAsArray(markIndex + 2, _stack.__len__());
- PopMark(markIndex);
- _stack.append(MakeInstance(context, cls, args));
- } else {
- throw PythonOps.TypeError("expected class or type as first argument to INST, got {0}", DynamicHelpers.GetPythonType(cls));
- }
- }
-
- private void LoadPersId(CodeContext/*!*/ context) {
- if (_pers_loader == null) {
- throw CannotUnpickle(context, "A load persistent ID instruction is present but no persistent_load function is available");
- }
- _stack.append(PythonContext.GetContext(context).CallSplat(_pers_loader, ReadLineNoNewline(context)));
- }
-
- private void LoadPop(CodeContext/*!*/ context) {
- _stack.pop();
- }
-
- private void LoadPopMark(CodeContext/*!*/ context) {
- PopMark(GetMarkIndex(context));
- }
-
- private void LoadProto(CodeContext/*!*/ context) {
- int proto = ReadUInt8(context);
- if (proto > 2) throw PythonOps.ValueError("unsupported pickle protocol: {0}", proto);
- // discard result
- }
-
- private void LoadPut(CodeContext/*!*/ context) {
- MemoPut((long)(int)ReadIntFromString(context), _stack[-1]);
- }
-
- private void LoadReduce(CodeContext/*!*/ context) {
- object args = _stack.pop();
- object callable = _stack.pop();
- if (args == null) {
- _stack.append(PythonCalls.Call(context, PythonOps.GetBoundAttr(context, callable, "__basicnew__")));
- } else if (!DynamicHelpers.GetPythonType(args).Equals(TypeCache.PythonTuple)) {
- throw PythonOps.TypeError(
- "while executing REDUCE, expected tuple at the top of the stack, but got {0}",
- DynamicHelpers.GetPythonType(args)
- );
- }
- _stack.append(PythonOps.CallWithArgsTupleAndContext(context, callable, ArrayUtils.EmptyObjects, args));
- }
-
- private void LoadSetItem(CodeContext/*!*/ context) {
- object value = _stack.pop();
- object key = _stack.pop();
- PythonDictionary dict = _stack[-1] as PythonDictionary;
- if (dict == null) {
- throw PythonOps.TypeError(
- "while executing SETITEM, expected dict at stack[-3], but got {0}",
- DynamicHelpers.GetPythonType(_stack[-1])
- );
- }
- dict[key] = value;
- }
-
- private void LoadSetItems(CodeContext/*!*/ context) {
- int markIndex = GetMarkIndex(context);
- PythonDictionary dict = _stack[markIndex - 1] as PythonDictionary;
- if (dict == null) {
- throw PythonOps.TypeError(
- "while executing SETITEMS, expected dict below last mark, but got {0}",
- DynamicHelpers.GetPythonType(_stack[markIndex - 1])
- );
- }
- SetItems(dict, markIndex);
- }
-
- private void LoadShortBinstring(CodeContext/*!*/ context) {
- _stack.append(Read(context, ReadUInt8(context)));
- }
-
- private void LoadString(CodeContext/*!*/ context) {
- string repr = ReadLineNoNewline(context);
- if (repr.Length < 2 ||
- !(
- repr[0] == '"' && repr[repr.Length - 1] == '"' ||
- repr[0] == '\'' && repr[repr.Length - 1] == '\''
- )
- ) {
- throw PythonOps.ValueError("while executing STRING, expected string that starts and ends with quotes");
- }
- _stack.append(StringOps.decode(context, repr.Substring(1, repr.Length - 2), "string-escape", "strict"));
- }
-
- private void LoadTuple(CodeContext/*!*/ context) {
- int markIndex = GetMarkIndex(context);
- PythonTuple tuple = PythonTuple.MakeTuple(_stack.GetSliceAsArray(markIndex + 1, _stack.__len__()));
- PopMark(markIndex);
- _stack.append(tuple);
- }
-
- private void LoadTuple1(CodeContext/*!*/ context) {
- object item0 = _stack.pop();
- _stack.append(PythonTuple.MakeTuple(item0));
- }
-
- private void LoadTuple2(CodeContext/*!*/ context) {
- object item1 = _stack.pop();
- object item0 = _stack.pop();
- _stack.append(PythonTuple.MakeTuple(item0, item1));
- }
-
- private void LoadTuple3(CodeContext/*!*/ context) {
- object item2 = _stack.pop();
- object item1 = _stack.pop();
- object item0 = _stack.pop();
- _stack.append(PythonTuple.MakeTuple(item0, item1, item2));
- }
-
- private void LoadUnicode(CodeContext/*!*/ context) {
- _stack.append(StringOps.decode(context, ReadLineNoNewline(context), "raw-unicode-escape", "strict"));
- }
- }
-
- #endregion
-
- private static PythonType PicklingError(CodeContext/*!*/ context) {
- return (PythonType)PythonContext.GetContext(context).GetModuleState("PicklingError");
- }
-
- private static PythonType PickleError(CodeContext/*!*/ context) {
- return (PythonType)PythonContext.GetContext(context).GetModuleState("PickleError");
- }
-
- private static PythonType UnpicklingError(CodeContext/*!*/ context) {
- return (PythonType)PythonContext.GetContext(context).GetModuleState("UnpicklingError");
- }
-
- private static PythonType BadPickleGet(CodeContext/*!*/ context) {
- return (PythonType)PythonContext.GetContext(context).GetModuleState("BadPickleGet");
- }
- }
- }