/Languages/IronPython/IronPython.Modules/cPickle.cs
C# | 2389 lines | 2156 code | 155 blank | 78 comment | 223 complexity | a3c6fcc07ac2da828908469cc2621697 MD5 | raw file
Possible License(s): CPL-1.0, BSD-3-Clause, ISC, GPL-2.0, MPL-2.0-no-copyleft-exception
Large files files are truncated, but you can click here to view the full file
- /* ****************************************************************************
- *
- * Copyright (c) Microsoft Corporation.
- *
- * This source code is subject to terms and conditions of the Apache License, Version 2.0. A
- * copy of the license can be found in the License.html file at the root of this distribution. If
- * you cannot locate the Apache License, Version 2.0, please send an email to
- * ironpy@microsoft.com. By using this source code in any fashion, you are agreeing to be bound
- * by the terms of the Apache License, Version 2.0.
- *
- * You must not remove this notice, or any other, from this software.
- *
- *
- * ***************************************************************************/
- using System;
- using System.Collections;
- using System.Collections.Generic;
- using System.Diagnostics;
- using System.IO;
- using System.Runtime.InteropServices;
- using System.Text;
- using IronPython.Runtime;
- using IronPython.Runtime.Exceptions;
- using IronPython.Runtime.Operations;
- using IronPython.Runtime.Types;
- using Microsoft.Scripting;
- using Microsoft.Scripting.Runtime;
- using Microsoft.Scripting.Utils;
- #if FEATURE_NUMERICS
- using System.Numerics;
- #else
- using Microsoft.Scripting.Math;
- #endif
- [assembly: PythonModule("cPickle", typeof(IronPython.Modules.PythonPickle))]
- namespace IronPython.Modules {
- public static class PythonPickle {
- public const string __doc__ = "Fast object serialization/deserialization.\n\n"
- + "Differences from CPython:\n"
- + " - does not implement the undocumented fast mode\n";
- [System.Runtime.CompilerServices.SpecialName]
- public static void PerformModuleReload(PythonContext/*!*/ context, PythonDictionary/*!*/ dict) {
- context.EnsureModuleException("PickleError", dict, "PickleError", "cPickle");
- context.EnsureModuleException("PicklingError", dict, "PicklingError", "cPickle");
- context.EnsureModuleException("UnpicklingError", dict, "UnpicklingError", "cPickle");
- context.EnsureModuleException("UnpickleableError", dict, "UnpickleableError", "cPickle");
- context.EnsureModuleException("BadPickleGet", dict, "BadPickleGet", "cPickle");
- dict["__builtins__"] = context.BuiltinModuleInstance;
- dict["compatible_formats"] = PythonOps.MakeList("1.0", "1.1", "1.2", "1.3", "2.0");
- }
- private static readonly PythonStruct.Struct _float64 = PythonStruct.Struct.Create(">d");
-
- private const int highestProtocol = 2;
-
- public const string __version__ = "1.71";
- public const string format_version = "2.0";
-
- public static int HIGHEST_PROTOCOL {
- get { return highestProtocol; }
- }
-
- private const string Newline = "\n";
- #region Public module-level functions
- [Documentation("dump(obj, file, protocol=0) -> None\n\n"
- + "Pickle obj and write the result to file.\n"
- + "\n"
- + "See documentation for Pickler() for a description the file, protocol, and\n"
- + "(deprecated) bin parameters."
- )]
- public static void dump(CodeContext/*!*/ context, object obj, object file, [DefaultParameterValue(null)] object protocol, [DefaultParameterValue(null)] object bin) {
- PicklerObject/*!*/ pickler = new PicklerObject(context, file, protocol, bin);
- pickler.dump(context, obj);
- }
- [Documentation("dumps(obj, protocol=0) -> pickle string\n\n"
- + "Pickle obj and return the result as a string.\n"
- + "\n"
- + "See the documentation for Pickler() for a description of the protocol and\n"
- + "(deprecated) bin parameters."
- )]
- public static string dumps(CodeContext/*!*/ context, object obj, [DefaultParameterValue(null)] object protocol, [DefaultParameterValue(null)] object bin) {
- //??? possible perf enhancement: use a C# TextWriter-backed IFileOutput and
- // thus avoid Python call overhead. Also do similar thing for LoadFromString.
- var stringIO = new StringBuilderOutput();
- PicklerObject/*!*/ pickler = new PicklerObject(context, stringIO, protocol, bin);
- pickler.dump(context, obj);
- return stringIO.GetString();
- }
- [Documentation("load(file) -> unpickled object\n\n"
- + "Read pickle data from the open file object and return the corresponding\n"
- + "unpickled object. Data after the first pickle found is ignored, but the file\n"
- + "cursor is not reset, so if a file objects contains multiple pickles, then\n"
- + "load() may be called multiple times to unpickle them.\n"
- + "\n"
- + "file: an object (such as an open file or a StringIO) with read(num_chars) and\n"
- + " readline() methods that return strings\n"
- + "\n"
- + "load() automatically determines if the pickle data was written in binary or\n"
- + "text mode."
- )]
- public static object load(CodeContext/*!*/ context, object file) {
- return new UnpicklerObject(context, file).load(context);
- }
- [Documentation("loads(string) -> unpickled object\n\n"
- + "Read a pickle object from a string, unpickle it, and return the resulting\n"
- + "reconstructed object. Characters in the string beyond the end of the first\n"
- + "pickle are ignored."
- )]
- public static object loads(CodeContext/*!*/ context, [BytesConversion]string @string) {
- return new UnpicklerObject(context, new PythonStringInput(@string)).load(context);
- }
- #endregion
- #region File I/O wrappers
- /// <summary>
- /// Interface for "file-like objects" that implement the protocol needed by load() and friends.
- /// This enables the creation of thin wrappers that make fast .NET types and slow Python types look the same.
- /// </summary>
- internal abstract class FileInput {
- public abstract string Read(CodeContext/*!*/ context, int size);
- public abstract string ReadLine(CodeContext/*!*/ context);
- public virtual string ReadLineNoNewLine(CodeContext/*!*/ context) {
- var raw = ReadLine(context);
- return raw.Substring(0, raw.Length - 1);
- }
- public virtual char ReadChar(CodeContext context) {
- string res = Read(context, 1);
- if (res.Length < 1) {
- throw PythonOps.EofError("unexpected EOF while unpickling");
- }
- return res[0];
- }
- public virtual int ReadInt(CodeContext context) {
- return (int)ReadChar(context) |
- ((int)ReadChar(context)) << 8 |
- ((int)ReadChar(context)) << 16 |
- ((int)ReadChar(context)) << 24;
- }
- }
- /// <summary>
- /// Interface for "file-like objects" that implement the protocol needed by dump() and friends.
- /// This enables the creation of thin wrappers that make fast .NET types and slow Python types look the same.
- /// </summary>
- internal abstract class FileOutput {
- private readonly char[] int32chars = new char[4];
- public abstract void Write(CodeContext/*!*/ context, string data);
- public virtual void Write(CodeContext context, int data) {
- int32chars[0] = (char)(int)((data & 0xff));
- int32chars[1] = (char)(int)((data >> 8) & 0xff);
- int32chars[2] = (char)(int)((data >> 16) & 0xff);
- int32chars[3] = (char)(int)((data >> 24) & 0xff);
- Write(context, new string(int32chars));
- }
- public virtual void Write(CodeContext context, char data) {
- Write(context, ScriptingRuntimeHelpers.CharToString(data));
- }
- }
- private class PythonFileInput : FileInput {
- private object _readMethod;
- private object _readLineMethod;
- public PythonFileInput(CodeContext/*!*/ context, object file) {
- if (!PythonOps.TryGetBoundAttr(context, file, "read", out _readMethod) ||
- !PythonOps.IsCallable(context, _readMethod) ||
- !PythonOps.TryGetBoundAttr(context, file, "readline", out _readLineMethod) ||
- !PythonOps.IsCallable(context, _readLineMethod)
- ) {
- throw PythonOps.TypeError("argument must have callable 'read' and 'readline' attributes");
- }
- }
- public override string Read(CodeContext/*!*/ context, int size) {
- return Converter.ConvertToString(PythonCalls.Call(context, _readMethod, size));
- }
- public override string ReadLine(CodeContext/*!*/ context) {
- return Converter.ConvertToString(PythonCalls.Call(context, _readLineMethod));
- }
- }
- internal class PythonStringInput : FileInput {
- private readonly string _data;
- int _offset;
- public PythonStringInput(string data) {
- _data = data;
- }
- public override string Read(CodeContext context, int size) {
- var res = _data.Substring(_offset, size);
- _offset += size;
- return res;
- }
- public override string ReadLine(CodeContext context) {
- return ReadLineWorker(true);
- }
- public override string ReadLineNoNewLine(CodeContext context) {
- return ReadLineWorker(false);
- }
- public override char ReadChar(CodeContext context) {
- if (_offset < _data.Length) {
- return _data[_offset++];
- }
- throw PythonOps.EofError("unexpected EOF while unpickling");
- }
- public override int ReadInt(CodeContext context) {
- if (_offset + 4 <= _data.Length) {
- int res = _data[_offset]|
- ((int)_data[_offset + 1]) << 8 |
- ((int)_data[_offset + 2]) << 16 |
- ((int)_data[_offset + 3]) << 24;
- _offset += 4;
- return res;
- }
- throw PythonOps.EofError("unexpected EOF while unpickling");
- }
- private string ReadLineWorker(bool includeNewLine) {
- string res;
- for (int i = _offset; i < _data.Length; i++) {
- if (_data[i] == '\n') {
- res = _data.Substring(_offset, i - _offset + (includeNewLine ? 1 : 0));
- _offset = i + 1;
- return res;
- }
- }
- res = _data.Substring(_offset);
- _offset = _data.Length;
- return res;
- }
- }
- private class PythonFileLikeOutput : FileOutput {
- private object _writeMethod;
- public PythonFileLikeOutput(CodeContext/*!*/ context, object file) {
- if (!PythonOps.TryGetBoundAttr(context, file, "write", out _writeMethod) ||
- !PythonOps.IsCallable(context, this._writeMethod)
- ) {
- throw PythonOps.TypeError("argument must have callable 'write' attribute");
- }
- }
- public override void Write(CodeContext/*!*/ context, string data) {
- PythonCalls.Call(context, _writeMethod, data);
- }
- }
- private class PythonFileOutput : FileOutput {
- private readonly PythonFile _file;
- public PythonFileOutput(PythonFile file) {
- _file = file;
- }
- public override void Write(CodeContext/*!*/ context, string data) {
- _file.write(data);
- }
- }
- private class StringBuilderOutput : FileOutput {
- private readonly StringBuilder _builder = new StringBuilder(4096);
- public string GetString() {
- return _builder.ToString();
- }
- public override void Write(CodeContext context, char data) {
- _builder.Append(data);
- }
- public override void Write(CodeContext context, int data) {
- _builder.Append((char)(int)((data) & 0xff));
- _builder.Append((char)(int)((data >> 8) & 0xff));
- _builder.Append((char)(int)((data >> 16) & 0xff));
- _builder.Append((char)(int)((data >> 24) & 0xff));
- }
- public override void Write(CodeContext context, string data) {
- _builder.Append(data);
- }
- }
- private class PythonReadableFileOutput : PythonFileLikeOutput {
- private object _getValueMethod;
- public PythonReadableFileOutput(CodeContext/*!*/ context, object file)
- : base(context, file) {
- if (!PythonOps.TryGetBoundAttr(context, file, "getvalue", out _getValueMethod) ||
- !PythonOps.IsCallable(context, _getValueMethod)
- ) {
- throw PythonOps.TypeError("argument must have callable 'getvalue' attribute");
- }
- }
- public object GetValue(CodeContext/*!*/ context) {
- return PythonCalls.Call(context, _getValueMethod);
- }
- }
- #endregion
- #region Opcode constants
- internal static class Opcode {
- public const char Append = 'a';
- public const char Appends = 'e';
- public const char BinFloat = 'G';
- public const char BinGet = 'h';
- public const char BinInt = 'J';
- public const char BinInt1 = 'K';
- public const char BinInt2 = 'M';
- public const char BinPersid = 'Q';
- public const char BinPut = 'q';
- public const char BinString = 'T';
- public const char BinUnicode = 'X';
- public const char Build = 'b';
- public const char Dict = 'd';
- public const char Dup = '2';
- public const char EmptyDict = '}';
- public const char EmptyList = ']';
- public const char EmptyTuple = ')';
- public const char Ext1 = '\x82';
- public const char Ext2 = '\x83';
- public const char Ext4 = '\x84';
- public const char Float = 'F';
- public const char Get = 'g';
- public const char Global = 'c';
- public const char Inst = 'i';
- public const char Int = 'I';
- public const char List = 'l';
- public const char Long = 'L';
- public const char Long1 = '\x8a';
- public const char Long4 = '\x8b';
- public const char LongBinGet = 'j';
- public const char LongBinPut = 'r';
- public const char Mark = '(';
- public const char NewFalse = '\x89';
- public const char NewObj = '\x81';
- public const char NewTrue = '\x88';
- public const char NoneValue = 'N';
- public const char Obj = 'o';
- public const char PersId = 'P';
- public const char Pop = '0';
- public const char PopMark = '1';
- public const char Proto = '\x80';
- public const char Put = 'p';
- public const char Reduce = 'R';
- public const char SetItem = 's';
- public const char SetItems = 'u';
- public const char ShortBinstring = 'U';
- public const char Stop = '.';
- public const char String = 'S';
- public const char Tuple = 't';
- public const char Tuple1 = '\x85';
- public const char Tuple2 = '\x86';
- public const char Tuple3 = '\x87';
- public const char Unicode = 'V';
- }
- #endregion
- #region Pickler object
- public static PicklerObject/*!*/ Pickler(CodeContext/*!*/ context, [DefaultParameterValue(null)]object file, [DefaultParameterValue(null)]object protocol, [DefaultParameterValue(null)]object bin) {
- return new PicklerObject(context, file, protocol, bin);
- }
- [Documentation("Pickler(file, protocol=0) -> Pickler object\n\n"
- + "A Pickler object serializes Python objects to a pickle bytecode stream, which\n"
- + "can then be converted back into equivalent objects using an Unpickler.\n"
- + "\n"
- + "file: an object (such as an open file) that has a write(string) method.\n"
- + "protocol: if omitted, protocol 0 is used. If HIGHEST_PROTOCOL or a negative\n"
- + " number, the highest available protocol is used.\n"
- + "bin: (deprecated; use protocol instead) for backwards compability, a 'bin'\n"
- + " keyword parameter is supported. When protocol is specified it is ignored.\n"
- + " If protocol is not specified, then protocol 0 is used if bin is false, and\n"
- + " protocol 1 is used if bin is true."
- )]
- [PythonType("Pickler"), PythonHidden]
- public class PicklerObject {
- private const char LowestPrintableChar = (char)32;
- private const char HighestPrintableChar = (char)126;
- // max elements that can be set/appended at a time using SETITEMS/APPENDS
- private delegate void PickleFunction(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object value);
- private static readonly Dictionary<Type, PickleFunction> _dispatchTable;
- private int _batchSize = 1000;
- private FileOutput _file;
- private int _protocol;
- private PythonDictionary _memo; // memo if the user accesses the memo property
- private Dictionary<object, int> _privMemo; // internal fast memo which we can use if the user doesn't access memo
- private object _persist_id;
- static PicklerObject() {
- _dispatchTable = new Dictionary<Type, PickleFunction>();
- _dispatchTable[typeof(PythonDictionary)] = SaveDict;
- _dispatchTable[typeof(PythonTuple)] = SaveTuple;
- _dispatchTable[typeof(List)] = SaveList;
- _dispatchTable[typeof(OldClass)] = SaveGlobal;
- _dispatchTable[typeof(PythonFunction)] = SaveGlobal;
- _dispatchTable[typeof(BuiltinFunction)] = SaveGlobal;
- _dispatchTable[typeof(PythonType)] = SaveGlobal;
- _dispatchTable[typeof(OldInstance)] = SaveInstance;
- }
- #region Public API
- [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
- public PythonDictionary memo {
- get {
- if (_memo == null) {
- // create publicly viewable memo
- PythonDictionary resMemo = new PythonDictionary();
- foreach (var v in _privMemo) {
- resMemo._storage.AddNoLock(
- ref resMemo._storage,
- Builtin.id(v.Key),
- PythonTuple.MakeTuple(v.Value, v.Key)
- );
- }
- _memo = resMemo;
- }
- return _memo;
- }
- set {
- _memo = value;
- _privMemo = null;
- }
- }
- public int proto {
- get { return _protocol; }
- set { _protocol = value; }
- }
- public int _BATCHSIZE {
- get { return _batchSize; }
- set { _batchSize = value; }
- }
- public object persistent_id {
- get {
- return _persist_id;
- }
- set {
- _persist_id = value;
- }
- }
- public int binary {
- get { return _protocol == 0 ? 1 : 0; }
- set { _protocol = value; }
- }
- public int fast {
- // We don't implement fast, but we silently ignore it when it's set so that test_cpickle works.
- // For a description of fast, see http://mail.python.org/pipermail/python-bugs-list/2001-October/007695.html
- get { return 0; }
- set { /* ignore */ }
- }
- public PicklerObject(CodeContext/*!*/ context, object file, object protocol, object bin) {
- int intProtocol;
- if (file == null) {
- _file = new PythonReadableFileOutput(context, new PythonStringIO.StringO());
- } else if (Converter.TryConvertToInt32(file, out intProtocol)) {
- // For undocumented (yet tested in official CPython tests) list-based pickler, the
- // user could do something like Pickler(1), which would create a protocol-1 pickler
- // with an internal string output buffer (retrievable using getvalue()). For a little
- // more info, see
- // https://sourceforge.net/tracker/?func=detail&atid=105470&aid=939395&group_id=5470
- _file = new PythonReadableFileOutput(context, new PythonStringIO.StringO());
- protocol = file;
- } else if (file is PythonFile) {
- _file = new PythonFileOutput((PythonFile)file);
- } else if (file is FileOutput) {
- _file = (FileOutput)file;
- } else {
- _file = new PythonFileLikeOutput(context, file);
- }
- _privMemo = new Dictionary<object, int>(256, ReferenceEqualityComparer.Instance);
- if (protocol == null) protocol = PythonOps.IsTrue(bin) ? 1 : 0;
- intProtocol = context.LanguageContext.ConvertToInt32(protocol);
- if (intProtocol > highestProtocol) {
- throw PythonOps.ValueError("pickle protocol {0} asked for; the highest available protocol is {1}", intProtocol, highestProtocol);
- } else if (intProtocol < 0) {
- this._protocol = highestProtocol;
- } else {
- this._protocol = intProtocol;
- }
- }
- [Documentation("dump(obj) -> None\n\n"
- + "Pickle obj and write the result to the file object that was passed to the\n"
- + "constructor\n."
- + "\n"
- + "Note that you may call dump() multiple times to pickle multiple objects. To\n"
- + "unpickle the stream, you will need to call Unpickler's load() method a\n"
- + "corresponding number of times.\n"
- + "\n"
- + "The first time a particular object is encountered, it will be pickled normally.\n"
- + "If the object is encountered again (in the same or a later dump() call), a\n"
- + "reference to the previously generated value will be pickled. Unpickling will\n"
- + "then create multiple references to a single object."
- )]
- public void dump(CodeContext/*!*/ context, object obj) {
- if (_protocol >= 2) WriteProto(context);
- Save(context, obj);
- Write(context, Opcode.Stop);
- }
- [Documentation("clear_memo() -> None\n\n"
- + "Clear the memo, which is used internally by the pickler to keep track of which\n"
- + "objects have already been pickled (so that shared or recursive objects are\n"
- + "pickled only once)."
- )]
- public void clear_memo() {
- if (_memo != null) {
- _memo.Clear();
- } else {
- _privMemo.Clear();
- }
- }
- private void Memoize(object obj) {
- if (_memo != null) {
- if (!MemoContains(PythonOps.Id(obj))) {
- _memo[PythonOps.Id(obj)] = PythonTuple.MakeTuple(_memo.Count, obj);
- }
- } else {
- if(!_privMemo.ContainsKey(obj)) {
- _privMemo[obj] = _privMemo.Count;
- }
- }
- }
- private int MemoizeNew(object obj) {
-
- int res;
- if (_memo != null) {
- Debug.Assert(!_memo.ContainsKey(obj));
- _memo[PythonOps.Id(obj)] = PythonTuple.MakeTuple(res = _memo.Count, obj);
- } else {
- Debug.Assert(!_privMemo.ContainsKey(obj));
- _privMemo[obj] = res = _privMemo.Count;
- }
- return res;
- }
- private bool MemoContains(object obj) {
- if (_memo != null) {
- return _memo.Contains(PythonOps.Id(obj));
- }
- return _privMemo.ContainsKey(obj);
- }
- private bool TryWriteFastGet(CodeContext context, object obj) {
- int value;
- if (_memo != null) {
- return TryWriteSlowGet(context, obj);
- } else if (_privMemo.TryGetValue(obj, out value)) {
- WriteGetOrPut(context, true, value);
- return true;
- }
- return false;
- }
- private bool TryWriteSlowGet(CodeContext context, object obj) {
- object value;
- if (_memo.TryGetValue(obj, out value)) {
- WriteGetOrPut(context, true, (PythonTuple)value);
- return true;
- }
- return false;
- }
- [Documentation("getvalue() -> string\n\n"
- + "Return the value of the internal string. Raises PicklingError if a file object\n"
- + "was passed to this pickler's constructor."
- )]
- public object getvalue(CodeContext/*!*/ context) {
- if (_file is PythonReadableFileOutput) {
- return ((PythonReadableFileOutput)_file).GetValue(context);
- }
- throw PythonExceptions.CreateThrowable(PicklingError(context), "Attempt to getvalue() a non-list-based pickler");
- }
- #endregion
- #region Save functions
- private void Save(CodeContext/*!*/ context, object obj) {
- if (_persist_id == null || !TrySavePersistId(context, obj)) {
- PickleFunction pickleFunction;
- // several typees are never memoized, check for these first.
- if (obj == null) {
- SaveNone(this, context, obj);
- } else if (obj is int) {
- SaveInteger(this, context, obj);
- } else if(obj is BigInteger) {
- SaveLong(this, context, obj);
- } else if (obj is bool) {
- SaveBoolean(this, context, obj);
- } else if (obj is double) {
- SaveFloat(this, context, obj);
- } else if(!TryWriteFastGet(context, obj)) {
- if (obj is string) {
- // strings are common, specialize them.
- SaveUnicode(this, context, obj);
- } else {
- if (!_dispatchTable.TryGetValue(obj.GetType(), out pickleFunction)) {
- if (obj is PythonType) {
- // treat classes with metaclasses like regular classes
- pickleFunction = SaveGlobal;
- } else {
- pickleFunction = SaveObject;
- }
- }
- pickleFunction(this, context, obj);
- }
- }
- }
- }
- private bool TrySavePersistId(CodeContext context, object obj) {
- Debug.Assert(_persist_id != null);
- string res = Converter.ConvertToString(PythonContext.GetContext(context).CallSplat(_persist_id, obj));
- if (res != null) {
- SavePersId(context, res);
- return true;
- }
-
- return false;
- }
- private void SavePersId(CodeContext/*!*/ context, string res) {
- if (this.binary != 0) {
- Save(context, res);
- Write(context, Opcode.BinPersid);
- } else {
- Write(context, Opcode.PersId);
- Write(context, res);
- Write(context, "\n");
- }
- }
- private static void SaveBoolean(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Boolean), "arg must be bool");
- if (pickler._protocol < 2) {
- pickler.Write(context, Opcode.Int);
- pickler.Write(context, String.Format("0{0}", ((bool)obj) ? 1 : 0));
- pickler.Write(context, Newline);
- } else {
- if ((bool)obj) {
- pickler.Write(context, Opcode.NewTrue);
- } else {
- pickler.Write(context, Opcode.NewFalse);
- }
- }
- }
- private static void SaveDict(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Dict), "arg must be dict");
- Debug.Assert(!pickler.MemoContains(obj));
- int index = pickler.MemoizeNew(obj);
- if (pickler._protocol < 1) {
- pickler.Write(context, Opcode.Mark);
- pickler.Write(context, Opcode.Dict);
- } else {
- pickler.Write(context, Opcode.EmptyDict);
- }
- pickler.WritePut(context, index);
- pickler.BatchSetItems(context, (PythonDictionary)obj);
- }
- private static void SaveFloat(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Double), "arg must be float");
- if (pickler._protocol < 1) {
- pickler.Write(context, Opcode.Float);
- pickler.WriteFloatAsString(context, obj);
- } else {
- pickler.Write(context, Opcode.BinFloat);
- pickler.WriteFloat64(context, obj);
- }
- }
- private static void SaveGlobal(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
- Debug.Assert(
- DynamicHelpers.GetPythonType(obj).Equals(TypeCache.OldClass) ||
- DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Function) ||
- DynamicHelpers.GetPythonType(obj).Equals(TypeCache.BuiltinFunction) ||
- DynamicHelpers.GetPythonType(obj).Equals(TypeCache.PythonType) ||
- DynamicHelpers.GetPythonType(obj).IsSubclassOf(TypeCache.PythonType),
- "arg must be classic class, function, built-in function or method, or new-style type"
- );
- PythonType pt = obj as PythonType;
- if (pt != null) {
- pickler.SaveGlobalByName(context, obj, pt.Name);
- } else {
- object name;
- if (PythonOps.TryGetBoundAttr(context, obj, "__name__", out name)) {
- pickler.SaveGlobalByName(context, obj, name);
- } else {
- throw pickler.CannotPickle(context, obj, "could not determine its __name__");
- }
- }
- }
- private void SaveGlobalByName(CodeContext/*!*/ context, object obj, object name) {
- Debug.Assert(!MemoContains(obj));
- object moduleName = FindModuleForGlobal(context, obj, name);
- if (_protocol >= 2) {
- object code;
- if (PythonCopyReg.GetExtensionRegistry(context).TryGetValue(PythonTuple.MakeTuple(moduleName, name), out code)) {
- if (IsUInt8(context, code)) {
- Write(context, Opcode.Ext1);
- WriteUInt8(context, code);
- } else if (IsUInt16(context, code)) {
- Write(context, Opcode.Ext2);
- WriteUInt16(context, code);
- } else if (IsInt32(context, code)) {
- Write(context, Opcode.Ext4);
- WriteInt32(context, code);
- } else {
- throw PythonOps.RuntimeError("unrecognized integer format");
- }
- return;
- }
- }
- MemoizeNew(obj);
- Write(context, Opcode.Global);
- WriteStringPair(context, moduleName, name);
- WritePut(context, obj);
- }
- private static void SaveInstance(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.OldInstance), "arg must be old-class instance");
- Debug.Assert(!pickler.MemoContains(obj));
- pickler.Write(context, Opcode.Mark);
- // Memoize() call isn't in the usual spot to allow class to be memoized before
- // instance (when using proto other than 0) to match CPython's bytecode output
- object objClass;
- if (!PythonOps.TryGetBoundAttr(context, obj, "__class__", out objClass)) {
- throw pickler.CannotPickle(context, obj, "could not determine its __class__");
- }
- if (pickler._protocol < 1) {
- object className, classModuleName;
- if (!PythonOps.TryGetBoundAttr(context, objClass, "__name__", out className)) {
- throw pickler.CannotPickle(context, obj, "its __class__ has no __name__");
- }
- classModuleName = pickler.FindModuleForGlobal(context, objClass, className);
- Debug.Assert(!pickler.MemoContains(obj));
- pickler.MemoizeNew(obj);
- pickler.WriteInitArgs(context, obj);
- pickler.Write(context, Opcode.Inst);
- pickler.WriteStringPair(context, classModuleName, className);
- } else {
- pickler.Save(context, objClass);
- pickler.Memoize(obj);
- pickler.WriteInitArgs(context, obj);
- pickler.Write(context, Opcode.Obj);
- }
- pickler.WritePut(context, obj);
- object getStateCallable;
- if (PythonOps.TryGetBoundAttr(context, obj, "__getstate__", out getStateCallable)) {
- pickler.Save(context, PythonCalls.Call(context, getStateCallable));
- } else {
- pickler.Save(context, PythonOps.GetBoundAttr(context, obj, "__dict__"));
- }
- pickler.Write(context, Opcode.Build);
- }
- private static void SaveInteger(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Int32), "arg must be int");
- if (pickler._protocol < 1) {
- pickler.Write(context, Opcode.Int);
- pickler.WriteIntAsString(context, obj);
- } else {
- if (IsUInt8(context, obj)) {
- pickler.Write(context, Opcode.BinInt1);
- pickler.WriteUInt8(context, obj);
- } else if (IsUInt16(context, obj)) {
- pickler.Write(context, Opcode.BinInt2);
- pickler.WriteUInt16(context, obj);
- } else if (IsInt32(context, obj)) {
- pickler.Write(context, Opcode.BinInt);
- pickler.WriteInt32(context, obj);
- } else {
- throw PythonOps.RuntimeError("unrecognized integer format");
- }
- }
- }
- private static void SaveList(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.List), "arg must be list");
- Debug.Assert(!pickler.MemoContains(obj));
- int index = pickler.MemoizeNew(obj);
- if (pickler._protocol < 1) {
- pickler.Write(context, Opcode.Mark);
- pickler.Write(context, Opcode.List);
- } else {
- pickler.Write(context, Opcode.EmptyList);
- }
- pickler.WritePut(context, index);
- pickler.BatchAppends(context, ((IEnumerable)obj).GetEnumerator());
- }
- #if CLR2
- private static readonly BigInteger MaxInt = BigInteger.Create(Int32.MaxValue);
- private static readonly BigInteger MinInt = BigInteger.Create(Int32.MinValue);
- #else
- private static readonly BigInteger MaxInt = new BigInteger(Int32.MaxValue);
- private static readonly BigInteger MinInt = new BigInteger(Int32.MinValue);
- #endif
- private static void SaveLong(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.BigInteger), "arg must be long");
- BigInteger bi = (BigInteger)obj;
- if (pickler._protocol < 2) {
- pickler.Write(context, Opcode.Long);
- pickler.WriteLongAsString(context, obj);
- } else if (bi.IsZero()) {
- pickler.Write(context, Opcode.Long1);
- pickler.WriteUInt8(context, 0);
- } else if (bi <= MaxInt && bi >= MinInt) {
- pickler.Write(context, Opcode.Long1);
- int value = (int)bi;
- if (IsInt8(value)) {
- pickler.WriteUInt8(context, 1);
- pickler._file.Write(context, (char)(byte)value);
- } else if (IsInt16(value)) {
- pickler.WriteUInt8(context, 2);
- pickler.WriteUInt8(context, value & 0xff);
- pickler.WriteUInt8(context, (value >> 8) & 0xff);
- } else {
- pickler.WriteUInt8(context, 4);
- pickler.WriteInt32(context, value);
- }
- } else {
- byte[] dataBytes = bi.ToByteArray();
- if (dataBytes.Length < 256) {
- pickler.Write(context, Opcode.Long1);
- pickler.WriteUInt8(context, dataBytes.Length);
- } else {
- pickler.Write(context, Opcode.Long4);
- pickler.WriteInt32(context, dataBytes.Length);
- }
- foreach (byte b in dataBytes) {
- pickler.WriteUInt8(context, b);
- }
- }
- }
- private static void SaveNone(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Null), "arg must be None");
- pickler.Write(context, Opcode.NoneValue);
- }
- /// <summary>
- /// Call the appropriate reduce method for obj and pickle the object using
- /// the resulting data. Use the first available of
- /// copy_reg.dispatch_table[type(obj)], obj.__reduce_ex__, and obj.__reduce__.
- /// </summary>
- private void SaveObject(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
- Debug.Assert(!MemoContains(obj));
- MemoizeNew(obj);
- object reduceCallable, result;
- PythonType objType = DynamicHelpers.GetPythonType(obj);
- if (((IDictionary<object, object>)PythonCopyReg.GetDispatchTable(context)).TryGetValue(objType, out reduceCallable)) {
- result = PythonCalls.Call(context, reduceCallable, obj);
- } else if (PythonOps.TryGetBoundAttr(context, obj, "__reduce_ex__", out reduceCallable)) {
- if (obj is PythonType) {
- result = context.LanguageContext.Call(context, reduceCallable, obj, _protocol);
- } else {
- result = context.LanguageContext.Call(context, reduceCallable, _protocol);
- }
- } else if (PythonOps.TryGetBoundAttr(context, obj, "__reduce__", out reduceCallable)) {
- if (obj is PythonType) {
- result = context.LanguageContext.Call(context, reduceCallable, obj);
- } else {
- result = context.LanguageContext.Call(context, reduceCallable);
- }
- } else {
- throw PythonOps.AttributeError("no reduce function found for {0}", obj);
- }
- if (objType.Equals(TypeCache.String)) {
- if (!TryWriteFastGet(context, obj)) {
- SaveGlobalByName(context, obj, result);
- }
- } else if (result is PythonTuple) {
- PythonTuple rt = (PythonTuple)result;
- switch (rt.__len__()) {
- case 2:
- SaveReduce(context, obj, reduceCallable, rt[0], rt[1], null, null, null);
- break;
- case 3:
- SaveReduce(context, obj, reduceCallable, rt[0], rt[1], rt[2], null, null);
- break;
- case 4:
- SaveReduce(context, obj, reduceCallable, rt[0], rt[1], rt[2], rt[3], null);
- break;
- case 5:
- SaveReduce(context, obj, reduceCallable, rt[0], rt[1], rt[2], rt[3], rt[4]);
- break;
- default:
- throw CannotPickle(context, obj, "tuple returned by {0} must have to to five elements", reduceCallable);
- }
- } else {
- throw CannotPickle(context, obj, "{0} must return string or tuple", reduceCallable);
- }
- }
- /// <summary>
- /// Pickle the result of a reduce function.
- ///
- /// Only context, obj, func, and reduceCallable are required; all other arguments may be null.
- /// </summary>
- private void SaveReduce(CodeContext/*!*/ context, object obj, object reduceCallable, object func, object args, object state, object listItems, object dictItems) {
- if (!PythonOps.IsCallable(context, func)) {
- throw CannotPickle(context, obj, "func from reduce() should be callable");
- } else if (!(args is PythonTuple) && args != null) {
- throw CannotPickle(context, obj, "args from reduce() should be a tuple");
- } else if (listItems != null && !(listItems is IEnumerator)) {
- throw CannotPickle(context, obj, "listitems from reduce() should be a list iterator");
- } else if (dictItems != null && !(dictItems is IEnumerator)) {
- throw CannotPickle(context, obj, "dictitems from reduce() should be a dict iterator");
- }
- object funcName;
- string funcNameString;
- if (func is PythonType) {
- funcNameString = ((PythonType)func).Name;
- } else {
- if (!PythonOps.TryGetBoundAttr(context, func, "__name__", out funcName)) {
- throw CannotPickle(context, obj, "func from reduce() ({0}) should have a __name__ attribute");
- } else if (!Converter.TryConvertToString(funcName, out funcNameString) || funcNameString == null) {
- throw CannotPickle(context, obj, "__name__ of func from reduce() must be string");
- }
- }
- if (_protocol >= 2 && "__newobj__" == funcNameString) {
- if (args == null) {
- throw CannotPickle(context, obj, "__newobj__ arglist is None");
- }
- PythonTuple argsTuple = (PythonTuple)args;
- if (argsTuple.__len__() == 0) {
- throw CannotPickle(context, obj, "__newobj__ arglist is empty");
- } else if (!DynamicHelpers.GetPythonType(obj).Equals(argsTuple[0])) {
- throw CannotPickle(context, obj, "args[0] from __newobj__ args has the wrong class");
- }
- Save(context, argsTuple[0]);
- Save(context, argsTuple[new Slice(1, null)]);
- Write(context, Opcode.NewObj);
- } else {
- Save(context, func);
- Save(context, args);
- Write(context, Opcode.Reduce);
- }
- WritePut(context, obj);
- if (state != null) {
- Save(context, state);
- Write(context, Opcode.Build);
- }
- if (listItems != null) {
- BatchAppends(context, (IEnumerator)listItems);
- }
- if (dictItems != null) {
- BatchSetItems(context, (IEnumerator)dictItems);
- }
- }
- private static void SaveTuple(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.PythonTuple), "arg must be tuple");
- Debug.Assert(!pickler.MemoContains(obj));
- PythonTuple t = (PythonTuple)obj;
- char opcode;
- bool needMark = false;
- int len = t._data.Length;
- if (pickler._protocol > 0 && len == 0) {
- opcode = Opcode.EmptyTuple;
- } else if (pickler._protocol >= 2 && len == 1) {
- opcode = Opcode.Tuple1;
- } else if (pickler._protocol >= 2 && len == 2) {
- opcode = Opcode.Tuple2;
- } else if (pickler._protocol >= 2 && len == 3) {
- opcode = Opcode.Tuple3;
- } else {
- opcode = Opcode.Tuple;
- needMark = true;
- }
- if (needMark) pickler.Write(context, Opcode.Mark);
- var data = t._data;
- for (int i = 0; i < data.Length; i++) {
- pickler.Save(context, data[i]);
- }
- if (len > 0) {
- if (pickler.MemoContains(obj)) {
- // recursive tuple
- if (pickler._protocol == 1) {
- pickler.Write(context, Opcode.PopMark);
- } else {
- if (pickler._protocol == 0) {
- pickler.Write(context, Opcode.Pop);
- }
- for (int i = 0; i < len; i++) {
- pickler.Write(context, Opcode.Pop);
- }
- }
- pickler.WriteGet(context, obj);
- return;
- }
- pickler.Write(context, opcode);
- pickler.Memoize(t);
- pickler.WritePut(context, t);
- } else {
- pickler.Write(context, opcode);
- }
- }
- private static void SaveUnicode(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
- Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.String), "arg must be unicode");
- Debug.Assert(!pickler.MemoContains(obj));
- if (pickler._memo != null) {
- pickler.MemoizeNew(obj);
- if (pickler._protocol < 1) {
- …
Large files files are truncated, but you can click here to view the full file