PageRenderTime 58ms CodeModel.GetById 23ms RepoModel.GetById 0ms app.codeStats 1ms

/Languages/IronPython/IronPython.Modules/cPickle.cs

http://github.com/IronLanguages/main
C# | 2389 lines | 2156 code | 155 blank | 78 comment | 223 complexity | a3c6fcc07ac2da828908469cc2621697 MD5 | raw file
Possible License(s): CPL-1.0, BSD-3-Clause, ISC, GPL-2.0, MPL-2.0-no-copyleft-exception

Large files files are truncated, but you can click here to view the full file

  1. /* ****************************************************************************
  2. *
  3. * Copyright (c) Microsoft Corporation.
  4. *
  5. * This source code is subject to terms and conditions of the Apache License, Version 2.0. A
  6. * copy of the license can be found in the License.html file at the root of this distribution. If
  7. * you cannot locate the Apache License, Version 2.0, please send an email to
  8. * ironpy@microsoft.com. By using this source code in any fashion, you are agreeing to be bound
  9. * by the terms of the Apache License, Version 2.0.
  10. *
  11. * You must not remove this notice, or any other, from this software.
  12. *
  13. *
  14. * ***************************************************************************/
  15. using System;
  16. using System.Collections;
  17. using System.Collections.Generic;
  18. using System.Diagnostics;
  19. using System.IO;
  20. using System.Runtime.InteropServices;
  21. using System.Text;
  22. using IronPython.Runtime;
  23. using IronPython.Runtime.Exceptions;
  24. using IronPython.Runtime.Operations;
  25. using IronPython.Runtime.Types;
  26. using Microsoft.Scripting;
  27. using Microsoft.Scripting.Runtime;
  28. using Microsoft.Scripting.Utils;
  29. #if FEATURE_NUMERICS
  30. using System.Numerics;
  31. #else
  32. using Microsoft.Scripting.Math;
  33. #endif
  34. [assembly: PythonModule("cPickle", typeof(IronPython.Modules.PythonPickle))]
  35. namespace IronPython.Modules {
  36. public static class PythonPickle {
  37. public const string __doc__ = "Fast object serialization/deserialization.\n\n"
  38. + "Differences from CPython:\n"
  39. + " - does not implement the undocumented fast mode\n";
  40. [System.Runtime.CompilerServices.SpecialName]
  41. public static void PerformModuleReload(PythonContext/*!*/ context, PythonDictionary/*!*/ dict) {
  42. context.EnsureModuleException("PickleError", dict, "PickleError", "cPickle");
  43. context.EnsureModuleException("PicklingError", dict, "PicklingError", "cPickle");
  44. context.EnsureModuleException("UnpicklingError", dict, "UnpicklingError", "cPickle");
  45. context.EnsureModuleException("UnpickleableError", dict, "UnpickleableError", "cPickle");
  46. context.EnsureModuleException("BadPickleGet", dict, "BadPickleGet", "cPickle");
  47. dict["__builtins__"] = context.BuiltinModuleInstance;
  48. dict["compatible_formats"] = PythonOps.MakeList("1.0", "1.1", "1.2", "1.3", "2.0");
  49. }
  50. private static readonly PythonStruct.Struct _float64 = PythonStruct.Struct.Create(">d");
  51. private const int highestProtocol = 2;
  52. public const string __version__ = "1.71";
  53. public const string format_version = "2.0";
  54. public static int HIGHEST_PROTOCOL {
  55. get { return highestProtocol; }
  56. }
  57. private const string Newline = "\n";
  58. #region Public module-level functions
  59. [Documentation("dump(obj, file, protocol=0) -> None\n\n"
  60. + "Pickle obj and write the result to file.\n"
  61. + "\n"
  62. + "See documentation for Pickler() for a description the file, protocol, and\n"
  63. + "(deprecated) bin parameters."
  64. )]
  65. public static void dump(CodeContext/*!*/ context, object obj, object file, [DefaultParameterValue(null)] object protocol, [DefaultParameterValue(null)] object bin) {
  66. PicklerObject/*!*/ pickler = new PicklerObject(context, file, protocol, bin);
  67. pickler.dump(context, obj);
  68. }
  69. [Documentation("dumps(obj, protocol=0) -> pickle string\n\n"
  70. + "Pickle obj and return the result as a string.\n"
  71. + "\n"
  72. + "See the documentation for Pickler() for a description of the protocol and\n"
  73. + "(deprecated) bin parameters."
  74. )]
  75. public static string dumps(CodeContext/*!*/ context, object obj, [DefaultParameterValue(null)] object protocol, [DefaultParameterValue(null)] object bin) {
  76. //??? possible perf enhancement: use a C# TextWriter-backed IFileOutput and
  77. // thus avoid Python call overhead. Also do similar thing for LoadFromString.
  78. var stringIO = new StringBuilderOutput();
  79. PicklerObject/*!*/ pickler = new PicklerObject(context, stringIO, protocol, bin);
  80. pickler.dump(context, obj);
  81. return stringIO.GetString();
  82. }
  83. [Documentation("load(file) -> unpickled object\n\n"
  84. + "Read pickle data from the open file object and return the corresponding\n"
  85. + "unpickled object. Data after the first pickle found is ignored, but the file\n"
  86. + "cursor is not reset, so if a file objects contains multiple pickles, then\n"
  87. + "load() may be called multiple times to unpickle them.\n"
  88. + "\n"
  89. + "file: an object (such as an open file or a StringIO) with read(num_chars) and\n"
  90. + " readline() methods that return strings\n"
  91. + "\n"
  92. + "load() automatically determines if the pickle data was written in binary or\n"
  93. + "text mode."
  94. )]
  95. public static object load(CodeContext/*!*/ context, object file) {
  96. return new UnpicklerObject(context, file).load(context);
  97. }
  98. [Documentation("loads(string) -> unpickled object\n\n"
  99. + "Read a pickle object from a string, unpickle it, and return the resulting\n"
  100. + "reconstructed object. Characters in the string beyond the end of the first\n"
  101. + "pickle are ignored."
  102. )]
  103. public static object loads(CodeContext/*!*/ context, [BytesConversion]string @string) {
  104. return new UnpicklerObject(context, new PythonStringInput(@string)).load(context);
  105. }
  106. #endregion
  107. #region File I/O wrappers
  108. /// <summary>
  109. /// Interface for "file-like objects" that implement the protocol needed by load() and friends.
  110. /// This enables the creation of thin wrappers that make fast .NET types and slow Python types look the same.
  111. /// </summary>
  112. internal abstract class FileInput {
  113. public abstract string Read(CodeContext/*!*/ context, int size);
  114. public abstract string ReadLine(CodeContext/*!*/ context);
  115. public virtual string ReadLineNoNewLine(CodeContext/*!*/ context) {
  116. var raw = ReadLine(context);
  117. return raw.Substring(0, raw.Length - 1);
  118. }
  119. public virtual char ReadChar(CodeContext context) {
  120. string res = Read(context, 1);
  121. if (res.Length < 1) {
  122. throw PythonOps.EofError("unexpected EOF while unpickling");
  123. }
  124. return res[0];
  125. }
  126. public virtual int ReadInt(CodeContext context) {
  127. return (int)ReadChar(context) |
  128. ((int)ReadChar(context)) << 8 |
  129. ((int)ReadChar(context)) << 16 |
  130. ((int)ReadChar(context)) << 24;
  131. }
  132. }
  133. /// <summary>
  134. /// Interface for "file-like objects" that implement the protocol needed by dump() and friends.
  135. /// This enables the creation of thin wrappers that make fast .NET types and slow Python types look the same.
  136. /// </summary>
  137. internal abstract class FileOutput {
  138. private readonly char[] int32chars = new char[4];
  139. public abstract void Write(CodeContext/*!*/ context, string data);
  140. public virtual void Write(CodeContext context, int data) {
  141. int32chars[0] = (char)(int)((data & 0xff));
  142. int32chars[1] = (char)(int)((data >> 8) & 0xff);
  143. int32chars[2] = (char)(int)((data >> 16) & 0xff);
  144. int32chars[3] = (char)(int)((data >> 24) & 0xff);
  145. Write(context, new string(int32chars));
  146. }
  147. public virtual void Write(CodeContext context, char data) {
  148. Write(context, ScriptingRuntimeHelpers.CharToString(data));
  149. }
  150. }
  151. private class PythonFileInput : FileInput {
  152. private object _readMethod;
  153. private object _readLineMethod;
  154. public PythonFileInput(CodeContext/*!*/ context, object file) {
  155. if (!PythonOps.TryGetBoundAttr(context, file, "read", out _readMethod) ||
  156. !PythonOps.IsCallable(context, _readMethod) ||
  157. !PythonOps.TryGetBoundAttr(context, file, "readline", out _readLineMethod) ||
  158. !PythonOps.IsCallable(context, _readLineMethod)
  159. ) {
  160. throw PythonOps.TypeError("argument must have callable 'read' and 'readline' attributes");
  161. }
  162. }
  163. public override string Read(CodeContext/*!*/ context, int size) {
  164. return Converter.ConvertToString(PythonCalls.Call(context, _readMethod, size));
  165. }
  166. public override string ReadLine(CodeContext/*!*/ context) {
  167. return Converter.ConvertToString(PythonCalls.Call(context, _readLineMethod));
  168. }
  169. }
  170. internal class PythonStringInput : FileInput {
  171. private readonly string _data;
  172. int _offset;
  173. public PythonStringInput(string data) {
  174. _data = data;
  175. }
  176. public override string Read(CodeContext context, int size) {
  177. var res = _data.Substring(_offset, size);
  178. _offset += size;
  179. return res;
  180. }
  181. public override string ReadLine(CodeContext context) {
  182. return ReadLineWorker(true);
  183. }
  184. public override string ReadLineNoNewLine(CodeContext context) {
  185. return ReadLineWorker(false);
  186. }
  187. public override char ReadChar(CodeContext context) {
  188. if (_offset < _data.Length) {
  189. return _data[_offset++];
  190. }
  191. throw PythonOps.EofError("unexpected EOF while unpickling");
  192. }
  193. public override int ReadInt(CodeContext context) {
  194. if (_offset + 4 <= _data.Length) {
  195. int res = _data[_offset]|
  196. ((int)_data[_offset + 1]) << 8 |
  197. ((int)_data[_offset + 2]) << 16 |
  198. ((int)_data[_offset + 3]) << 24;
  199. _offset += 4;
  200. return res;
  201. }
  202. throw PythonOps.EofError("unexpected EOF while unpickling");
  203. }
  204. private string ReadLineWorker(bool includeNewLine) {
  205. string res;
  206. for (int i = _offset; i < _data.Length; i++) {
  207. if (_data[i] == '\n') {
  208. res = _data.Substring(_offset, i - _offset + (includeNewLine ? 1 : 0));
  209. _offset = i + 1;
  210. return res;
  211. }
  212. }
  213. res = _data.Substring(_offset);
  214. _offset = _data.Length;
  215. return res;
  216. }
  217. }
  218. private class PythonFileLikeOutput : FileOutput {
  219. private object _writeMethod;
  220. public PythonFileLikeOutput(CodeContext/*!*/ context, object file) {
  221. if (!PythonOps.TryGetBoundAttr(context, file, "write", out _writeMethod) ||
  222. !PythonOps.IsCallable(context, this._writeMethod)
  223. ) {
  224. throw PythonOps.TypeError("argument must have callable 'write' attribute");
  225. }
  226. }
  227. public override void Write(CodeContext/*!*/ context, string data) {
  228. PythonCalls.Call(context, _writeMethod, data);
  229. }
  230. }
  231. private class PythonFileOutput : FileOutput {
  232. private readonly PythonFile _file;
  233. public PythonFileOutput(PythonFile file) {
  234. _file = file;
  235. }
  236. public override void Write(CodeContext/*!*/ context, string data) {
  237. _file.write(data);
  238. }
  239. }
  240. private class StringBuilderOutput : FileOutput {
  241. private readonly StringBuilder _builder = new StringBuilder(4096);
  242. public string GetString() {
  243. return _builder.ToString();
  244. }
  245. public override void Write(CodeContext context, char data) {
  246. _builder.Append(data);
  247. }
  248. public override void Write(CodeContext context, int data) {
  249. _builder.Append((char)(int)((data) & 0xff));
  250. _builder.Append((char)(int)((data >> 8) & 0xff));
  251. _builder.Append((char)(int)((data >> 16) & 0xff));
  252. _builder.Append((char)(int)((data >> 24) & 0xff));
  253. }
  254. public override void Write(CodeContext context, string data) {
  255. _builder.Append(data);
  256. }
  257. }
  258. private class PythonReadableFileOutput : PythonFileLikeOutput {
  259. private object _getValueMethod;
  260. public PythonReadableFileOutput(CodeContext/*!*/ context, object file)
  261. : base(context, file) {
  262. if (!PythonOps.TryGetBoundAttr(context, file, "getvalue", out _getValueMethod) ||
  263. !PythonOps.IsCallable(context, _getValueMethod)
  264. ) {
  265. throw PythonOps.TypeError("argument must have callable 'getvalue' attribute");
  266. }
  267. }
  268. public object GetValue(CodeContext/*!*/ context) {
  269. return PythonCalls.Call(context, _getValueMethod);
  270. }
  271. }
  272. #endregion
  273. #region Opcode constants
  274. internal static class Opcode {
  275. public const char Append = 'a';
  276. public const char Appends = 'e';
  277. public const char BinFloat = 'G';
  278. public const char BinGet = 'h';
  279. public const char BinInt = 'J';
  280. public const char BinInt1 = 'K';
  281. public const char BinInt2 = 'M';
  282. public const char BinPersid = 'Q';
  283. public const char BinPut = 'q';
  284. public const char BinString = 'T';
  285. public const char BinUnicode = 'X';
  286. public const char Build = 'b';
  287. public const char Dict = 'd';
  288. public const char Dup = '2';
  289. public const char EmptyDict = '}';
  290. public const char EmptyList = ']';
  291. public const char EmptyTuple = ')';
  292. public const char Ext1 = '\x82';
  293. public const char Ext2 = '\x83';
  294. public const char Ext4 = '\x84';
  295. public const char Float = 'F';
  296. public const char Get = 'g';
  297. public const char Global = 'c';
  298. public const char Inst = 'i';
  299. public const char Int = 'I';
  300. public const char List = 'l';
  301. public const char Long = 'L';
  302. public const char Long1 = '\x8a';
  303. public const char Long4 = '\x8b';
  304. public const char LongBinGet = 'j';
  305. public const char LongBinPut = 'r';
  306. public const char Mark = '(';
  307. public const char NewFalse = '\x89';
  308. public const char NewObj = '\x81';
  309. public const char NewTrue = '\x88';
  310. public const char NoneValue = 'N';
  311. public const char Obj = 'o';
  312. public const char PersId = 'P';
  313. public const char Pop = '0';
  314. public const char PopMark = '1';
  315. public const char Proto = '\x80';
  316. public const char Put = 'p';
  317. public const char Reduce = 'R';
  318. public const char SetItem = 's';
  319. public const char SetItems = 'u';
  320. public const char ShortBinstring = 'U';
  321. public const char Stop = '.';
  322. public const char String = 'S';
  323. public const char Tuple = 't';
  324. public const char Tuple1 = '\x85';
  325. public const char Tuple2 = '\x86';
  326. public const char Tuple3 = '\x87';
  327. public const char Unicode = 'V';
  328. }
  329. #endregion
  330. #region Pickler object
  331. public static PicklerObject/*!*/ Pickler(CodeContext/*!*/ context, [DefaultParameterValue(null)]object file, [DefaultParameterValue(null)]object protocol, [DefaultParameterValue(null)]object bin) {
  332. return new PicklerObject(context, file, protocol, bin);
  333. }
  334. [Documentation("Pickler(file, protocol=0) -> Pickler object\n\n"
  335. + "A Pickler object serializes Python objects to a pickle bytecode stream, which\n"
  336. + "can then be converted back into equivalent objects using an Unpickler.\n"
  337. + "\n"
  338. + "file: an object (such as an open file) that has a write(string) method.\n"
  339. + "protocol: if omitted, protocol 0 is used. If HIGHEST_PROTOCOL or a negative\n"
  340. + " number, the highest available protocol is used.\n"
  341. + "bin: (deprecated; use protocol instead) for backwards compability, a 'bin'\n"
  342. + " keyword parameter is supported. When protocol is specified it is ignored.\n"
  343. + " If protocol is not specified, then protocol 0 is used if bin is false, and\n"
  344. + " protocol 1 is used if bin is true."
  345. )]
  346. [PythonType("Pickler"), PythonHidden]
  347. public class PicklerObject {
  348. private const char LowestPrintableChar = (char)32;
  349. private const char HighestPrintableChar = (char)126;
  350. // max elements that can be set/appended at a time using SETITEMS/APPENDS
  351. private delegate void PickleFunction(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object value);
  352. private static readonly Dictionary<Type, PickleFunction> _dispatchTable;
  353. private int _batchSize = 1000;
  354. private FileOutput _file;
  355. private int _protocol;
  356. private PythonDictionary _memo; // memo if the user accesses the memo property
  357. private Dictionary<object, int> _privMemo; // internal fast memo which we can use if the user doesn't access memo
  358. private object _persist_id;
  359. static PicklerObject() {
  360. _dispatchTable = new Dictionary<Type, PickleFunction>();
  361. _dispatchTable[typeof(PythonDictionary)] = SaveDict;
  362. _dispatchTable[typeof(PythonTuple)] = SaveTuple;
  363. _dispatchTable[typeof(List)] = SaveList;
  364. _dispatchTable[typeof(OldClass)] = SaveGlobal;
  365. _dispatchTable[typeof(PythonFunction)] = SaveGlobal;
  366. _dispatchTable[typeof(BuiltinFunction)] = SaveGlobal;
  367. _dispatchTable[typeof(PythonType)] = SaveGlobal;
  368. _dispatchTable[typeof(OldInstance)] = SaveInstance;
  369. }
  370. #region Public API
  371. [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
  372. public PythonDictionary memo {
  373. get {
  374. if (_memo == null) {
  375. // create publicly viewable memo
  376. PythonDictionary resMemo = new PythonDictionary();
  377. foreach (var v in _privMemo) {
  378. resMemo._storage.AddNoLock(
  379. ref resMemo._storage,
  380. Builtin.id(v.Key),
  381. PythonTuple.MakeTuple(v.Value, v.Key)
  382. );
  383. }
  384. _memo = resMemo;
  385. }
  386. return _memo;
  387. }
  388. set {
  389. _memo = value;
  390. _privMemo = null;
  391. }
  392. }
  393. public int proto {
  394. get { return _protocol; }
  395. set { _protocol = value; }
  396. }
  397. public int _BATCHSIZE {
  398. get { return _batchSize; }
  399. set { _batchSize = value; }
  400. }
  401. public object persistent_id {
  402. get {
  403. return _persist_id;
  404. }
  405. set {
  406. _persist_id = value;
  407. }
  408. }
  409. public int binary {
  410. get { return _protocol == 0 ? 1 : 0; }
  411. set { _protocol = value; }
  412. }
  413. public int fast {
  414. // We don't implement fast, but we silently ignore it when it's set so that test_cpickle works.
  415. // For a description of fast, see http://mail.python.org/pipermail/python-bugs-list/2001-October/007695.html
  416. get { return 0; }
  417. set { /* ignore */ }
  418. }
  419. public PicklerObject(CodeContext/*!*/ context, object file, object protocol, object bin) {
  420. int intProtocol;
  421. if (file == null) {
  422. _file = new PythonReadableFileOutput(context, new PythonStringIO.StringO());
  423. } else if (Converter.TryConvertToInt32(file, out intProtocol)) {
  424. // For undocumented (yet tested in official CPython tests) list-based pickler, the
  425. // user could do something like Pickler(1), which would create a protocol-1 pickler
  426. // with an internal string output buffer (retrievable using getvalue()). For a little
  427. // more info, see
  428. // https://sourceforge.net/tracker/?func=detail&atid=105470&aid=939395&group_id=5470
  429. _file = new PythonReadableFileOutput(context, new PythonStringIO.StringO());
  430. protocol = file;
  431. } else if (file is PythonFile) {
  432. _file = new PythonFileOutput((PythonFile)file);
  433. } else if (file is FileOutput) {
  434. _file = (FileOutput)file;
  435. } else {
  436. _file = new PythonFileLikeOutput(context, file);
  437. }
  438. _privMemo = new Dictionary<object, int>(256, ReferenceEqualityComparer.Instance);
  439. if (protocol == null) protocol = PythonOps.IsTrue(bin) ? 1 : 0;
  440. intProtocol = context.LanguageContext.ConvertToInt32(protocol);
  441. if (intProtocol > highestProtocol) {
  442. throw PythonOps.ValueError("pickle protocol {0} asked for; the highest available protocol is {1}", intProtocol, highestProtocol);
  443. } else if (intProtocol < 0) {
  444. this._protocol = highestProtocol;
  445. } else {
  446. this._protocol = intProtocol;
  447. }
  448. }
  449. [Documentation("dump(obj) -> None\n\n"
  450. + "Pickle obj and write the result to the file object that was passed to the\n"
  451. + "constructor\n."
  452. + "\n"
  453. + "Note that you may call dump() multiple times to pickle multiple objects. To\n"
  454. + "unpickle the stream, you will need to call Unpickler's load() method a\n"
  455. + "corresponding number of times.\n"
  456. + "\n"
  457. + "The first time a particular object is encountered, it will be pickled normally.\n"
  458. + "If the object is encountered again (in the same or a later dump() call), a\n"
  459. + "reference to the previously generated value will be pickled. Unpickling will\n"
  460. + "then create multiple references to a single object."
  461. )]
  462. public void dump(CodeContext/*!*/ context, object obj) {
  463. if (_protocol >= 2) WriteProto(context);
  464. Save(context, obj);
  465. Write(context, Opcode.Stop);
  466. }
  467. [Documentation("clear_memo() -> None\n\n"
  468. + "Clear the memo, which is used internally by the pickler to keep track of which\n"
  469. + "objects have already been pickled (so that shared or recursive objects are\n"
  470. + "pickled only once)."
  471. )]
  472. public void clear_memo() {
  473. if (_memo != null) {
  474. _memo.Clear();
  475. } else {
  476. _privMemo.Clear();
  477. }
  478. }
  479. private void Memoize(object obj) {
  480. if (_memo != null) {
  481. if (!MemoContains(PythonOps.Id(obj))) {
  482. _memo[PythonOps.Id(obj)] = PythonTuple.MakeTuple(_memo.Count, obj);
  483. }
  484. } else {
  485. if(!_privMemo.ContainsKey(obj)) {
  486. _privMemo[obj] = _privMemo.Count;
  487. }
  488. }
  489. }
  490. private int MemoizeNew(object obj) {
  491. int res;
  492. if (_memo != null) {
  493. Debug.Assert(!_memo.ContainsKey(obj));
  494. _memo[PythonOps.Id(obj)] = PythonTuple.MakeTuple(res = _memo.Count, obj);
  495. } else {
  496. Debug.Assert(!_privMemo.ContainsKey(obj));
  497. _privMemo[obj] = res = _privMemo.Count;
  498. }
  499. return res;
  500. }
  501. private bool MemoContains(object obj) {
  502. if (_memo != null) {
  503. return _memo.Contains(PythonOps.Id(obj));
  504. }
  505. return _privMemo.ContainsKey(obj);
  506. }
  507. private bool TryWriteFastGet(CodeContext context, object obj) {
  508. int value;
  509. if (_memo != null) {
  510. return TryWriteSlowGet(context, obj);
  511. } else if (_privMemo.TryGetValue(obj, out value)) {
  512. WriteGetOrPut(context, true, value);
  513. return true;
  514. }
  515. return false;
  516. }
  517. private bool TryWriteSlowGet(CodeContext context, object obj) {
  518. object value;
  519. if (_memo.TryGetValue(obj, out value)) {
  520. WriteGetOrPut(context, true, (PythonTuple)value);
  521. return true;
  522. }
  523. return false;
  524. }
  525. [Documentation("getvalue() -> string\n\n"
  526. + "Return the value of the internal string. Raises PicklingError if a file object\n"
  527. + "was passed to this pickler's constructor."
  528. )]
  529. public object getvalue(CodeContext/*!*/ context) {
  530. if (_file is PythonReadableFileOutput) {
  531. return ((PythonReadableFileOutput)_file).GetValue(context);
  532. }
  533. throw PythonExceptions.CreateThrowable(PicklingError(context), "Attempt to getvalue() a non-list-based pickler");
  534. }
  535. #endregion
  536. #region Save functions
  537. private void Save(CodeContext/*!*/ context, object obj) {
  538. if (_persist_id == null || !TrySavePersistId(context, obj)) {
  539. PickleFunction pickleFunction;
  540. // several typees are never memoized, check for these first.
  541. if (obj == null) {
  542. SaveNone(this, context, obj);
  543. } else if (obj is int) {
  544. SaveInteger(this, context, obj);
  545. } else if(obj is BigInteger) {
  546. SaveLong(this, context, obj);
  547. } else if (obj is bool) {
  548. SaveBoolean(this, context, obj);
  549. } else if (obj is double) {
  550. SaveFloat(this, context, obj);
  551. } else if(!TryWriteFastGet(context, obj)) {
  552. if (obj is string) {
  553. // strings are common, specialize them.
  554. SaveUnicode(this, context, obj);
  555. } else {
  556. if (!_dispatchTable.TryGetValue(obj.GetType(), out pickleFunction)) {
  557. if (obj is PythonType) {
  558. // treat classes with metaclasses like regular classes
  559. pickleFunction = SaveGlobal;
  560. } else {
  561. pickleFunction = SaveObject;
  562. }
  563. }
  564. pickleFunction(this, context, obj);
  565. }
  566. }
  567. }
  568. }
  569. private bool TrySavePersistId(CodeContext context, object obj) {
  570. Debug.Assert(_persist_id != null);
  571. string res = Converter.ConvertToString(PythonContext.GetContext(context).CallSplat(_persist_id, obj));
  572. if (res != null) {
  573. SavePersId(context, res);
  574. return true;
  575. }
  576. return false;
  577. }
  578. private void SavePersId(CodeContext/*!*/ context, string res) {
  579. if (this.binary != 0) {
  580. Save(context, res);
  581. Write(context, Opcode.BinPersid);
  582. } else {
  583. Write(context, Opcode.PersId);
  584. Write(context, res);
  585. Write(context, "\n");
  586. }
  587. }
  588. private static void SaveBoolean(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
  589. Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Boolean), "arg must be bool");
  590. if (pickler._protocol < 2) {
  591. pickler.Write(context, Opcode.Int);
  592. pickler.Write(context, String.Format("0{0}", ((bool)obj) ? 1 : 0));
  593. pickler.Write(context, Newline);
  594. } else {
  595. if ((bool)obj) {
  596. pickler.Write(context, Opcode.NewTrue);
  597. } else {
  598. pickler.Write(context, Opcode.NewFalse);
  599. }
  600. }
  601. }
  602. private static void SaveDict(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
  603. Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Dict), "arg must be dict");
  604. Debug.Assert(!pickler.MemoContains(obj));
  605. int index = pickler.MemoizeNew(obj);
  606. if (pickler._protocol < 1) {
  607. pickler.Write(context, Opcode.Mark);
  608. pickler.Write(context, Opcode.Dict);
  609. } else {
  610. pickler.Write(context, Opcode.EmptyDict);
  611. }
  612. pickler.WritePut(context, index);
  613. pickler.BatchSetItems(context, (PythonDictionary)obj);
  614. }
  615. private static void SaveFloat(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
  616. Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Double), "arg must be float");
  617. if (pickler._protocol < 1) {
  618. pickler.Write(context, Opcode.Float);
  619. pickler.WriteFloatAsString(context, obj);
  620. } else {
  621. pickler.Write(context, Opcode.BinFloat);
  622. pickler.WriteFloat64(context, obj);
  623. }
  624. }
  625. private static void SaveGlobal(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
  626. Debug.Assert(
  627. DynamicHelpers.GetPythonType(obj).Equals(TypeCache.OldClass) ||
  628. DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Function) ||
  629. DynamicHelpers.GetPythonType(obj).Equals(TypeCache.BuiltinFunction) ||
  630. DynamicHelpers.GetPythonType(obj).Equals(TypeCache.PythonType) ||
  631. DynamicHelpers.GetPythonType(obj).IsSubclassOf(TypeCache.PythonType),
  632. "arg must be classic class, function, built-in function or method, or new-style type"
  633. );
  634. PythonType pt = obj as PythonType;
  635. if (pt != null) {
  636. pickler.SaveGlobalByName(context, obj, pt.Name);
  637. } else {
  638. object name;
  639. if (PythonOps.TryGetBoundAttr(context, obj, "__name__", out name)) {
  640. pickler.SaveGlobalByName(context, obj, name);
  641. } else {
  642. throw pickler.CannotPickle(context, obj, "could not determine its __name__");
  643. }
  644. }
  645. }
  646. private void SaveGlobalByName(CodeContext/*!*/ context, object obj, object name) {
  647. Debug.Assert(!MemoContains(obj));
  648. object moduleName = FindModuleForGlobal(context, obj, name);
  649. if (_protocol >= 2) {
  650. object code;
  651. if (PythonCopyReg.GetExtensionRegistry(context).TryGetValue(PythonTuple.MakeTuple(moduleName, name), out code)) {
  652. if (IsUInt8(context, code)) {
  653. Write(context, Opcode.Ext1);
  654. WriteUInt8(context, code);
  655. } else if (IsUInt16(context, code)) {
  656. Write(context, Opcode.Ext2);
  657. WriteUInt16(context, code);
  658. } else if (IsInt32(context, code)) {
  659. Write(context, Opcode.Ext4);
  660. WriteInt32(context, code);
  661. } else {
  662. throw PythonOps.RuntimeError("unrecognized integer format");
  663. }
  664. return;
  665. }
  666. }
  667. MemoizeNew(obj);
  668. Write(context, Opcode.Global);
  669. WriteStringPair(context, moduleName, name);
  670. WritePut(context, obj);
  671. }
  672. private static void SaveInstance(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
  673. Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.OldInstance), "arg must be old-class instance");
  674. Debug.Assert(!pickler.MemoContains(obj));
  675. pickler.Write(context, Opcode.Mark);
  676. // Memoize() call isn't in the usual spot to allow class to be memoized before
  677. // instance (when using proto other than 0) to match CPython's bytecode output
  678. object objClass;
  679. if (!PythonOps.TryGetBoundAttr(context, obj, "__class__", out objClass)) {
  680. throw pickler.CannotPickle(context, obj, "could not determine its __class__");
  681. }
  682. if (pickler._protocol < 1) {
  683. object className, classModuleName;
  684. if (!PythonOps.TryGetBoundAttr(context, objClass, "__name__", out className)) {
  685. throw pickler.CannotPickle(context, obj, "its __class__ has no __name__");
  686. }
  687. classModuleName = pickler.FindModuleForGlobal(context, objClass, className);
  688. Debug.Assert(!pickler.MemoContains(obj));
  689. pickler.MemoizeNew(obj);
  690. pickler.WriteInitArgs(context, obj);
  691. pickler.Write(context, Opcode.Inst);
  692. pickler.WriteStringPair(context, classModuleName, className);
  693. } else {
  694. pickler.Save(context, objClass);
  695. pickler.Memoize(obj);
  696. pickler.WriteInitArgs(context, obj);
  697. pickler.Write(context, Opcode.Obj);
  698. }
  699. pickler.WritePut(context, obj);
  700. object getStateCallable;
  701. if (PythonOps.TryGetBoundAttr(context, obj, "__getstate__", out getStateCallable)) {
  702. pickler.Save(context, PythonCalls.Call(context, getStateCallable));
  703. } else {
  704. pickler.Save(context, PythonOps.GetBoundAttr(context, obj, "__dict__"));
  705. }
  706. pickler.Write(context, Opcode.Build);
  707. }
  708. private static void SaveInteger(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
  709. Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Int32), "arg must be int");
  710. if (pickler._protocol < 1) {
  711. pickler.Write(context, Opcode.Int);
  712. pickler.WriteIntAsString(context, obj);
  713. } else {
  714. if (IsUInt8(context, obj)) {
  715. pickler.Write(context, Opcode.BinInt1);
  716. pickler.WriteUInt8(context, obj);
  717. } else if (IsUInt16(context, obj)) {
  718. pickler.Write(context, Opcode.BinInt2);
  719. pickler.WriteUInt16(context, obj);
  720. } else if (IsInt32(context, obj)) {
  721. pickler.Write(context, Opcode.BinInt);
  722. pickler.WriteInt32(context, obj);
  723. } else {
  724. throw PythonOps.RuntimeError("unrecognized integer format");
  725. }
  726. }
  727. }
  728. private static void SaveList(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
  729. Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.List), "arg must be list");
  730. Debug.Assert(!pickler.MemoContains(obj));
  731. int index = pickler.MemoizeNew(obj);
  732. if (pickler._protocol < 1) {
  733. pickler.Write(context, Opcode.Mark);
  734. pickler.Write(context, Opcode.List);
  735. } else {
  736. pickler.Write(context, Opcode.EmptyList);
  737. }
  738. pickler.WritePut(context, index);
  739. pickler.BatchAppends(context, ((IEnumerable)obj).GetEnumerator());
  740. }
  741. #if CLR2
  742. private static readonly BigInteger MaxInt = BigInteger.Create(Int32.MaxValue);
  743. private static readonly BigInteger MinInt = BigInteger.Create(Int32.MinValue);
  744. #else
  745. private static readonly BigInteger MaxInt = new BigInteger(Int32.MaxValue);
  746. private static readonly BigInteger MinInt = new BigInteger(Int32.MinValue);
  747. #endif
  748. private static void SaveLong(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
  749. Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.BigInteger), "arg must be long");
  750. BigInteger bi = (BigInteger)obj;
  751. if (pickler._protocol < 2) {
  752. pickler.Write(context, Opcode.Long);
  753. pickler.WriteLongAsString(context, obj);
  754. } else if (bi.IsZero()) {
  755. pickler.Write(context, Opcode.Long1);
  756. pickler.WriteUInt8(context, 0);
  757. } else if (bi <= MaxInt && bi >= MinInt) {
  758. pickler.Write(context, Opcode.Long1);
  759. int value = (int)bi;
  760. if (IsInt8(value)) {
  761. pickler.WriteUInt8(context, 1);
  762. pickler._file.Write(context, (char)(byte)value);
  763. } else if (IsInt16(value)) {
  764. pickler.WriteUInt8(context, 2);
  765. pickler.WriteUInt8(context, value & 0xff);
  766. pickler.WriteUInt8(context, (value >> 8) & 0xff);
  767. } else {
  768. pickler.WriteUInt8(context, 4);
  769. pickler.WriteInt32(context, value);
  770. }
  771. } else {
  772. byte[] dataBytes = bi.ToByteArray();
  773. if (dataBytes.Length < 256) {
  774. pickler.Write(context, Opcode.Long1);
  775. pickler.WriteUInt8(context, dataBytes.Length);
  776. } else {
  777. pickler.Write(context, Opcode.Long4);
  778. pickler.WriteInt32(context, dataBytes.Length);
  779. }
  780. foreach (byte b in dataBytes) {
  781. pickler.WriteUInt8(context, b);
  782. }
  783. }
  784. }
  785. private static void SaveNone(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
  786. Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Null), "arg must be None");
  787. pickler.Write(context, Opcode.NoneValue);
  788. }
  789. /// <summary>
  790. /// Call the appropriate reduce method for obj and pickle the object using
  791. /// the resulting data. Use the first available of
  792. /// copy_reg.dispatch_table[type(obj)], obj.__reduce_ex__, and obj.__reduce__.
  793. /// </summary>
  794. private void SaveObject(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
  795. Debug.Assert(!MemoContains(obj));
  796. MemoizeNew(obj);
  797. object reduceCallable, result;
  798. PythonType objType = DynamicHelpers.GetPythonType(obj);
  799. if (((IDictionary<object, object>)PythonCopyReg.GetDispatchTable(context)).TryGetValue(objType, out reduceCallable)) {
  800. result = PythonCalls.Call(context, reduceCallable, obj);
  801. } else if (PythonOps.TryGetBoundAttr(context, obj, "__reduce_ex__", out reduceCallable)) {
  802. if (obj is PythonType) {
  803. result = context.LanguageContext.Call(context, reduceCallable, obj, _protocol);
  804. } else {
  805. result = context.LanguageContext.Call(context, reduceCallable, _protocol);
  806. }
  807. } else if (PythonOps.TryGetBoundAttr(context, obj, "__reduce__", out reduceCallable)) {
  808. if (obj is PythonType) {
  809. result = context.LanguageContext.Call(context, reduceCallable, obj);
  810. } else {
  811. result = context.LanguageContext.Call(context, reduceCallable);
  812. }
  813. } else {
  814. throw PythonOps.AttributeError("no reduce function found for {0}", obj);
  815. }
  816. if (objType.Equals(TypeCache.String)) {
  817. if (!TryWriteFastGet(context, obj)) {
  818. SaveGlobalByName(context, obj, result);
  819. }
  820. } else if (result is PythonTuple) {
  821. PythonTuple rt = (PythonTuple)result;
  822. switch (rt.__len__()) {
  823. case 2:
  824. SaveReduce(context, obj, reduceCallable, rt[0], rt[1], null, null, null);
  825. break;
  826. case 3:
  827. SaveReduce(context, obj, reduceCallable, rt[0], rt[1], rt[2], null, null);
  828. break;
  829. case 4:
  830. SaveReduce(context, obj, reduceCallable, rt[0], rt[1], rt[2], rt[3], null);
  831. break;
  832. case 5:
  833. SaveReduce(context, obj, reduceCallable, rt[0], rt[1], rt[2], rt[3], rt[4]);
  834. break;
  835. default:
  836. throw CannotPickle(context, obj, "tuple returned by {0} must have to to five elements", reduceCallable);
  837. }
  838. } else {
  839. throw CannotPickle(context, obj, "{0} must return string or tuple", reduceCallable);
  840. }
  841. }
  842. /// <summary>
  843. /// Pickle the result of a reduce function.
  844. ///
  845. /// Only context, obj, func, and reduceCallable are required; all other arguments may be null.
  846. /// </summary>
  847. private void SaveReduce(CodeContext/*!*/ context, object obj, object reduceCallable, object func, object args, object state, object listItems, object dictItems) {
  848. if (!PythonOps.IsCallable(context, func)) {
  849. throw CannotPickle(context, obj, "func from reduce() should be callable");
  850. } else if (!(args is PythonTuple) && args != null) {
  851. throw CannotPickle(context, obj, "args from reduce() should be a tuple");
  852. } else if (listItems != null && !(listItems is IEnumerator)) {
  853. throw CannotPickle(context, obj, "listitems from reduce() should be a list iterator");
  854. } else if (dictItems != null && !(dictItems is IEnumerator)) {
  855. throw CannotPickle(context, obj, "dictitems from reduce() should be a dict iterator");
  856. }
  857. object funcName;
  858. string funcNameString;
  859. if (func is PythonType) {
  860. funcNameString = ((PythonType)func).Name;
  861. } else {
  862. if (!PythonOps.TryGetBoundAttr(context, func, "__name__", out funcName)) {
  863. throw CannotPickle(context, obj, "func from reduce() ({0}) should have a __name__ attribute");
  864. } else if (!Converter.TryConvertToString(funcName, out funcNameString) || funcNameString == null) {
  865. throw CannotPickle(context, obj, "__name__ of func from reduce() must be string");
  866. }
  867. }
  868. if (_protocol >= 2 && "__newobj__" == funcNameString) {
  869. if (args == null) {
  870. throw CannotPickle(context, obj, "__newobj__ arglist is None");
  871. }
  872. PythonTuple argsTuple = (PythonTuple)args;
  873. if (argsTuple.__len__() == 0) {
  874. throw CannotPickle(context, obj, "__newobj__ arglist is empty");
  875. } else if (!DynamicHelpers.GetPythonType(obj).Equals(argsTuple[0])) {
  876. throw CannotPickle(context, obj, "args[0] from __newobj__ args has the wrong class");
  877. }
  878. Save(context, argsTuple[0]);
  879. Save(context, argsTuple[new Slice(1, null)]);
  880. Write(context, Opcode.NewObj);
  881. } else {
  882. Save(context, func);
  883. Save(context, args);
  884. Write(context, Opcode.Reduce);
  885. }
  886. WritePut(context, obj);
  887. if (state != null) {
  888. Save(context, state);
  889. Write(context, Opcode.Build);
  890. }
  891. if (listItems != null) {
  892. BatchAppends(context, (IEnumerator)listItems);
  893. }
  894. if (dictItems != null) {
  895. BatchSetItems(context, (IEnumerator)dictItems);
  896. }
  897. }
  898. private static void SaveTuple(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
  899. Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.PythonTuple), "arg must be tuple");
  900. Debug.Assert(!pickler.MemoContains(obj));
  901. PythonTuple t = (PythonTuple)obj;
  902. char opcode;
  903. bool needMark = false;
  904. int len = t._data.Length;
  905. if (pickler._protocol > 0 && len == 0) {
  906. opcode = Opcode.EmptyTuple;
  907. } else if (pickler._protocol >= 2 && len == 1) {
  908. opcode = Opcode.Tuple1;
  909. } else if (pickler._protocol >= 2 && len == 2) {
  910. opcode = Opcode.Tuple2;
  911. } else if (pickler._protocol >= 2 && len == 3) {
  912. opcode = Opcode.Tuple3;
  913. } else {
  914. opcode = Opcode.Tuple;
  915. needMark = true;
  916. }
  917. if (needMark) pickler.Write(context, Opcode.Mark);
  918. var data = t._data;
  919. for (int i = 0; i < data.Length; i++) {
  920. pickler.Save(context, data[i]);
  921. }
  922. if (len > 0) {
  923. if (pickler.MemoContains(obj)) {
  924. // recursive tuple
  925. if (pickler._protocol == 1) {
  926. pickler.Write(context, Opcode.PopMark);
  927. } else {
  928. if (pickler._protocol == 0) {
  929. pickler.Write(context, Opcode.Pop);
  930. }
  931. for (int i = 0; i < len; i++) {
  932. pickler.Write(context, Opcode.Pop);
  933. }
  934. }
  935. pickler.WriteGet(context, obj);
  936. return;
  937. }
  938. pickler.Write(context, opcode);
  939. pickler.Memoize(t);
  940. pickler.WritePut(context, t);
  941. } else {
  942. pickler.Write(context, opcode);
  943. }
  944. }
  945. private static void SaveUnicode(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {
  946. Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.String), "arg must be unicode");
  947. Debug.Assert(!pickler.MemoContains(obj));
  948. if (pickler._memo != null) {
  949. pickler.MemoizeNew(obj);
  950. if (pickler._protocol < 1) {

Large files files are truncated, but you can click here to view the full file