/SparkleLib/Git/SparkleFetcherGit.cs

http://github.com/hbons/SparkleShare · C# · 450 lines · 311 code · 108 blank · 31 comment · 63 complexity · 4393be380ebb85f5d216b981122d7708 MD5 · raw file

  1. // SparkleShare, a collaboration and sharing tool.
  2. // Copyright (C) 2010 Hylke Bons <hylkebons@gmail.com>
  3. //
  4. // This program is free software: you can redistribute it and/or modify
  5. // it under the terms of the GNU Lesser General Public License as
  6. // published by the Free Software Foundation, either version 3 of the
  7. // License, or (at your option) any later version.
  8. //
  9. // This program is distributed in the hope that it will be useful,
  10. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. // GNU General Public License for more details.
  13. //
  14. // You should have received a copy of the GNU General Public License
  15. // along with this program. If not, see <http://www.gnu.org/licenses/>.
  16. using System;
  17. using System.Diagnostics;
  18. using System.Globalization;
  19. using System.IO;
  20. using System.Text.RegularExpressions;
  21. using System.Threading;
  22. using SparkleLib;
  23. namespace SparkleLib.Git {
  24. public class SparkleFetcher : SparkleFetcherSSH {
  25. private SparkleGit git;
  26. private bool use_git_bin;
  27. private string cached_salt;
  28. private Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled);
  29. private Regex speed_regex = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled);
  30. private bool crypto_password_is_hashed = true;
  31. private string crypto_salt {
  32. get {
  33. if (!string.IsNullOrEmpty (this.cached_salt))
  34. return this.cached_salt;
  35. // Check if the repo's salt is stored in a branch...
  36. SparkleGit git = new SparkleGit (TargetFolder, "ls-remote --heads");
  37. string branches = git.StartAndReadStandardOutput ();
  38. Regex salt_regex = new Regex ("refs/heads/salt-([0-9a-f]+)");
  39. Match salt_match = salt_regex.Match (branches);
  40. if (salt_match.Success)
  41. this.cached_salt = salt_match.Groups [1].Value;
  42. // ...if not, create a new salt for the repo
  43. if (string.IsNullOrEmpty (this.cached_salt)) {
  44. this.cached_salt = GenerateCryptoSalt ();
  45. string salt_file_path = new string [] { TargetFolder, ".git", "salt" }.Combine ();
  46. // Temporarily store the salt in a file, so the Repo object can
  47. // push it to a branch on the host later
  48. File.WriteAllText (salt_file_path, this.cached_salt);
  49. }
  50. return this.cached_salt;
  51. }
  52. }
  53. public SparkleFetcher (SparkleFetcherInfo info) : base (info)
  54. {
  55. if (RemoteUrl.ToString ().StartsWith ("ssh+"))
  56. RemoteUrl = new Uri ("ssh" + RemoteUrl.ToString ().Substring (RemoteUrl.ToString ().IndexOf ("://")));
  57. Uri uri = RemoteUrl;
  58. if (!uri.Scheme.Equals ("ssh") && !uri.Scheme.Equals ("https") &&
  59. !uri.Scheme.Equals ("http") && !uri.Scheme.Equals ("git")) {
  60. uri = new Uri ("ssh://" + uri);
  61. }
  62. if (uri.Host.Equals ("gitorious.org") && !uri.Scheme.StartsWith ("http")) {
  63. if (!uri.AbsolutePath.Equals ("/") &&
  64. !uri.AbsolutePath.EndsWith (".git")) {
  65. uri = new Uri ("ssh://git@gitorious.org" + uri.AbsolutePath + ".git");
  66. } else {
  67. uri = new Uri ("ssh://git@gitorious.org" + uri.AbsolutePath);
  68. }
  69. } else if (uri.Host.Equals ("github.com") && !uri.Scheme.StartsWith ("http")) {
  70. uri = new Uri ("ssh://git@github.com" + uri.AbsolutePath);
  71. } else if (uri.Host.Equals ("bitbucket.org") && !uri.Scheme.StartsWith ("http")) {
  72. // Nothing really
  73. } else {
  74. if (string.IsNullOrEmpty (uri.UserInfo) && !uri.Scheme.StartsWith ("http")) {
  75. if (uri.Port == -1)
  76. uri = new Uri (uri.Scheme + "://storage@" + uri.Host + uri.AbsolutePath);
  77. else
  78. uri = new Uri (uri.Scheme + "://storage@" + uri.Host + ":" + uri.Port + uri.AbsolutePath);
  79. }
  80. this.use_git_bin = false; // TODO
  81. }
  82. RemoteUrl = uri;
  83. }
  84. public override bool Fetch ()
  85. {
  86. if (!base.Fetch ())
  87. return false;
  88. if (FetchPriorHistory) {
  89. this.git = new SparkleGit (SparkleConfig.DefaultConfig.TmpPath,
  90. "clone --progress --no-checkout \"" + RemoteUrl + "\" \"" + TargetFolder + "\"");
  91. } else {
  92. this.git = new SparkleGit (SparkleConfig.DefaultConfig.TmpPath,
  93. "clone --progress --no-checkout --depth=1 \"" + RemoteUrl + "\" \"" + TargetFolder + "\"");
  94. }
  95. this.git.StartInfo.RedirectStandardError = true;
  96. this.git.Start ();
  97. double percentage = 1.0;
  98. DateTime last_change = DateTime.Now;
  99. TimeSpan change_interval = new TimeSpan (0, 0, 0, 1);
  100. try {
  101. while (!this.git.StandardError.EndOfStream) {
  102. string line = this.git.StandardError.ReadLine ();
  103. Match match = this.progress_regex.Match (line);
  104. double number = 0.0;
  105. double speed = 0.0;
  106. if (match.Success) {
  107. try {
  108. number = double.Parse (match.Groups [1].Value, new CultureInfo ("en-US"));
  109. } catch (FormatException) {
  110. SparkleLogger.LogInfo ("Git", "Error parsing progress: \"" + match.Groups [1] + "\"");
  111. }
  112. // The pushing progress consists of two stages: the "Compressing
  113. // objects" stage which we count as 20% of the total progress, and
  114. // the "Writing objects" stage which we count as the last 80%
  115. if (line.Contains ("Compressing")) {
  116. // "Compressing objects" stage
  117. number = (number / 100 * 20);
  118. } else {
  119. // "Writing objects" stage
  120. number = (number / 100 * 80 + 20);
  121. Match speed_match = this.speed_regex.Match (line);
  122. if (speed_match.Success) {
  123. try {
  124. speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024;
  125. } catch (FormatException) {
  126. SparkleLogger.LogInfo ("Git", "Error parsing speed: \"" + speed_match.Groups [1] + "\"");
  127. }
  128. if (speed_match.Groups [2].Value.Equals ("M"))
  129. speed = speed * 1024;
  130. }
  131. }
  132. } else {
  133. SparkleLogger.LogInfo ("Fetcher", line);
  134. line = line.Trim (new char [] {' ', '@'});
  135. if (line.StartsWith ("fatal:", StringComparison.InvariantCultureIgnoreCase) ||
  136. line.StartsWith ("error:", StringComparison.InvariantCultureIgnoreCase)) {
  137. base.errors.Add (line);
  138. } else if (line.StartsWith ("WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!")) {
  139. base.errors.Add ("warning: Remote host identification has changed!");
  140. } else if (line.StartsWith ("WARNING: POSSIBLE DNS SPOOFING DETECTED!")) {
  141. base.errors.Add ("warning: Possible DNS spoofing detected!");
  142. }
  143. }
  144. if (number >= percentage) {
  145. percentage = number;
  146. if (DateTime.Compare (last_change, DateTime.Now.Subtract (change_interval)) < 0) {
  147. base.OnProgressChanged (percentage, speed);
  148. last_change = DateTime.Now;
  149. }
  150. }
  151. }
  152. } catch (Exception) {
  153. IsActive = false;
  154. return false;
  155. }
  156. this.git.WaitForExit ();
  157. if (this.git.ExitCode == 0) {
  158. while (percentage < 100) {
  159. percentage += 25;
  160. if (percentage >= 100)
  161. break;
  162. Thread.Sleep (500);
  163. base.OnProgressChanged (percentage, 0);
  164. }
  165. base.OnProgressChanged (100, 0);
  166. InstallConfiguration ();
  167. InstallExcludeRules ();
  168. InstallAttributeRules ();
  169. return true;
  170. } else {
  171. return false;
  172. }
  173. }
  174. public override bool IsFetchedRepoEmpty {
  175. get {
  176. SparkleGit git = new SparkleGit (TargetFolder, "rev-parse HEAD");
  177. git.StartAndWaitForExit ();
  178. return (git.ExitCode != 0);
  179. }
  180. }
  181. public override void EnableFetchedRepoCrypto (string password)
  182. {
  183. // Set up the encryption filter
  184. SparkleGit git_config_smudge = new SparkleGit (TargetFolder,
  185. "config filter.encryption.smudge \"openssl enc -d -aes-256-cbc -base64 -S " + this.crypto_salt +
  186. " -pass file:.git/info/encryption_password\"");
  187. SparkleGit git_config_clean = new SparkleGit (TargetFolder,
  188. "config filter.encryption.clean \"openssl enc -e -aes-256-cbc -base64 -S " + this.crypto_salt +
  189. " -pass file:.git/info/encryption_password\"");
  190. git_config_smudge.StartAndWaitForExit ();
  191. git_config_clean.StartAndWaitForExit ();
  192. // Pass all files through the encryption filter
  193. string git_attributes_file_path = new string [] { TargetFolder, ".git", "info", "attributes" }.Combine ();
  194. File.WriteAllText (git_attributes_file_path, "\n* filter=encryption");
  195. // Store the password
  196. string password_file_path = new string [] { TargetFolder, ".git", "info", "encryption_password" }.Combine ();
  197. if (this.crypto_password_is_hashed)
  198. File.WriteAllText (password_file_path, password.SHA256 (this.crypto_salt));
  199. else
  200. File.WriteAllText (password_file_path, password);
  201. }
  202. public override bool IsFetchedRepoPasswordCorrect (string password)
  203. {
  204. string password_check_file_path = Path.Combine (TargetFolder, ".sparkleshare");
  205. if (!File.Exists (password_check_file_path)) {
  206. SparkleGit git = new SparkleGit (TargetFolder, "show HEAD:.sparkleshare");
  207. string output = git.StartAndReadStandardOutput ();
  208. if (git.ExitCode == 0)
  209. File.WriteAllText (password_check_file_path, output);
  210. else
  211. return false;
  212. }
  213. Process process = new Process ();
  214. process.EnableRaisingEvents = true;
  215. process.StartInfo.FileName = "openssl";
  216. process.StartInfo.WorkingDirectory = TargetFolder;
  217. process.StartInfo.UseShellExecute = false;
  218. process.StartInfo.RedirectStandardOutput = true;
  219. process.StartInfo.CreateNoWindow = true;
  220. string [] possible_passwords = new string [] {
  221. password.SHA256 (this.crypto_salt),
  222. password
  223. };
  224. int i = 0;
  225. foreach (string possible_password in possible_passwords) {
  226. process.StartInfo.Arguments = "enc -d -aes-256-cbc -base64 -pass pass:\"" + possible_password + "\"" +
  227. " -in \"" + password_check_file_path + "\"";
  228. SparkleLogger.LogInfo ("Cmd | " + System.IO.Path.GetFileName (process.StartInfo.WorkingDirectory),
  229. System.IO.Path.GetFileName (process.StartInfo.FileName) + " " + process.StartInfo.Arguments);
  230. process.Start ();
  231. process.WaitForExit ();
  232. if (process.ExitCode == 0) {
  233. if (i > 0)
  234. this.crypto_password_is_hashed = false;
  235. File.Delete (password_check_file_path);
  236. return true;
  237. }
  238. i++;
  239. }
  240. return false;
  241. }
  242. public override void Stop ()
  243. {
  244. try {
  245. if (this.git != null && !this.git.HasExited) {
  246. this.git.Kill ();
  247. this.git.Dispose ();
  248. }
  249. } catch (Exception e) {
  250. SparkleLogger.LogInfo ("Fetcher", "Failed to dispose properly", e);
  251. }
  252. if (Directory.Exists (TargetFolder)) {
  253. try {
  254. Directory.Delete (TargetFolder, true /* Recursive */ );
  255. SparkleLogger.LogInfo ("Fetcher", "Deleted '" + TargetFolder + "'");
  256. } catch (Exception e) {
  257. SparkleLogger.LogInfo ("Fetcher", "Failed to delete '" + TargetFolder + "'", e);
  258. }
  259. }
  260. }
  261. public override void Complete ()
  262. {
  263. if (!IsFetchedRepoEmpty) {
  264. SparkleGit git = new SparkleGit (TargetFolder, "checkout --quiet HEAD");
  265. git.StartAndWaitForExit ();
  266. }
  267. base.Complete ();
  268. }
  269. private void InstallConfiguration ()
  270. {
  271. string [] settings = new string [] {
  272. "core.autocrlf input",
  273. "core.quotepath false", // Don't quote "unusual" characters in path names
  274. "core.ignorecase false", // Be case sensitive explicitly to work on Mac
  275. "core.filemode false", // Ignore permission changes
  276. "core.precomposeunicode true", // Use the same Unicode form on all filesystems
  277. "core.safecrlf false",
  278. "core.excludesfile \"\"",
  279. "core.packedGitLimit 128m", // Some memory limiting options
  280. "core.packedGitWindowSize 128m",
  281. "pack.deltaCacheSize 128m",
  282. "pack.packSizeLimit 128m",
  283. "pack.windowMemory 128m",
  284. "push.default matching"
  285. };
  286. if (SparkleBackend.Platform == PlatformID.Win32NT)
  287. settings [0] = "core.autocrlf true";
  288. foreach (string setting in settings) {
  289. SparkleGit git_config = new SparkleGit (TargetFolder, "config " + setting);
  290. git_config.StartAndWaitForExit ();
  291. }
  292. if (this.use_git_bin)
  293. InstallGitBinConfiguration ();
  294. }
  295. public void InstallGitBinConfiguration ()
  296. {
  297. string [] settings = new string [] {
  298. "core.bigFileThreshold 1024g",
  299. "filter.bin.clean \"git bin clean %f\"",
  300. "filter.bin.smudge \"git bin smudge\""
  301. };
  302. foreach (string setting in settings) {
  303. SparkleGit git_config = new SparkleGit (TargetFolder, "config " + setting);
  304. git_config.StartAndWaitForExit ();
  305. }
  306. }
  307. // Add a .gitignore file to the repo
  308. private void InstallExcludeRules ()
  309. {
  310. string git_info_path = new string [] { TargetFolder, ".git", "info" }.Combine ();
  311. if (!Directory.Exists (git_info_path))
  312. Directory.CreateDirectory (git_info_path);
  313. string exclude_rules = string.Join (Environment.NewLine, ExcludeRules);
  314. string exclude_rules_file_path = new string [] { git_info_path, "exclude" }.Combine ();
  315. File.WriteAllText (exclude_rules_file_path, exclude_rules);
  316. }
  317. private void InstallAttributeRules ()
  318. {
  319. string attribute_rules_file_path = new string [] { TargetFolder, ".git", "info", "attributes" }.Combine ();
  320. TextWriter writer = new StreamWriter (attribute_rules_file_path);
  321. if (this.use_git_bin) {
  322. writer.WriteLine ("* filter=bin binary");
  323. } else {
  324. // Compile a list of files we don't want Git to compress.
  325. // Not compressing already compressed files decreases memory usage and increases speed
  326. string [] extensions = new string [] {
  327. "jpg", "jpeg", "png", "tiff", "gif", // Images
  328. "flac", "mp3", "ogg", "oga", // Audio
  329. "avi", "mov", "mpg", "mpeg", "mkv", "ogv", "ogx", "webm", // Video
  330. "zip", "gz", "bz", "bz2", "rpm", "deb", "tgz", "rar", "ace", "7z", "pak", "tc", "iso", ".dmg" // Archives
  331. };
  332. foreach (string extension in extensions) {
  333. writer.WriteLine ("*." + extension + " -delta");
  334. writer.WriteLine ("*." + extension.ToUpper () + " -delta");
  335. }
  336. writer.WriteLine ("*.txt text");
  337. writer.WriteLine ("*.TXT text");
  338. }
  339. writer.Close ();
  340. }
  341. }
  342. }