/SparkleLib/Git/SparkleFetcherGit.cs
C# | 450 lines | 311 code | 108 blank | 31 comment | 63 complexity | 4393be380ebb85f5d216b981122d7708 MD5 | raw file
1// SparkleShare, a collaboration and sharing tool. 2// Copyright (C) 2010 Hylke Bons <hylkebons@gmail.com> 3// 4// This program is free software: you can redistribute it and/or modify 5// it under the terms of the GNU Lesser General Public License as 6// published by the Free Software Foundation, either version 3 of the 7// License, or (at your option) any later version. 8// 9// This program is distributed in the hope that it will be useful, 10// but WITHOUT ANY WARRANTY; without even the implied warranty of 11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12// GNU General Public License for more details. 13// 14// You should have received a copy of the GNU General Public License 15// along with this program. If not, see <http://www.gnu.org/licenses/>. 16 17 18using System; 19using System.Diagnostics; 20using System.Globalization; 21using System.IO; 22using System.Text.RegularExpressions; 23using System.Threading; 24using SparkleLib; 25 26namespace SparkleLib.Git { 27 28 public class SparkleFetcher : SparkleFetcherSSH { 29 30 private SparkleGit git; 31 private bool use_git_bin; 32 private string cached_salt; 33 34 private Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled); 35 private Regex speed_regex = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled); 36 37 private bool crypto_password_is_hashed = true; 38 39 private string crypto_salt { 40 get { 41 if (!string.IsNullOrEmpty (this.cached_salt)) 42 return this.cached_salt; 43 44 // Check if the repo's salt is stored in a branch... 45 SparkleGit git = new SparkleGit (TargetFolder, "ls-remote --heads"); 46 string branches = git.StartAndReadStandardOutput (); 47 Regex salt_regex = new Regex ("refs/heads/salt-([0-9a-f]+)"); 48 Match salt_match = salt_regex.Match (branches); 49 50 if (salt_match.Success) 51 this.cached_salt = salt_match.Groups [1].Value; 52 53 // ...if not, create a new salt for the repo 54 if (string.IsNullOrEmpty (this.cached_salt)) { 55 this.cached_salt = GenerateCryptoSalt (); 56 string salt_file_path = new string [] { TargetFolder, ".git", "salt" }.Combine (); 57 58 // Temporarily store the salt in a file, so the Repo object can 59 // push it to a branch on the host later 60 File.WriteAllText (salt_file_path, this.cached_salt); 61 } 62 63 return this.cached_salt; 64 } 65 } 66 67 68 public SparkleFetcher (SparkleFetcherInfo info) : base (info) 69 { 70 if (RemoteUrl.ToString ().StartsWith ("ssh+")) 71 RemoteUrl = new Uri ("ssh" + RemoteUrl.ToString ().Substring (RemoteUrl.ToString ().IndexOf ("://"))); 72 73 Uri uri = RemoteUrl; 74 75 if (!uri.Scheme.Equals ("ssh") && !uri.Scheme.Equals ("https") && 76 !uri.Scheme.Equals ("http") && !uri.Scheme.Equals ("git")) { 77 78 uri = new Uri ("ssh://" + uri); 79 } 80 81 if (uri.Host.Equals ("gitorious.org") && !uri.Scheme.StartsWith ("http")) { 82 if (!uri.AbsolutePath.Equals ("/") && 83 !uri.AbsolutePath.EndsWith (".git")) { 84 85 uri = new Uri ("ssh://git@gitorious.org" + uri.AbsolutePath + ".git"); 86 87 } else { 88 uri = new Uri ("ssh://git@gitorious.org" + uri.AbsolutePath); 89 } 90 91 } else if (uri.Host.Equals ("github.com") && !uri.Scheme.StartsWith ("http")) { 92 uri = new Uri ("ssh://git@github.com" + uri.AbsolutePath); 93 94 } else if (uri.Host.Equals ("bitbucket.org") && !uri.Scheme.StartsWith ("http")) { 95 // Nothing really 96 97 } else { 98 if (string.IsNullOrEmpty (uri.UserInfo) && !uri.Scheme.StartsWith ("http")) { 99 if (uri.Port == -1) 100 uri = new Uri (uri.Scheme + "://storage@" + uri.Host + uri.AbsolutePath); 101 else 102 uri = new Uri (uri.Scheme + "://storage@" + uri.Host + ":" + uri.Port + uri.AbsolutePath); 103 } 104 105 this.use_git_bin = false; // TODO 106 } 107 108 RemoteUrl = uri; 109 } 110 111 112 public override bool Fetch () 113 { 114 if (!base.Fetch ()) 115 return false; 116 117 if (FetchPriorHistory) { 118 this.git = new SparkleGit (SparkleConfig.DefaultConfig.TmpPath, 119 "clone --progress --no-checkout \"" + RemoteUrl + "\" \"" + TargetFolder + "\""); 120 121 } else { 122 this.git = new SparkleGit (SparkleConfig.DefaultConfig.TmpPath, 123 "clone --progress --no-checkout --depth=1 \"" + RemoteUrl + "\" \"" + TargetFolder + "\""); 124 } 125 126 this.git.StartInfo.RedirectStandardError = true; 127 this.git.Start (); 128 129 double percentage = 1.0; 130 131 DateTime last_change = DateTime.Now; 132 TimeSpan change_interval = new TimeSpan (0, 0, 0, 1); 133 134 try { 135 while (!this.git.StandardError.EndOfStream) { 136 string line = this.git.StandardError.ReadLine (); 137 Match match = this.progress_regex.Match (line); 138 139 double number = 0.0; 140 double speed = 0.0; 141 if (match.Success) { 142 try { 143 number = double.Parse (match.Groups [1].Value, new CultureInfo ("en-US")); 144 145 } catch (FormatException) { 146 SparkleLogger.LogInfo ("Git", "Error parsing progress: \"" + match.Groups [1] + "\""); 147 } 148 149 // The pushing progress consists of two stages: the "Compressing 150 // objects" stage which we count as 20% of the total progress, and 151 // the "Writing objects" stage which we count as the last 80% 152 if (line.Contains ("Compressing")) { 153 // "Compressing objects" stage 154 number = (number / 100 * 20); 155 156 } else { 157 // "Writing objects" stage 158 number = (number / 100 * 80 + 20); 159 Match speed_match = this.speed_regex.Match (line); 160 161 if (speed_match.Success) { 162 try { 163 speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024; 164 165 } catch (FormatException) { 166 SparkleLogger.LogInfo ("Git", "Error parsing speed: \"" + speed_match.Groups [1] + "\""); 167 } 168 169 if (speed_match.Groups [2].Value.Equals ("M")) 170 speed = speed * 1024; 171 } 172 } 173 174 } else { 175 SparkleLogger.LogInfo ("Fetcher", line); 176 line = line.Trim (new char [] {' ', '@'}); 177 178 if (line.StartsWith ("fatal:", StringComparison.InvariantCultureIgnoreCase) || 179 line.StartsWith ("error:", StringComparison.InvariantCultureIgnoreCase)) { 180 181 base.errors.Add (line); 182 183 } else if (line.StartsWith ("WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!")) { 184 base.errors.Add ("warning: Remote host identification has changed!"); 185 186 } else if (line.StartsWith ("WARNING: POSSIBLE DNS SPOOFING DETECTED!")) { 187 base.errors.Add ("warning: Possible DNS spoofing detected!"); 188 } 189 } 190 191 if (number >= percentage) { 192 percentage = number; 193 194 if (DateTime.Compare (last_change, DateTime.Now.Subtract (change_interval)) < 0) { 195 base.OnProgressChanged (percentage, speed); 196 last_change = DateTime.Now; 197 } 198 } 199 } 200 201 } catch (Exception) { 202 IsActive = false; 203 return false; 204 } 205 206 this.git.WaitForExit (); 207 208 if (this.git.ExitCode == 0) { 209 while (percentage < 100) { 210 percentage += 25; 211 212 if (percentage >= 100) 213 break; 214 215 Thread.Sleep (500); 216 base.OnProgressChanged (percentage, 0); 217 } 218 219 base.OnProgressChanged (100, 0); 220 221 InstallConfiguration (); 222 InstallExcludeRules (); 223 InstallAttributeRules (); 224 225 return true; 226 227 } else { 228 return false; 229 } 230 } 231 232 233 public override bool IsFetchedRepoEmpty { 234 get { 235 SparkleGit git = new SparkleGit (TargetFolder, "rev-parse HEAD"); 236 git.StartAndWaitForExit (); 237 238 return (git.ExitCode != 0); 239 } 240 } 241 242 243 public override void EnableFetchedRepoCrypto (string password) 244 { 245 // Set up the encryption filter 246 SparkleGit git_config_smudge = new SparkleGit (TargetFolder, 247 "config filter.encryption.smudge \"openssl enc -d -aes-256-cbc -base64 -S " + this.crypto_salt + 248 " -pass file:.git/info/encryption_password\""); 249 250 SparkleGit git_config_clean = new SparkleGit (TargetFolder, 251 "config filter.encryption.clean \"openssl enc -e -aes-256-cbc -base64 -S " + this.crypto_salt + 252 " -pass file:.git/info/encryption_password\""); 253 254 git_config_smudge.StartAndWaitForExit (); 255 git_config_clean.StartAndWaitForExit (); 256 257 // Pass all files through the encryption filter 258 string git_attributes_file_path = new string [] { TargetFolder, ".git", "info", "attributes" }.Combine (); 259 File.WriteAllText (git_attributes_file_path, "\n* filter=encryption"); 260 261 // Store the password 262 string password_file_path = new string [] { TargetFolder, ".git", "info", "encryption_password" }.Combine (); 263 264 if (this.crypto_password_is_hashed) 265 File.WriteAllText (password_file_path, password.SHA256 (this.crypto_salt)); 266 else 267 File.WriteAllText (password_file_path, password); 268 } 269 270 271 public override bool IsFetchedRepoPasswordCorrect (string password) 272 { 273 string password_check_file_path = Path.Combine (TargetFolder, ".sparkleshare"); 274 275 if (!File.Exists (password_check_file_path)) { 276 SparkleGit git = new SparkleGit (TargetFolder, "show HEAD:.sparkleshare"); 277 string output = git.StartAndReadStandardOutput (); 278 279 if (git.ExitCode == 0) 280 File.WriteAllText (password_check_file_path, output); 281 else 282 return false; 283 } 284 285 Process process = new Process (); 286 process.EnableRaisingEvents = true; 287 process.StartInfo.FileName = "openssl"; 288 process.StartInfo.WorkingDirectory = TargetFolder; 289 process.StartInfo.UseShellExecute = false; 290 process.StartInfo.RedirectStandardOutput = true; 291 process.StartInfo.CreateNoWindow = true; 292 293 string [] possible_passwords = new string [] { 294 password.SHA256 (this.crypto_salt), 295 password 296 }; 297 298 int i = 0; 299 foreach (string possible_password in possible_passwords) { 300 process.StartInfo.Arguments = "enc -d -aes-256-cbc -base64 -pass pass:\"" + possible_password + "\"" + 301 " -in \"" + password_check_file_path + "\""; 302 303 SparkleLogger.LogInfo ("Cmd | " + System.IO.Path.GetFileName (process.StartInfo.WorkingDirectory), 304 System.IO.Path.GetFileName (process.StartInfo.FileName) + " " + process.StartInfo.Arguments); 305 306 process.Start (); 307 process.WaitForExit (); 308 309 if (process.ExitCode == 0) { 310 if (i > 0) 311 this.crypto_password_is_hashed = false; 312 313 File.Delete (password_check_file_path); 314 return true; 315 } 316 317 i++; 318 } 319 320 return false; 321 } 322 323 324 public override void Stop () 325 { 326 try { 327 if (this.git != null && !this.git.HasExited) { 328 this.git.Kill (); 329 this.git.Dispose (); 330 } 331 332 } catch (Exception e) { 333 SparkleLogger.LogInfo ("Fetcher", "Failed to dispose properly", e); 334 } 335 336 if (Directory.Exists (TargetFolder)) { 337 try { 338 Directory.Delete (TargetFolder, true /* Recursive */ ); 339 SparkleLogger.LogInfo ("Fetcher", "Deleted '" + TargetFolder + "'"); 340 341 } catch (Exception e) { 342 SparkleLogger.LogInfo ("Fetcher", "Failed to delete '" + TargetFolder + "'", e); 343 } 344 } 345 } 346 347 348 public override void Complete () 349 { 350 if (!IsFetchedRepoEmpty) { 351 SparkleGit git = new SparkleGit (TargetFolder, "checkout --quiet HEAD"); 352 git.StartAndWaitForExit (); 353 } 354 355 base.Complete (); 356 } 357 358 359 private void InstallConfiguration () 360 { 361 string [] settings = new string [] { 362 "core.autocrlf input", 363 "core.quotepath false", // Don't quote "unusual" characters in path names 364 "core.ignorecase false", // Be case sensitive explicitly to work on Mac 365 "core.filemode false", // Ignore permission changes 366 "core.precomposeunicode true", // Use the same Unicode form on all filesystems 367 "core.safecrlf false", 368 "core.excludesfile \"\"", 369 "core.packedGitLimit 128m", // Some memory limiting options 370 "core.packedGitWindowSize 128m", 371 "pack.deltaCacheSize 128m", 372 "pack.packSizeLimit 128m", 373 "pack.windowMemory 128m", 374 "push.default matching" 375 }; 376 377 if (SparkleBackend.Platform == PlatformID.Win32NT) 378 settings [0] = "core.autocrlf true"; 379 380 foreach (string setting in settings) { 381 SparkleGit git_config = new SparkleGit (TargetFolder, "config " + setting); 382 git_config.StartAndWaitForExit (); 383 } 384 385 if (this.use_git_bin) 386 InstallGitBinConfiguration (); 387 } 388 389 390 public void InstallGitBinConfiguration () 391 { 392 string [] settings = new string [] { 393 "core.bigFileThreshold 1024g", 394 "filter.bin.clean \"git bin clean %f\"", 395 "filter.bin.smudge \"git bin smudge\"" 396 }; 397 398 foreach (string setting in settings) { 399 SparkleGit git_config = new SparkleGit (TargetFolder, "config " + setting); 400 git_config.StartAndWaitForExit (); 401 } 402 } 403 404 405 // Add a .gitignore file to the repo 406 private void InstallExcludeRules () 407 { 408 string git_info_path = new string [] { TargetFolder, ".git", "info" }.Combine (); 409 410 if (!Directory.Exists (git_info_path)) 411 Directory.CreateDirectory (git_info_path); 412 413 string exclude_rules = string.Join (Environment.NewLine, ExcludeRules); 414 string exclude_rules_file_path = new string [] { git_info_path, "exclude" }.Combine (); 415 416 File.WriteAllText (exclude_rules_file_path, exclude_rules); 417 } 418 419 420 private void InstallAttributeRules () 421 { 422 string attribute_rules_file_path = new string [] { TargetFolder, ".git", "info", "attributes" }.Combine (); 423 TextWriter writer = new StreamWriter (attribute_rules_file_path); 424 425 if (this.use_git_bin) { 426 writer.WriteLine ("* filter=bin binary"); 427 428 } else { 429 // Compile a list of files we don't want Git to compress. 430 // Not compressing already compressed files decreases memory usage and increases speed 431 string [] extensions = new string [] { 432 "jpg", "jpeg", "png", "tiff", "gif", // Images 433 "flac", "mp3", "ogg", "oga", // Audio 434 "avi", "mov", "mpg", "mpeg", "mkv", "ogv", "ogx", "webm", // Video 435 "zip", "gz", "bz", "bz2", "rpm", "deb", "tgz", "rar", "ace", "7z", "pak", "tc", "iso", ".dmg" // Archives 436 }; 437 438 foreach (string extension in extensions) { 439 writer.WriteLine ("*." + extension + " -delta"); 440 writer.WriteLine ("*." + extension.ToUpper () + " -delta"); 441 } 442 443 writer.WriteLine ("*.txt text"); 444 writer.WriteLine ("*.TXT text"); 445 } 446 447 writer.Close (); 448 } 449 } 450}