/SparkleLib/Git/SparkleFetcherGit.cs

  1//   SparkleShare, a collaboration and sharing tool.
  2//   Copyright (C) 2010  Hylke Bons <hylkebons@gmail.com>
  3//
  4//   This program is free software: you can redistribute it and/or modify
  5//   it under the terms of the GNU Lesser General Public License as
  6//   published by the Free Software Foundation, either version 3 of the
  7//   License, or (at your option) any later version.
  8//
  9//   This program is distributed in the hope that it will be useful,
 10//   but WITHOUT ANY WARRANTY; without even the implied warranty of
 11//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 12//   GNU General Public License for more details.
 13//
 14//   You should have received a copy of the GNU General Public License
 15//   along with this program. If not, see <http://www.gnu.org/licenses/>.
 16
 17
 18using System;
 19using System.Diagnostics;
 20using System.Globalization;
 21using System.IO;
 22using System.Text.RegularExpressions;
 23using System.Threading;
 24using SparkleLib;
 25
 26namespace SparkleLib.Git {
 27
 28    public class SparkleFetcher : SparkleFetcherSSH {
 29
 30        private SparkleGit git;
 31        private bool use_git_bin;
 32        private string cached_salt;
 33
 34        private Regex progress_regex = new Regex (@"([0-9]+)%", RegexOptions.Compiled);
 35        private Regex speed_regex    = new Regex (@"([0-9\.]+) ([KM])iB/s", RegexOptions.Compiled);
 36
 37        private bool crypto_password_is_hashed = true;
 38
 39        private string crypto_salt {
 40            get {
 41                if (!string.IsNullOrEmpty (this.cached_salt))
 42                    return this.cached_salt;
 43
 44                // Check if the repo's salt is stored in a branch...
 45                SparkleGit git   = new SparkleGit (TargetFolder, "ls-remote --heads");
 46                string branches  = git.StartAndReadStandardOutput ();
 47                Regex salt_regex = new Regex ("refs/heads/salt-([0-9a-f]+)");
 48				Match salt_match = salt_regex.Match (branches);
 49
 50				if (salt_match.Success)
 51					this.cached_salt = salt_match.Groups [1].Value;
 52
 53                // ...if not, create a new salt for the repo
 54                if (string.IsNullOrEmpty (this.cached_salt)) {
 55                    this.cached_salt      = GenerateCryptoSalt ();
 56                    string salt_file_path = new string [] { TargetFolder, ".git", "salt" }.Combine ();
 57
 58                    // Temporarily store the salt in a file, so the Repo object can
 59                    // push it to a branch on the host later
 60                    File.WriteAllText (salt_file_path, this.cached_salt);
 61                }
 62
 63                return this.cached_salt;
 64            }
 65        }
 66
 67
 68        public SparkleFetcher (SparkleFetcherInfo info) : base (info)
 69        {
 70            if (RemoteUrl.ToString ().StartsWith ("ssh+"))
 71                RemoteUrl = new Uri ("ssh" + RemoteUrl.ToString ().Substring (RemoteUrl.ToString ().IndexOf ("://")));
 72
 73            Uri uri = RemoteUrl;
 74
 75            if (!uri.Scheme.Equals ("ssh") && !uri.Scheme.Equals ("https") &&
 76                !uri.Scheme.Equals ("http") && !uri.Scheme.Equals ("git")) {
 77
 78                uri = new Uri ("ssh://" + uri);
 79            }
 80
 81            if (uri.Host.Equals ("gitorious.org") && !uri.Scheme.StartsWith ("http")) {
 82                if (!uri.AbsolutePath.Equals ("/") &&
 83                    !uri.AbsolutePath.EndsWith (".git")) {
 84
 85                    uri = new Uri ("ssh://git@gitorious.org" + uri.AbsolutePath + ".git");
 86
 87                } else {
 88                    uri = new Uri ("ssh://git@gitorious.org" + uri.AbsolutePath);
 89                }
 90
 91            } else if (uri.Host.Equals ("github.com") && !uri.Scheme.StartsWith ("http")) {
 92                uri = new Uri ("ssh://git@github.com" + uri.AbsolutePath);
 93
 94            } else if (uri.Host.Equals ("bitbucket.org") && !uri.Scheme.StartsWith ("http")) {
 95                // Nothing really
 96
 97            } else {
 98                if (string.IsNullOrEmpty (uri.UserInfo) && !uri.Scheme.StartsWith ("http")) {
 99                    if (uri.Port == -1)
100                        uri = new Uri (uri.Scheme + "://storage@" + uri.Host + uri.AbsolutePath);
101                    else
102                        uri = new Uri (uri.Scheme + "://storage@" + uri.Host + ":" + uri.Port + uri.AbsolutePath);
103                }
104
105                this.use_git_bin = false; // TODO
106            }
107
108            RemoteUrl = uri;
109        }
110
111
112        public override bool Fetch ()
113        {
114            if (!base.Fetch ())
115                return false;
116
117            if (FetchPriorHistory) {
118                this.git = new SparkleGit (SparkleConfig.DefaultConfig.TmpPath,
119                    "clone --progress --no-checkout \"" + RemoteUrl + "\" \"" + TargetFolder + "\"");
120
121            } else {
122                this.git = new SparkleGit (SparkleConfig.DefaultConfig.TmpPath,
123                    "clone --progress --no-checkout --depth=1 \"" + RemoteUrl + "\" \"" + TargetFolder + "\"");
124            }
125
126            this.git.StartInfo.RedirectStandardError = true;
127            this.git.Start ();
128
129            double percentage = 1.0;
130
131            DateTime last_change     = DateTime.Now;
132            TimeSpan change_interval = new TimeSpan (0, 0, 0, 1);
133
134            try {
135                while (!this.git.StandardError.EndOfStream) {
136                    string line = this.git.StandardError.ReadLine ();
137                    Match match = this.progress_regex.Match (line);
138
139                    double number = 0.0;
140                    double speed  = 0.0;
141                    if (match.Success) {
142                        try {
143                            number = double.Parse (match.Groups [1].Value, new CultureInfo ("en-US"));
144
145                        } catch (FormatException) {
146                            SparkleLogger.LogInfo ("Git", "Error parsing progress: \"" + match.Groups [1] + "\"");
147                        }
148
149                        // The pushing progress consists of two stages: the "Compressing
150                        // objects" stage which we count as 20% of the total progress, and
151                        // the "Writing objects" stage which we count as the last 80%
152                        if (line.Contains ("Compressing")) {
153                            // "Compressing objects" stage
154                            number = (number / 100 * 20);
155
156                        } else {
157                            // "Writing objects" stage
158                            number = (number / 100 * 80 + 20);
159                            Match speed_match = this.speed_regex.Match (line);
160
161                            if (speed_match.Success) {
162                                try {
163                                    speed = double.Parse (speed_match.Groups [1].Value, new CultureInfo ("en-US")) * 1024;
164
165                                } catch (FormatException) {
166                                    SparkleLogger.LogInfo ("Git", "Error parsing speed: \"" + speed_match.Groups [1] + "\"");
167                                }
168
169                                if (speed_match.Groups [2].Value.Equals ("M"))
170                                    speed = speed * 1024;
171                            }
172                        }
173
174                    } else {
175                        SparkleLogger.LogInfo ("Fetcher", line);
176                        line = line.Trim (new char [] {' ', '@'});
177
178                        if (line.StartsWith ("fatal:", StringComparison.InvariantCultureIgnoreCase) ||
179                            line.StartsWith ("error:", StringComparison.InvariantCultureIgnoreCase)) {
180
181                            base.errors.Add (line);
182
183                        } else if (line.StartsWith ("WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!")) {
184                            base.errors.Add ("warning: Remote host identification has changed!");
185
186                        } else if (line.StartsWith ("WARNING: POSSIBLE DNS SPOOFING DETECTED!")) {
187                            base.errors.Add ("warning: Possible DNS spoofing detected!");
188                        }
189                    }
190
191                    if (number >= percentage) {
192                        percentage = number;
193
194                        if (DateTime.Compare (last_change, DateTime.Now.Subtract (change_interval)) < 0) {
195                            base.OnProgressChanged (percentage, speed);
196                            last_change = DateTime.Now;
197                        }
198                    }
199                }
200
201            } catch (Exception) {
202                IsActive = false;
203                return false;
204            }
205
206            this.git.WaitForExit ();
207
208            if (this.git.ExitCode == 0) {
209                while (percentage < 100) {
210                    percentage += 25;
211
212                    if (percentage >= 100)
213                        break;
214
215                    Thread.Sleep (500);
216                    base.OnProgressChanged (percentage, 0);
217                }
218
219                base.OnProgressChanged (100, 0);
220
221                InstallConfiguration ();
222                InstallExcludeRules ();
223                InstallAttributeRules ();
224
225                return true;
226
227            } else {
228                return false;
229            }
230        }
231
232
233        public override bool IsFetchedRepoEmpty {
234            get {
235                SparkleGit git = new SparkleGit (TargetFolder, "rev-parse HEAD");
236                git.StartAndWaitForExit ();
237
238                return (git.ExitCode != 0);
239            }
240        }
241
242
243        public override void EnableFetchedRepoCrypto (string password)
244        {
245            // Set up the encryption filter
246            SparkleGit git_config_smudge = new SparkleGit (TargetFolder,
247                "config filter.encryption.smudge \"openssl enc -d -aes-256-cbc -base64 -S " + this.crypto_salt +
248                " -pass file:.git/info/encryption_password\"");
249
250            SparkleGit git_config_clean = new SparkleGit (TargetFolder,
251                "config filter.encryption.clean  \"openssl enc -e -aes-256-cbc -base64 -S " + this.crypto_salt +
252                " -pass file:.git/info/encryption_password\"");
253
254            git_config_smudge.StartAndWaitForExit ();
255            git_config_clean.StartAndWaitForExit ();
256
257            // Pass all files through the encryption filter
258            string git_attributes_file_path = new string [] { TargetFolder, ".git", "info", "attributes" }.Combine ();
259            File.WriteAllText (git_attributes_file_path, "\n* filter=encryption");
260
261            // Store the password
262            string password_file_path = new string [] { TargetFolder, ".git", "info", "encryption_password" }.Combine ();
263
264            if (this.crypto_password_is_hashed)
265                File.WriteAllText (password_file_path, password.SHA256 (this.crypto_salt));
266            else
267                File.WriteAllText (password_file_path, password);
268        }
269
270
271        public override bool IsFetchedRepoPasswordCorrect (string password)
272        {
273            string password_check_file_path = Path.Combine (TargetFolder, ".sparkleshare");
274
275            if (!File.Exists (password_check_file_path)) {
276                SparkleGit git = new SparkleGit (TargetFolder, "show HEAD:.sparkleshare");
277                string output = git.StartAndReadStandardOutput ();
278
279                if (git.ExitCode == 0)
280                    File.WriteAllText (password_check_file_path, output);
281                else
282                    return false;
283            }
284
285            Process process = new Process ();
286            process.EnableRaisingEvents              = true;
287            process.StartInfo.FileName               = "openssl";
288            process.StartInfo.WorkingDirectory       = TargetFolder;
289            process.StartInfo.UseShellExecute        = false;
290            process.StartInfo.RedirectStandardOutput = true;
291            process.StartInfo.CreateNoWindow         = true;
292
293            string [] possible_passwords = new string [] {
294                password.SHA256 (this.crypto_salt),
295                password
296            };
297
298            int i = 0;
299            foreach (string possible_password in possible_passwords) {
300                process.StartInfo.Arguments = "enc -d -aes-256-cbc -base64 -pass pass:\"" + possible_password + "\"" +
301                    " -in \"" + password_check_file_path + "\"";
302
303                SparkleLogger.LogInfo ("Cmd | " + System.IO.Path.GetFileName (process.StartInfo.WorkingDirectory),
304                    System.IO.Path.GetFileName (process.StartInfo.FileName) + " " + process.StartInfo.Arguments);
305
306                process.Start ();
307                process.WaitForExit ();
308
309                if (process.ExitCode == 0) {
310                    if (i > 0)
311                        this.crypto_password_is_hashed = false;
312
313                    File.Delete (password_check_file_path);
314                    return true;
315                }
316
317                i++;
318            }
319
320            return false;
321        }
322
323
324        public override void Stop ()
325        {
326            try {
327                if (this.git != null && !this.git.HasExited) {
328                    this.git.Kill ();
329                    this.git.Dispose ();
330                }
331
332            } catch (Exception e) {
333                SparkleLogger.LogInfo ("Fetcher", "Failed to dispose properly", e);
334            }
335
336            if (Directory.Exists (TargetFolder)) {
337                try {
338                    Directory.Delete (TargetFolder, true /* Recursive */ );
339                    SparkleLogger.LogInfo ("Fetcher", "Deleted '" + TargetFolder + "'");
340
341                } catch (Exception e) {
342                    SparkleLogger.LogInfo ("Fetcher", "Failed to delete '" + TargetFolder + "'", e);
343                }
344            }
345        }
346
347
348        public override void Complete ()
349        {
350            if (!IsFetchedRepoEmpty) {
351                SparkleGit git = new SparkleGit (TargetFolder, "checkout --quiet HEAD");
352                git.StartAndWaitForExit ();
353            }
354
355            base.Complete ();
356        }
357
358
359        private void InstallConfiguration ()
360        {
361            string [] settings = new string [] {
362                "core.autocrlf input",
363                "core.quotepath false", // Don't quote "unusual" characters in path names
364                "core.ignorecase false", // Be case sensitive explicitly to work on Mac
365                "core.filemode false", // Ignore permission changes
366                "core.precomposeunicode true", // Use the same Unicode form on all filesystems
367                "core.safecrlf false",
368                "core.excludesfile \"\"",
369                "core.packedGitLimit 128m", // Some memory limiting options
370                "core.packedGitWindowSize 128m",
371                "pack.deltaCacheSize 128m",
372                "pack.packSizeLimit 128m",
373                "pack.windowMemory 128m",
374                "push.default matching"
375            };
376
377            if (SparkleBackend.Platform == PlatformID.Win32NT)
378                settings [0] = "core.autocrlf true";
379
380            foreach (string setting in settings) {
381                SparkleGit git_config = new SparkleGit (TargetFolder, "config " + setting);
382                git_config.StartAndWaitForExit ();
383            }
384
385            if (this.use_git_bin)
386                InstallGitBinConfiguration ();
387        }
388
389
390        public void InstallGitBinConfiguration ()
391        {
392            string [] settings = new string [] {
393                "core.bigFileThreshold 1024g",
394                "filter.bin.clean \"git bin clean %f\"",
395                "filter.bin.smudge \"git bin smudge\""
396            };
397
398            foreach (string setting in settings) {
399                SparkleGit git_config = new SparkleGit (TargetFolder, "config " + setting);
400                git_config.StartAndWaitForExit ();
401            }
402        }
403
404
405        // Add a .gitignore file to the repo
406        private void InstallExcludeRules ()
407        {
408            string git_info_path = new string [] { TargetFolder, ".git", "info" }.Combine ();
409
410            if (!Directory.Exists (git_info_path))
411                Directory.CreateDirectory (git_info_path);
412
413            string exclude_rules           = string.Join (Environment.NewLine, ExcludeRules);
414            string exclude_rules_file_path = new string [] { git_info_path, "exclude" }.Combine ();
415
416            File.WriteAllText (exclude_rules_file_path, exclude_rules);
417        }
418
419
420        private void InstallAttributeRules ()
421        {
422            string attribute_rules_file_path = new string [] { TargetFolder, ".git", "info", "attributes" }.Combine ();
423            TextWriter writer                = new StreamWriter (attribute_rules_file_path);
424
425            if (this.use_git_bin) {
426                writer.WriteLine ("* filter=bin binary");
427
428            } else {
429                // Compile a list of files we don't want Git to compress.
430                // Not compressing already compressed files decreases memory usage and increases speed
431                string [] extensions = new string [] {
432                    "jpg", "jpeg", "png", "tiff", "gif", // Images
433                    "flac", "mp3", "ogg", "oga", // Audio
434                    "avi", "mov", "mpg", "mpeg", "mkv", "ogv", "ogx", "webm", // Video
435                    "zip", "gz", "bz", "bz2", "rpm", "deb", "tgz", "rar", "ace", "7z", "pak", "tc", "iso", ".dmg" // Archives
436                };
437
438                foreach (string extension in extensions) {
439                    writer.WriteLine ("*." + extension + " -delta");
440                    writer.WriteLine ("*." + extension.ToUpper () + " -delta");
441                }
442
443                writer.WriteLine ("*.txt text");
444                writer.WriteLine ("*.TXT text");
445            }
446
447            writer.Close ();
448        }
449    }
450}