/Accelerator/Synchronization/BlobSync.cs

using System;

using System.Collections.Generic;

using System.ComponentModel;

using System.IO;

using System.Linq;

using System.Threading;

using Microsoft.Synchronization;

using Microsoft.WindowsAzure.Accelerator.Diagnostics;

using Microsoft.WindowsAzure.StorageClient;

using Microsoft.WindowsAzure.Diagnostics;



namespace Microsoft.WindowsAzure.Accelerator.Synchronization

{

    /// <summary>

    /// Provides data for updating a file event.

    /// </summary>

    public class UpdatingFileEventArgs : CancelEventArgs

    {

        public CloudBlob Blob;

        public String LocalPath;



        /// <summary>

        /// Initializes a new instance of the <see cref="UpdatingFileEventArgs"/> class.

        /// </summary>

        /// <param name="blob">The BLOB.</param>

        /// <param name="localPath">The local path.</param>

        public UpdatingFileEventArgs(CloudBlob blob, String localPath)

        {

            Blob = blob;

            LocalPath = localPath;

        }

    }







    /// <summary>

    /// Performs synchronization between blob and local file system storage.

    /// </summary>

    public class BlobSync : ISyncProvider

    {

        public event UpdatingFileHandler UpdatingFile;



        #region | FIELDS



        private readonly Dictionary<String, String> _localSyncList = new Dictionary<String, String>();

        private Thread _blobSyncThread;



        #endregion



        public Boolean IsStarted { get { return _blobSyncThread != null && _blobSyncThread.ThreadState != System.Threading.ThreadState.Running; } }

        public String LocalPath { get; set; }

        public Boolean IgnoreAdditionalFiles { get; set; }

        public SyncDirectionOrder SyncDirection { get; set; }

        public Dictionary<String, String> LocalSyncList { get { return _localSyncList; } }    

        public CloudStorageAccount Account { get; set; }

        public String ContainerName { get; set; }



        #region | CONSTRUCTORS



        /// <summary>

        /// Initializes a new instance of the <see cref="BlobSync"/> class.

        /// </summary>

        public BlobSync(){ }



        /// <summary>

        /// Initializes a new instance of the <see cref="BlobSync"/> class.

        /// </summary>

        /// <param name="containerName">Name of the container.</param>

        /// <param name="blobDirectory">The root cloud blob directory.</param>

        /// <param name="localPath">The local path.</param>

        /// <param name="direction">The direction.</param>

        /// <param name="ignoreExisting">if set to <c>true</c> [ignore existing].</param>

        public BlobSync(String containerName, String blobDirectory, String localPath, SyncDirectionOrder direction, Boolean ignoreExisting)

        {

            ContainerName = containerName;

            LocalPath = localPath;

            SyncDirection = SyncDirection;

            IgnoreAdditionalFiles = ignoreExisting;

            SyncDirection = direction;

        }



        #endregion

        #region | EVENTS



        /// <summary>

        /// Validates this instance.

        /// </summary>

        /// <returns></returns>

        public Boolean Validate()

        {

            LogLevel.Information.TraceContent("BlobSync", "Sync Validation...", this.ToTraceString());

            return true;

        }



        /// <summary>

        /// Start the thread to sync blobs at intervals until Stop() is called.

        /// </summary>

        public void Start(TimeSpan interval)

        {

            _blobSyncThread = new Thread(() =>

                                             {

                                                 while (true) {

                                                     try {

                                                         StartSync();

                                                     }

                                                     catch (Exception ex) {

                                                         LogLevel.Error.TraceException("BlobSync", ex, "An exception occurred when performing blob to local storage synchroniztion");

                                                     }

                                                     Thread.Sleep(interval);

                                                 }

                                             });

            _blobSyncThread.Start();

        }



        /// <summary>

        /// Stops the blob sync, terminating any in-progress transfers.

        /// </summary>

        public void Stop()

        {

            lock (_blobSyncThread)

            {

                if (IsStarted && _blobSyncThread != null && _blobSyncThread.ThreadState == System.Threading.ThreadState.Running)

                    _blobSyncThread.Abort();

            }

        }



        /// <summary>

        /// Performs the synchronization of file system and blob storage.

        /// </summary>

        public void StartSync()

        {

            switch (SyncDirection)

            {

                case (SyncDirectionOrder.DownloadAndUpload): 

                    Sync(SyncDirectionOrder.Download);

                    Sync(SyncDirectionOrder.Upload);

                    break;

                case (SyncDirectionOrder.UploadAndDownload):

                    Sync(SyncDirectionOrder.Upload);

                    Sync(SyncDirectionOrder.Download);

                    break;

                default: 

                    Sync(SyncDirection);

                    break;

            }

        }



        /// <summary>

        /// Performs the synchronization of file system and blob storage.

        /// </summary>

        private void Sync(SyncDirectionOrder syncDirection)

        {

            if (!Validate())

                return;



            var blobDirectory = Account.CreateCloudBlobClient().GetBlobDirectoryReference(ContainerName);

            var directory = blobDirectory.Uri.ToString();

            var container = blobDirectory.Container;



            //i|

            //i| Get all of the blob in container.

            //i|

            Dictionary<String, CloudBlob> cloudBlobs = blobDirectory.ListBlobs(new BlobRequestOptions

                                                                                      {

                                                                                          UseFlatBlobListing = true,

                                                                                          BlobListingDetails = BlobListingDetails.Metadata

                                                                                      }).OfType<CloudBlob>().ToDictionary(b => b.Uri.ToString(), b => b);



            if (!IgnoreAdditionalFiles)

            {

                var cloudBlobNames = new HashSet<String>(cloudBlobs.Keys);

                var localBlobNames = new HashSet<String>(LocalSyncList.Keys);



                if (syncDirection == SyncDirectionOrder.Download)

                {

                    localBlobNames.ExceptWith(cloudBlobNames);

                    foreach (var uri in localBlobNames)

                    {

                        //i|

                        //i| Delete all local files without corresponding blob files.

                        //i|

                        String localPath = GetLocalPath(uri, directory);

                        LogLevel.Information.Trace("BlobSync", "FileDelete : Deleting Local File : {{ [Filename: '{0}'] }}.", localPath);

                        File.Delete(localPath);

                        LocalSyncList.Remove(uri);

                    }

                }

                else if (syncDirection == SyncDirectionOrder.Upload)

                {

                    cloudBlobNames.ExceptWith(localBlobNames);

                    foreach (var uri in cloudBlobNames)

                    {

                        //i|

                        //i| Delete all azure storage blogs without corresponding local files.

                        //i|

                        CloudBlob blob = cloudBlobs[uri];

                        LogLevel.Information.Trace("BlobSync", "FileDelete : Deleting Storage Blob : {{ [BlobUri: '{0}'] }}.", uri);

                        blob.DeleteIfExists();

                        cloudBlobs.Remove(uri);

                    }

                }

            }



            if (syncDirection == SyncDirectionOrder.Download)

            {

                foreach (var kvp in cloudBlobs)

                {

                    var blobUri = kvp.Key;

                    var blob = kvp.Value;

                    if (!LocalSyncList.ContainsKey(blobUri) || LocalSyncList[blobUri] != blob.Attributes.Properties.ETag)

                    {

                        var localPath = GetLocalPath(blobUri, directory);

                        var localDirectory = System.IO.Path.GetDirectoryName(localPath);

                        var args = new UpdatingFileEventArgs(blob, localPath);

                        LogLevel.Information.Trace("BlobSync", "Local File Update : {{ [LocalSyncRoot: '{0}'], [SourceBlobUri: '{1}'] }}.", localPath, blobUri);

                        if (UpdatingFile != null)

                        {

                            UpdatingFile(this, args);

                        }

                        if (!args.Cancel)

                        {

                            Directory.CreateDirectory(localDirectory);

                            using (var stream = File.Create(localPath))

                                blob.DownloadToStream(stream);

                        }

                        LocalSyncList[blobUri] = blob.Properties.ETag;

                    }

                }

            }

            else if (syncDirection == SyncDirectionOrder.Upload)

            {

                var syncList = new Dictionary<String, FileInfo>().SyncListFromDirectory(new DirectoryInfo(LocalPath), directory);

                var toSyncList = (from sl in syncList

                                  where !cloudBlobs.ContainsKey(sl.Key) || cloudBlobs[sl.Key].Properties.Length != sl.Value.Length

                                  select sl

                                 );

                foreach (var kvp in toSyncList)

                {

                    LogLevel.Information.Trace("BlobSync", "Updating Blob : {{ [SourceFile]: '{0}', [BlobUri]: '{1}' }}.", kvp.Value.FullName, kvp.Key);

                    CloudBlob blob = container.GetBlobReference(kvp.Key);

                    blob.UploadFile(kvp.Value.FullName);

                }

            }

        }



        #endregion

        #region | UTILITY METHODS



        /// <summary>

        /// Gets the local path.

        /// </summary>

        /// <param name="uri">The URI.</param>

        /// <param name="syncDirectory">The sync directory.</param>

        /// <returns></returns>

        private String GetLocalPath(String uri, String syncDirectory)

        {

            return System.IO.Path.Combine(LocalPath, uri.Substring(syncDirectory.Length).SetPathChar('\\'));

        }

        #endregion

    }



    /// <summary>

    /// Delegate for handling the file update event.

    /// </summary>

    public delegate void UpdatingFileHandler(Object sender, UpdatingFileEventArgs args);



    /// <summary>

    /// Sync extension methods.

    /// </summary>

    public static class SyncExtensions

    {

        public static Dictionary<String, FileInfo> SyncListFromDirectory(this Dictionary<String, FileInfo> syncList, DirectoryInfo directory, String syncUri)

        {

            foreach (var subdir in directory.GetDirectories())

            {

                String suburi = String.Format("{0}/{1}", syncUri.TrimEnd(' ', '\\', '/'), subdir.Name);

                syncList.SyncListFromDirectory(subdir, suburi);

            }

            foreach (var file in directory.GetFiles())

            {

                String fileuri = String.Format("{0}/{1}", syncUri.TrimEnd(' ', '\\', '/'), file.Name);

                syncList[fileuri] = file;

            }

            return syncList;

        }

    }

}