PageRenderTime 48ms CodeModel.GetById 2ms app.highlight 37ms RepoModel.GetById 1ms app.codeStats 0ms

/Plugins/BuildServerIntegration/AppVeyorIntegration/AppVeyorAdapter.cs

http://github.com/spdr870/gitextensions
C# | 542 lines | 458 code | 74 blank | 10 comment | 55 complexity | 843ab92a8e1061adcff27f40f56e5926 MD5 | raw file
  1using System;
  2using System.Collections.Generic;
  3using System.ComponentModel.Composition;
  4using System.IO;
  5using System.Linq;
  6using System.Net.Http;
  7using System.Net.Http.Headers;
  8using System.Reactive.Concurrency;
  9using System.Reactive.Linq;
 10using System.Threading;
 11using System.Threading.Tasks;
 12using GitCommands.Utils;
 13using GitUI;
 14using GitUIPluginInterfaces;
 15using GitUIPluginInterfaces.BuildServerIntegration;
 16using JetBrains.Annotations;
 17using Newtonsoft.Json.Linq;
 18
 19namespace AppVeyorIntegration
 20{
 21    [MetadataAttribute]
 22    [AttributeUsage(AttributeTargets.Class)]
 23    public class AppVeyorIntegrationMetadata : BuildServerAdapterMetadataAttribute
 24    {
 25        public AppVeyorIntegrationMetadata(string buildServerType)
 26            : base(buildServerType)
 27        {
 28        }
 29
 30        public override string CanBeLoaded
 31        {
 32            get
 33            {
 34                if (EnvUtils.IsNet4FullOrHigher())
 35                {
 36                    return null;
 37                }
 38
 39                return ".Net 4 full framework required";
 40            }
 41        }
 42    }
 43
 44    [Export(typeof(IBuildServerAdapter))]
 45    [AppVeyorIntegrationMetadata(PluginName)]
 46    [PartCreationPolicy(CreationPolicy.NonShared)]
 47    internal class AppVeyorAdapter : IBuildServerAdapter
 48    {
 49        public const string PluginName = "AppVeyor";
 50        private const uint ProjectsToRetrieveCount = 25;
 51        private const string WebSiteUrl = "https://ci.appveyor.com";
 52        private const string ApiBaseUrl = WebSiteUrl + "/api/projects/";
 53
 54        private IBuildServerWatcher _buildServerWatcher;
 55
 56        private HttpClient _httpClientAppVeyor;
 57
 58        private List<AppVeyorBuildInfo> _allBuilds = new List<AppVeyorBuildInfo>();
 59        private HashSet<ObjectId> _fetchBuilds;
 60        private string _accountToken;
 61        private static readonly Dictionary<string, Project> Projects = new Dictionary<string, Project>();
 62        private Func<ObjectId, bool> _isCommitInRevisionGrid;
 63        private bool _shouldLoadTestResults;
 64
 65        public void Initialize(
 66            IBuildServerWatcher buildServerWatcher,
 67            ISettingsSource config,
 68            Func<ObjectId, bool> isCommitInRevisionGrid = null)
 69        {
 70            if (_buildServerWatcher != null)
 71            {
 72                throw new InvalidOperationException("Already initialized");
 73            }
 74
 75            _buildServerWatcher = buildServerWatcher;
 76            _isCommitInRevisionGrid = isCommitInRevisionGrid;
 77            var accountName = config.GetString("AppVeyorAccountName", null);
 78            _accountToken = config.GetString("AppVeyorAccountToken", null);
 79            var projectNamesSetting = config.GetString("AppVeyorProjectName", null);
 80            if (accountName.IsNullOrWhiteSpace() && projectNamesSetting.IsNullOrWhiteSpace())
 81            {
 82                return;
 83            }
 84
 85            _shouldLoadTestResults = config.GetBool("AppVeyorLoadTestsResults", false);
 86
 87            _fetchBuilds = new HashSet<ObjectId>();
 88
 89            _httpClientAppVeyor = GetHttpClient(WebSiteUrl, _accountToken);
 90
 91            var useAllProjects = string.IsNullOrWhiteSpace(projectNamesSetting);
 92            string[] projectNames = null;
 93            if (!useAllProjects)
 94            {
 95                projectNames = _buildServerWatcher.ReplaceVariables(projectNamesSetting)
 96                    .Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
 97            }
 98
 99            if (Projects.Count == 0 ||
100                (!useAllProjects && Projects.Keys.Intersect(projectNames).Count() != projectNames.Length))
101            {
102                Projects.Clear();
103                if (_accountToken.IsNullOrWhiteSpace())
104                {
105                    FillProjectsFromSettings(accountName, projectNames);
106                }
107                else
108                {
109                    if (accountName.IsNullOrWhiteSpace())
110                    {
111                        return;
112                    }
113
114                    ThreadHelper.JoinableTaskFactory.Run(
115                        async () =>
116                        {
117                            var result = await GetResponseAsync(_httpClientAppVeyor, ApiBaseUrl, CancellationToken.None).ConfigureAwait(false);
118
119                            if (result.IsNullOrWhiteSpace())
120                            {
121                                return;
122                            }
123
124                            var projects = JArray.Parse(result);
125                            foreach (var project in projects)
126                            {
127                                var projectId = project["slug"].ToString();
128                                projectId = accountName.Combine("/", projectId);
129                                var projectName = project["name"].ToString();
130                                var projectObj = new Project
131                                {
132                                    Name = projectName,
133                                    Id = projectId,
134                                    QueryUrl = BuildQueryUrl(projectId)
135                                };
136
137                                if (useAllProjects || projectNames.Contains(projectObj.Name))
138                                {
139                                    Projects.Add(projectObj.Name, projectObj);
140                                }
141                            }
142                        });
143                }
144            }
145
146            var builds = Projects.Where(p => useAllProjects || projectNames.Contains(p.Value.Name)).Select(p => p.Value);
147            _allBuilds =
148                FilterBuilds(builds.SelectMany(project => QueryBuildsResults(project)));
149        }
150
151        private static void FillProjectsFromSettings(string accountName, [InstantHandle] IEnumerable<string> projectNames)
152        {
153            foreach (var projectName in projectNames)
154            {
155                var projectId = accountName.Combine("/", projectName);
156                Projects.Add(projectName, new Project
157                {
158                    Name = projectName,
159                    Id = projectId,
160                    QueryUrl = BuildQueryUrl(projectId)
161                });
162            }
163        }
164
165        private static HttpClient GetHttpClient(string baseUrl, string accountToken)
166        {
167            var httpClient = new HttpClient(new HttpClientHandler { UseDefaultCredentials = true })
168            {
169                Timeout = TimeSpan.FromMinutes(2),
170                BaseAddress = new Uri(baseUrl, UriKind.Absolute),
171            };
172            if (accountToken.IsNotNullOrWhitespace())
173            {
174                httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accountToken);
175            }
176
177            return httpClient;
178        }
179
180        private static string BuildQueryUrl(string projectId)
181        {
182            return ApiBaseUrl + projectId + "/history?recordsNumber=" + ProjectsToRetrieveCount;
183        }
184
185        private class Project
186        {
187            public string Name;
188            public string Id;
189            public string QueryUrl;
190        }
191
192        private IEnumerable<AppVeyorBuildInfo> QueryBuildsResults(Project project)
193        {
194            try
195            {
196                return ThreadHelper.JoinableTaskFactory.Run(
197                    async () =>
198                    {
199                        var result = await GetResponseAsync(_httpClientAppVeyor, project.QueryUrl, CancellationToken.None).ConfigureAwait(false);
200
201                        if (string.IsNullOrWhiteSpace(result))
202                        {
203                            return Enumerable.Empty<AppVeyorBuildInfo>();
204                        }
205
206                        var builds = JObject.Parse(result)["builds"].Children();
207                        var baseApiUrl = ApiBaseUrl + project.Id;
208                        var baseWebUrl = WebSiteUrl + "/project/" + project.Id + "/build/";
209
210                        var buildDetails = new List<AppVeyorBuildInfo>();
211                        foreach (var b in builds)
212                        {
213                            try
214                            {
215                                if (!ObjectId.TryParse((b["pullRequestHeadCommitId"] ?? b["commitId"]).ToObject<string>(), out var objectId) || !_isCommitInRevisionGrid(objectId))
216                                {
217                                    continue;
218                                }
219
220                                var pullRequestId = b["pullRequestId"];
221                                var version = b["version"].ToObject<string>();
222                                var status = ParseBuildStatus(b["status"].ToObject<string>());
223                                long? duration = null;
224                                if (status == BuildInfo.BuildStatus.Success || status == BuildInfo.BuildStatus.Failure)
225                                {
226                                    duration = GetBuildDuration(b);
227                                }
228
229                                buildDetails.Add(new AppVeyorBuildInfo
230                                {
231                                    Id = version,
232                                    BuildId = b["buildId"].ToObject<string>(),
233                                    Branch = b["branch"].ToObject<string>(),
234                                    CommitId = objectId,
235                                    CommitHashList = new[] { objectId },
236                                    Status = status,
237                                    StartDate = b["started"]?.ToObject<DateTime>() ?? DateTime.MinValue,
238                                    BaseWebUrl = baseWebUrl,
239                                    Url = WebSiteUrl + "/project/" + project.Id + "/build/" + version,
240                                    BaseApiUrl = baseApiUrl,
241                                    AppVeyorBuildReportUrl = baseApiUrl + "/build/" + version,
242                                    PullRequestText = pullRequestId != null ? " PR#" + pullRequestId.Value<string>() : string.Empty,
243                                    Duration = duration,
244                                    TestsResultText = string.Empty
245                                });
246                            }
247                            catch (Exception)
248                            {
249                                // Failure on reading data on a build detail should not prevent to display the others build results
250                            }
251                        }
252
253                        return buildDetails;
254                    });
255            }
256            catch
257            {
258                return Enumerable.Empty<AppVeyorBuildInfo>();
259            }
260        }
261
262        /// <summary>
263        /// Gets a unique key which identifies this build server.
264        /// </summary>
265        public string UniqueKey => _httpClientAppVeyor.BaseAddress.Host;
266
267        public IObservable<BuildInfo> GetFinishedBuildsSince(IScheduler scheduler, DateTime? sinceDate = null)
268        {
269            // AppVeyor api is different than TeamCity one and all build results are fetch in one call without
270            // filter parameters possible (so this call is useless!)
271            return Observable.Empty<BuildInfo>();
272        }
273
274        public IObservable<BuildInfo> GetRunningBuilds(IScheduler scheduler)
275        {
276            return GetBuilds(scheduler);
277        }
278
279        private IObservable<BuildInfo> GetBuilds(IScheduler scheduler)
280        {
281            return Observable.Create<BuildInfo>((observer, cancellationToken) =>
282                Task.Run(
283                    () => scheduler.Schedule(() => ObserveBuilds(observer, cancellationToken))));
284        }
285
286        private void ObserveBuilds(IObserver<BuildInfo> observer, CancellationToken cancellationToken)
287        {
288            try
289            {
290                if (_allBuilds == null)
291                {
292                    return;
293                }
294
295                // Display all builds found
296                foreach (var build in _allBuilds)
297                {
298                    UpdateDisplay(observer, build);
299                }
300
301                // Update finished build with tests results
302                if (_shouldLoadTestResults)
303                {
304                    foreach (var build in _allBuilds.Where(b => b.Status == BuildInfo.BuildStatus.Success
305                                                                || b.Status == BuildInfo.BuildStatus.Failure))
306                    {
307                        UpdateDescription(build, cancellationToken);
308                        UpdateDisplay(observer, build);
309                    }
310                }
311
312                // Manage in progress builds...
313                var inProgressBuilds = _allBuilds.Where(b => b.Status == BuildInfo.BuildStatus.InProgress).ToList();
314                _allBuilds = null;
315                do
316                {
317                    Thread.Sleep(5000);
318                    foreach (var build in inProgressBuilds)
319                    {
320                        UpdateDescription(build, cancellationToken);
321                        UpdateDisplay(observer, build);
322                    }
323
324                    inProgressBuilds = inProgressBuilds.Where(b => b.Status == BuildInfo.BuildStatus.InProgress).ToList();
325                }
326                while (inProgressBuilds.Any());
327
328                observer.OnCompleted();
329            }
330            catch (OperationCanceledException)
331            {
332                // Do nothing, the observer is already stopped
333            }
334            catch (Exception ex)
335            {
336                observer.OnError(ex);
337            }
338        }
339
340        private static void UpdateDisplay(IObserver<BuildInfo> observer, AppVeyorBuildInfo build)
341        {
342            build.UpdateDescription();
343            observer.OnNext(build);
344        }
345
346        private List<AppVeyorBuildInfo> FilterBuilds(IEnumerable<AppVeyorBuildInfo> allBuilds)
347        {
348            var filteredBuilds = new List<AppVeyorBuildInfo>();
349            foreach (var build in allBuilds.OrderByDescending(b => b.StartDate))
350            {
351                if (!_fetchBuilds.Contains(build.CommitId))
352                {
353                    filteredBuilds.Add(build);
354                    _fetchBuilds.Add(build.CommitId);
355                }
356            }
357
358            return filteredBuilds;
359        }
360
361        private void UpdateDescription(AppVeyorBuildInfo buildDetails, CancellationToken cancellationToken)
362        {
363            var buildDetailsParsed = ThreadHelper.JoinableTaskFactory.Run(() => FetchBuildDetailsManagingVersionUpdateAsync(buildDetails, cancellationToken));
364            if (buildDetailsParsed == null)
365            {
366                return;
367            }
368
369            var buildData = buildDetailsParsed["build"];
370            var buildDescription = buildData["jobs"].Last();
371
372            var status = buildDescription["status"].ToObject<string>();
373            buildDetails.Status = ParseBuildStatus(status);
374
375            buildDetails.ChangeProgressCounter();
376            if (!buildDetails.IsRunning)
377            {
378                buildDetails.Duration = GetBuildDuration(buildData);
379            }
380
381            int testCount = buildDescription["testsCount"].ToObject<int>();
382            if (testCount != 0)
383            {
384                int failedTestCount = buildDescription["failedTestsCount"].ToObject<int>();
385                int skippedTestCount = testCount - buildDescription["passedTestsCount"].ToObject<int>();
386                var testResults = " : " + testCount + " tests";
387                if (failedTestCount != 0 || skippedTestCount != 0)
388                {
389                    testResults += string.Format(" ( {0} failed, {1} skipped )", failedTestCount, skippedTestCount);
390                }
391
392                buildDetails.TestsResultText = " " + testResults;
393            }
394        }
395
396        private static long GetBuildDuration(JToken buildData)
397        {
398            var startTime = buildData["started"].ToObject<DateTime>();
399            var updateTime = buildData["updated"].ToObject<DateTime>();
400            return (long)(updateTime - startTime).TotalMilliseconds;
401        }
402
403        private async Task<JObject> FetchBuildDetailsManagingVersionUpdateAsync(AppVeyorBuildInfo buildDetails, CancellationToken cancellationToken)
404        {
405            try
406            {
407                return JObject.Parse(await GetResponseAsync(_httpClientAppVeyor, buildDetails.AppVeyorBuildReportUrl, cancellationToken).ConfigureAwait(false));
408            }
409            catch
410            {
411                var buildHistoryUrl = buildDetails.BaseApiUrl + "/history?recordsNumber=1&startBuildId=" + (int.Parse(buildDetails.BuildId) + 1);
412                var builds = JObject.Parse(await GetResponseAsync(_httpClientAppVeyor, buildHistoryUrl, cancellationToken).ConfigureAwait(false));
413
414                var version = builds["builds"][0]["version"].ToObject<string>();
415                buildDetails.Id = version;
416                buildDetails.AppVeyorBuildReportUrl = buildDetails.BaseApiUrl + "/build/" + version;
417                buildDetails.Url = buildDetails.BaseWebUrl + version;
418
419                return JObject.Parse(await GetResponseAsync(_httpClientAppVeyor, buildDetails.AppVeyorBuildReportUrl, cancellationToken).ConfigureAwait(false));
420            }
421        }
422
423        private static BuildInfo.BuildStatus ParseBuildStatus(string statusValue)
424        {
425            switch (statusValue)
426            {
427                case "success":
428                    return BuildInfo.BuildStatus.Success;
429                case "failed":
430                    return BuildInfo.BuildStatus.Failure;
431                case "cancelled":
432                    return BuildInfo.BuildStatus.Stopped;
433                case "queued":
434                case "running":
435                    return BuildInfo.BuildStatus.InProgress;
436                default:
437                    return BuildInfo.BuildStatus.Unknown;
438            }
439        }
440
441        private Task<Stream> GetStreamAsync(HttpClient httpClient, string restServicePath, CancellationToken cancellationToken)
442        {
443            cancellationToken.ThrowIfCancellationRequested();
444
445            return httpClient.GetAsync(restServicePath, HttpCompletionOption.ResponseHeadersRead, cancellationToken)
446                             .ContinueWith(
447                                 task => GetStreamFromHttpResponseAsync(httpClient, task, restServicePath, cancellationToken),
448                                 cancellationToken,
449                                 restServicePath.Contains("github") ? TaskContinuationOptions.None : TaskContinuationOptions.AttachedToParent,
450                                 TaskScheduler.Current)
451                             .Unwrap();
452        }
453
454        private Task<Stream> GetStreamFromHttpResponseAsync(HttpClient httpClient, Task<HttpResponseMessage> task, string restServicePath, CancellationToken cancellationToken)
455        {
456            var retry = task.IsCanceled && !cancellationToken.IsCancellationRequested;
457
458            if (retry)
459            {
460                return GetStreamAsync(httpClient, restServicePath, cancellationToken);
461            }
462
463            if (task.Status == TaskStatus.RanToCompletion && task.CompletedResult().IsSuccessStatusCode)
464            {
465                return task.CompletedResult().Content.ReadAsStreamAsync();
466            }
467
468            return null;
469        }
470
471        private Task<string> GetResponseAsync(HttpClient httpClient, string relativePath, CancellationToken cancellationToken)
472        {
473            var getStreamTask = GetStreamAsync(httpClient, relativePath, cancellationToken);
474
475            var taskContinuationOptions = relativePath.Contains("github") ? TaskContinuationOptions.None : TaskContinuationOptions.AttachedToParent;
476            return getStreamTask.ContinueWith(
477                task =>
478                {
479                    if (task.Status != TaskStatus.RanToCompletion)
480                    {
481                        return string.Empty;
482                    }
483
484                    using (var responseStream = task.Result)
485                    {
486                        return new StreamReader(responseStream).ReadToEnd();
487                    }
488                },
489                cancellationToken,
490                taskContinuationOptions,
491                TaskScheduler.Current);
492        }
493
494        public void Dispose()
495        {
496            GC.SuppressFinalize(this);
497
498            _httpClientAppVeyor?.Dispose();
499        }
500    }
501
502    internal sealed class AppVeyorBuildInfo : BuildInfo
503    {
504        private static readonly IBuildDurationFormatter _buildDurationFormatter = new BuildDurationFormatter();
505
506        private int _buildProgressCount;
507
508        public string BuildId { get; set; }
509        public ObjectId CommitId { get; set; }
510        public string AppVeyorBuildReportUrl { get; set; }
511        public string Branch { get; set; }
512        public string BaseApiUrl { get; set; }
513        public string BaseWebUrl { get; set; }
514        public string PullRequestText { get; set; }
515        public string TestsResultText { get; set; }
516
517        public bool IsRunning => Status == BuildStatus.InProgress;
518
519        public void ChangeProgressCounter()
520        {
521            _buildProgressCount = (_buildProgressCount % 3) + 1;
522        }
523
524        public void UpdateDescription()
525        {
526            Description = Id + " " + DisplayStatus + " " + _buildDurationFormatter.Format(Duration) + TestsResultText + PullRequestText;
527        }
528
529        private string DisplayStatus
530        {
531            get
532            {
533                if (Status != BuildStatus.InProgress)
534                {
535                    return Status.ToString("G");
536                }
537
538                return "In progress" + new string('.', _buildProgressCount) + new string(' ', 3 - _buildProgressCount);
539            }
540        }
541    }
542}