PageRenderTime 114ms CodeModel.GetById 92ms app.highlight 17ms RepoModel.GetById 1ms app.codeStats 0ms

/Plugins/BuildServerIntegration/AppVeyorIntegration/AppVeyorAdapter.cs

https://github.com/gitextensions/gitextensions
C# | 542 lines | 461 code | 71 blank | 10 comment | 60 complexity | 86e9e97226ce59e5d736cc47561a190a MD5 | raw file
  1using System;
  2using System.Collections.Generic;
  3using System.ComponentModel.Composition;
  4using System.IO;
  5using System.Linq;
  6using System.Net.Http;
  7using System.Net.Http.Headers;
  8using System.Reactive.Concurrency;
  9using System.Reactive.Linq;
 10using System.Threading;
 11using System.Threading.Tasks;
 12using GitCommands.Utils;
 13using GitUI;
 14using GitUIPluginInterfaces;
 15using GitUIPluginInterfaces.BuildServerIntegration;
 16using JetBrains.Annotations;
 17using Newtonsoft.Json.Linq;
 18
 19namespace AppVeyorIntegration
 20{
 21    [MetadataAttribute]
 22    [AttributeUsage(AttributeTargets.Class)]
 23    public class AppVeyorIntegrationMetadata : BuildServerAdapterMetadataAttribute
 24    {
 25        public AppVeyorIntegrationMetadata(string buildServerType)
 26            : base(buildServerType)
 27        {
 28        }
 29
 30        public override string CanBeLoaded
 31        {
 32            get
 33            {
 34                if (EnvUtils.IsNet4FullOrHigher())
 35                {
 36                    return null;
 37                }
 38
 39                return ".Net 4 full framework required";
 40            }
 41        }
 42    }
 43
 44    [Export(typeof(IBuildServerAdapter))]
 45    [AppVeyorIntegrationMetadata(PluginName)]
 46    [PartCreationPolicy(CreationPolicy.NonShared)]
 47    internal class AppVeyorAdapter : IBuildServerAdapter
 48    {
 49        public const string PluginName = "AppVeyor";
 50        private const uint ProjectsToRetrieveCount = 25;
 51        private const string WebSiteUrl = "https://ci.appveyor.com";
 52        private const string ApiBaseUrl = WebSiteUrl + "/api/projects/";
 53
 54        private IBuildServerWatcher _buildServerWatcher;
 55
 56        private HttpClient _httpClientAppVeyor;
 57
 58        private List<AppVeyorBuildInfo> _allBuilds = new List<AppVeyorBuildInfo>();
 59        private HashSet<ObjectId> _fetchBuilds;
 60        private string _accountToken;
 61        private static readonly Dictionary<string, Project> Projects = new Dictionary<string, Project>();
 62        private Func<ObjectId, bool> _isCommitInRevisionGrid;
 63        private bool _shouldLoadTestResults;
 64
 65        public void Initialize(
 66            IBuildServerWatcher buildServerWatcher,
 67            ISettingsSource config,
 68            Func<ObjectId, bool> isCommitInRevisionGrid = null)
 69        {
 70            if (_buildServerWatcher != null)
 71            {
 72                throw new InvalidOperationException("Already initialized");
 73            }
 74
 75            _buildServerWatcher = buildServerWatcher;
 76            _isCommitInRevisionGrid = isCommitInRevisionGrid;
 77            var accountName = config.GetString("AppVeyorAccountName", null);
 78            _accountToken = config.GetString("AppVeyorAccountToken", null);
 79            var projectNamesSetting = config.GetString("AppVeyorProjectName", null);
 80            if (string.IsNullOrWhiteSpace(accountName) && string.IsNullOrWhiteSpace(projectNamesSetting))
 81            {
 82                return;
 83            }
 84
 85            _shouldLoadTestResults = config.GetBool("AppVeyorLoadTestsResults", false);
 86
 87            _fetchBuilds = new HashSet<ObjectId>();
 88
 89            _httpClientAppVeyor = GetHttpClient(WebSiteUrl, _accountToken);
 90
 91            var useAllProjects = string.IsNullOrWhiteSpace(projectNamesSetting);
 92            string[] projectNames = null;
 93            if (!useAllProjects)
 94            {
 95                projectNames = _buildServerWatcher.ReplaceVariables(projectNamesSetting)
 96                    .Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
 97            }
 98
 99            if (Projects.Count == 0 ||
100                (!useAllProjects && Projects.Keys.Intersect(projectNames).Count() != projectNames.Length))
101            {
102                Projects.Clear();
103                if (string.IsNullOrWhiteSpace(_accountToken))
104                {
105                    FillProjectsFromSettings(accountName, projectNames);
106                }
107                else
108                {
109                    if (string.IsNullOrWhiteSpace(accountName))
110                    {
111                        return;
112                    }
113
114                    ThreadHelper.JoinableTaskFactory.Run(
115                        async () =>
116                        {
117                            var result = await GetResponseAsync(_httpClientAppVeyor, ApiBaseUrl, CancellationToken.None).ConfigureAwait(false);
118
119                            if (string.IsNullOrWhiteSpace(result))
120                            {
121                                return;
122                            }
123
124                            var projects = JArray.Parse(result);
125                            foreach (var project in projects)
126                            {
127                                var projectId = project["slug"].ToString();
128                                projectId = accountName.Combine("/", projectId);
129                                var projectName = project["name"].ToString();
130                                var projectObj = new Project
131                                {
132                                    Name = projectName,
133                                    Id = projectId,
134                                    QueryUrl = BuildQueryUrl(projectId)
135                                };
136
137                                if (useAllProjects || projectNames.Contains(projectObj.Name))
138                                {
139                                    Projects.Add(projectObj.Name, projectObj);
140                                }
141                            }
142                        });
143                }
144            }
145
146            var builds = Projects.Where(p => useAllProjects || projectNames.Contains(p.Value.Name)).Select(p => p.Value);
147            _allBuilds =
148                FilterBuilds(builds.SelectMany(project => QueryBuildsResults(project)));
149        }
150
151        private static void FillProjectsFromSettings(string accountName, [InstantHandle] IEnumerable<string> projectNames)
152        {
153            foreach (var projectName in projectNames)
154            {
155                var projectId = accountName.Combine("/", projectName);
156                if (!Projects.ContainsKey(projectName))
157                {
158                    Projects.Add(projectName, new Project
159                    {
160                        Name = projectName,
161                        Id = projectId,
162                        QueryUrl = BuildQueryUrl(projectId)
163                    });
164                }
165            }
166        }
167
168        private static HttpClient GetHttpClient(string baseUrl, string accountToken)
169        {
170            var httpClient = new HttpClient(new HttpClientHandler { UseDefaultCredentials = true })
171            {
172                Timeout = TimeSpan.FromMinutes(2),
173                BaseAddress = new Uri(baseUrl, UriKind.Absolute),
174            };
175            if (!string.IsNullOrWhiteSpace(accountToken))
176            {
177                httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accountToken);
178            }
179
180            return httpClient;
181        }
182
183        private static string BuildQueryUrl(string projectId)
184        {
185            return ApiBaseUrl + projectId + "/history?recordsNumber=" + ProjectsToRetrieveCount;
186        }
187
188        internal class Project
189        {
190            public string Name;
191            public string Id;
192            public string QueryUrl;
193        }
194
195        private string BuildPullRequetUrl(string repositoryType, string repositoryName, string pullRequestId)
196        {
197            switch (repositoryType.ToLowerInvariant())
198            {
199                case "bitbucket":
200                    return $"https://bitbucket.org/{repositoryName}/pull-requests/{pullRequestId}";
201                case "github":
202                    return $"https://github.com/{repositoryName}/pull/{pullRequestId}";
203                case "gitlab":
204                    return $"https://gitlab.com/{repositoryName}/merge_requests/{pullRequestId}";
205                case "vso":
206                case "git":
207                default:
208                    return null;
209            }
210        }
211
212        private IEnumerable<AppVeyorBuildInfo> QueryBuildsResults(Project project)
213        {
214            try
215            {
216                return ThreadHelper.JoinableTaskFactory.Run(
217                    async () =>
218                    {
219                        var result = await GetResponseAsync(_httpClientAppVeyor, project.QueryUrl, CancellationToken.None).ConfigureAwait(false);
220
221                        return ExtractBuildInfo(project, result);
222                    });
223            }
224            catch
225            {
226                return Enumerable.Empty<AppVeyorBuildInfo>();
227            }
228        }
229
230        internal IEnumerable<AppVeyorBuildInfo> ExtractBuildInfo(Project project, string result)
231        {
232            if (string.IsNullOrWhiteSpace(result))
233            {
234                return Enumerable.Empty<AppVeyorBuildInfo>();
235            }
236
237            var content = JObject.Parse(result);
238
239            var projectData = content["project"];
240            var repositoryName = projectData["repositoryName"];
241            var repositoryType = projectData["repositoryType"];
242
243            var builds = content["builds"].Children();
244            var baseApiUrl = ApiBaseUrl + project.Id;
245            var baseWebUrl = WebSiteUrl + "/project/" + project.Id + "/build/";
246
247            var buildDetails = new List<AppVeyorBuildInfo>();
248            foreach (var b in builds)
249            {
250                try
251                {
252                    if (!ObjectId.TryParse((b["pullRequestHeadCommitId"] ?? b["commitId"]).ToObject<string>(),
253                            out var objectId) || !_isCommitInRevisionGrid(objectId))
254                    {
255                        continue;
256                    }
257
258                    var pullRequestId = b["pullRequestId"];
259                    var version = b["version"].ToObject<string>();
260                    var status = ParseBuildStatus(b["status"].ToObject<string>());
261                    long? duration = null;
262                    if (status == BuildInfo.BuildStatus.Success || status == BuildInfo.BuildStatus.Failure)
263                    {
264                        duration = GetBuildDuration(b);
265                    }
266
267                    var pullRequestTitle = b["pullRequestName"];
268
269                    buildDetails.Add(new AppVeyorBuildInfo
270                    {
271                        Id = version,
272                        BuildId = b["buildId"].ToObject<string>(),
273                        Branch = b["branch"].ToObject<string>(),
274                        CommitId = objectId,
275                        CommitHashList = new[] { objectId },
276                        Status = status,
277                        StartDate = b["started"]?.ToObject<DateTime>() ?? DateTime.MinValue,
278                        BaseWebUrl = baseWebUrl,
279                        Url = WebSiteUrl + "/project/" + project.Id + "/build/" + version,
280                        PullRequestUrl = repositoryType != null && repositoryName != null && pullRequestId != null
281                            ? BuildPullRequetUrl(repositoryType.Value<string>(), repositoryName.Value<string>(),
282                                pullRequestId.Value<string>())
283                            : null,
284                        BaseApiUrl = baseApiUrl,
285                        AppVeyorBuildReportUrl = baseApiUrl + "/build/" + version,
286                        PullRequestText = pullRequestId != null ? "PR#" + pullRequestId.Value<string>() : string.Empty,
287                        PullRequestTitle = pullRequestTitle != null ? pullRequestTitle.Value<string>() : string.Empty,
288                        Duration = duration,
289                        TestsResultText = string.Empty
290                    });
291                }
292                catch (Exception)
293                {
294                    // Failure on reading data on a build detail should not prevent to display the others build results
295                }
296            }
297
298            return buildDetails;
299        }
300
301        /// <summary>
302        /// Gets a unique key which identifies this build server.
303        /// </summary>
304        public string UniqueKey => _httpClientAppVeyor.BaseAddress.Host;
305
306        public IObservable<BuildInfo> GetFinishedBuildsSince(IScheduler scheduler, DateTime? sinceDate = null)
307        {
308            // AppVeyor api is different than TeamCity one and all build results are fetch in one call without
309            // filter parameters possible (so this call is useless!)
310            return Observable.Empty<BuildInfo>();
311        }
312
313        public IObservable<BuildInfo> GetRunningBuilds(IScheduler scheduler)
314        {
315            return GetBuilds(scheduler);
316        }
317
318        private IObservable<BuildInfo> GetBuilds(IScheduler scheduler)
319        {
320            return Observable.Create<BuildInfo>((observer, cancellationToken) =>
321                Task.Run(
322                    () => scheduler.Schedule(() => ObserveBuilds(observer, cancellationToken))));
323        }
324
325        private void ObserveBuilds(IObserver<BuildInfo> observer, CancellationToken cancellationToken)
326        {
327            try
328            {
329                if (_allBuilds == null)
330                {
331                    return;
332                }
333
334                // Display all builds found
335                foreach (var build in _allBuilds)
336                {
337                    UpdateDisplay(observer, build);
338                }
339
340                // Update finished build with tests results
341                if (_shouldLoadTestResults)
342                {
343                    foreach (var build in _allBuilds.Where(b => b.Status == BuildInfo.BuildStatus.Success
344                                                                || b.Status == BuildInfo.BuildStatus.Failure))
345                    {
346                        UpdateDescription(build, cancellationToken);
347                        UpdateDisplay(observer, build);
348                    }
349                }
350
351                // Manage in progress builds...
352                var inProgressBuilds = _allBuilds.Where(b => b.Status == BuildInfo.BuildStatus.InProgress).ToList();
353                _allBuilds = null;
354                do
355                {
356                    Thread.Sleep(5000);
357                    foreach (var build in inProgressBuilds)
358                    {
359                        UpdateDescription(build, cancellationToken);
360                        UpdateDisplay(observer, build);
361                    }
362
363                    inProgressBuilds = inProgressBuilds.Where(b => b.Status == BuildInfo.BuildStatus.InProgress).ToList();
364                }
365                while (inProgressBuilds.Any());
366
367                observer.OnCompleted();
368            }
369            catch (OperationCanceledException)
370            {
371                // Do nothing, the observer is already stopped
372            }
373            catch (Exception ex)
374            {
375                observer.OnError(ex);
376            }
377        }
378
379        private static void UpdateDisplay(IObserver<BuildInfo> observer, AppVeyorBuildInfo build)
380        {
381            build.UpdateDescription();
382            observer.OnNext(build);
383        }
384
385        private List<AppVeyorBuildInfo> FilterBuilds(IEnumerable<AppVeyorBuildInfo> allBuilds)
386        {
387            var filteredBuilds = new List<AppVeyorBuildInfo>();
388            foreach (var build in allBuilds.OrderByDescending(b => b.StartDate))
389            {
390                if (!_fetchBuilds.Contains(build.CommitId))
391                {
392                    filteredBuilds.Add(build);
393                    _fetchBuilds.Add(build.CommitId);
394                }
395            }
396
397            return filteredBuilds;
398        }
399
400        private void UpdateDescription(AppVeyorBuildInfo buildDetails, CancellationToken cancellationToken)
401        {
402            var buildDetailsParsed = ThreadHelper.JoinableTaskFactory.Run(() => FetchBuildDetailsManagingVersionUpdateAsync(buildDetails, cancellationToken));
403            if (buildDetailsParsed == null)
404            {
405                return;
406            }
407
408            var buildData = buildDetailsParsed["build"];
409            var buildDescription = buildData["jobs"].Last();
410
411            var status = buildDescription["status"].ToObject<string>();
412            buildDetails.Status = ParseBuildStatus(status);
413
414            buildDetails.ChangeProgressCounter();
415            if (!buildDetails.IsRunning)
416            {
417                buildDetails.Duration = GetBuildDuration(buildData);
418            }
419
420            int testCount = buildDescription["testsCount"].ToObject<int>();
421            if (testCount != 0)
422            {
423                int failedTestCount = buildDescription["failedTestsCount"].ToObject<int>();
424                int skippedTestCount = testCount - buildDescription["passedTestsCount"].ToObject<int>();
425                var testResults = testCount + " tests";
426                if (failedTestCount != 0 || skippedTestCount != 0)
427                {
428                    testResults += $" ( {failedTestCount} failed, {skippedTestCount} skipped )";
429                }
430
431                buildDetails.TestsResultText = testResults;
432            }
433        }
434
435        private static long GetBuildDuration(JToken buildData)
436        {
437            var startTime = buildData["started"].ToObject<DateTime>();
438            var updateTime = buildData["updated"].ToObject<DateTime>();
439            return (long)(updateTime - startTime).TotalMilliseconds;
440        }
441
442        private async Task<JObject> FetchBuildDetailsManagingVersionUpdateAsync(AppVeyorBuildInfo buildDetails, CancellationToken cancellationToken)
443        {
444            try
445            {
446                return JObject.Parse(await GetResponseAsync(_httpClientAppVeyor, buildDetails.AppVeyorBuildReportUrl, cancellationToken).ConfigureAwait(false));
447            }
448            catch
449            {
450                var buildHistoryUrl = buildDetails.BaseApiUrl + "/history?recordsNumber=1&startBuildId=" + (int.Parse(buildDetails.BuildId) + 1);
451                var builds = JObject.Parse(await GetResponseAsync(_httpClientAppVeyor, buildHistoryUrl, cancellationToken).ConfigureAwait(false));
452
453                var version = builds["builds"][0]["version"].ToObject<string>();
454                buildDetails.Id = version;
455                buildDetails.AppVeyorBuildReportUrl = buildDetails.BaseApiUrl + "/build/" + version;
456                buildDetails.Url = buildDetails.BaseWebUrl + version;
457
458                return JObject.Parse(await GetResponseAsync(_httpClientAppVeyor, buildDetails.AppVeyorBuildReportUrl, cancellationToken).ConfigureAwait(false));
459            }
460        }
461
462        private static BuildInfo.BuildStatus ParseBuildStatus(string statusValue)
463        {
464            switch (statusValue)
465            {
466                case "success":
467                    return BuildInfo.BuildStatus.Success;
468                case "failed":
469                    return BuildInfo.BuildStatus.Failure;
470                case "cancelled":
471                    return BuildInfo.BuildStatus.Stopped;
472                case "queued":
473                case "running":
474                    return BuildInfo.BuildStatus.InProgress;
475                default:
476                    return BuildInfo.BuildStatus.Unknown;
477            }
478        }
479
480        private Task<Stream> GetStreamAsync(HttpClient httpClient, string restServicePath, CancellationToken cancellationToken)
481        {
482            cancellationToken.ThrowIfCancellationRequested();
483
484            return httpClient.GetAsync(restServicePath, HttpCompletionOption.ResponseHeadersRead, cancellationToken)
485                             .ContinueWith(
486#pragma warning disable VSTHRD003 // Avoid awaiting foreign Tasks (task is a completed task)
487                                 task => GetStreamFromHttpResponseAsync(httpClient, task, restServicePath, cancellationToken),
488#pragma warning restore VSTHRD003 // Avoid awaiting foreign Tasks
489                                 cancellationToken,
490                                 restServicePath.Contains("github") ? TaskContinuationOptions.None : TaskContinuationOptions.AttachedToParent,
491                                 TaskScheduler.Current)
492                             .Unwrap();
493        }
494
495        private Task<Stream> GetStreamFromHttpResponseAsync(HttpClient httpClient, Task<HttpResponseMessage> task, string restServicePath, CancellationToken cancellationToken)
496        {
497            var retry = task.IsCanceled && !cancellationToken.IsCancellationRequested;
498
499            if (retry)
500            {
501                return GetStreamAsync(httpClient, restServicePath, cancellationToken);
502            }
503
504            if (task.Status == TaskStatus.RanToCompletion && task.CompletedResult().IsSuccessStatusCode)
505            {
506                return task.CompletedResult().Content.ReadAsStreamAsync();
507            }
508
509            return null;
510        }
511
512        private Task<string> GetResponseAsync(HttpClient httpClient, string relativePath, CancellationToken cancellationToken)
513        {
514            var getStreamTask = GetStreamAsync(httpClient, relativePath, cancellationToken);
515
516            var taskContinuationOptions = relativePath.Contains("github") ? TaskContinuationOptions.None : TaskContinuationOptions.AttachedToParent;
517            return getStreamTask.ContinueWith(
518                task =>
519                {
520                    if (task.Status != TaskStatus.RanToCompletion)
521                    {
522                        return string.Empty;
523                    }
524
525                    using (var responseStream = task.Result)
526                    {
527                        return new StreamReader(responseStream).ReadToEnd();
528                    }
529                },
530                cancellationToken,
531                taskContinuationOptions,
532                TaskScheduler.Current);
533        }
534
535        public void Dispose()
536        {
537            GC.SuppressFinalize(this);
538
539            _httpClientAppVeyor?.Dispose();
540        }
541    }
542}