PageRenderTime 45ms CodeModel.GetById 2ms app.highlight 35ms RepoModel.GetById 1ms app.codeStats 1ms

/Plugins/BuildServerIntegration/JenkinsIntegration/JenkinsAdapter.cs

https://github.com/gitextensions/gitextensions
C# | 561 lines | 456 code | 71 blank | 34 comment | 73 complexity | 7f70b847c7b06a8dfaa0086efe0a27fb MD5 | raw file
  1using System;
  2using System.Collections.Generic;
  3using System.ComponentModel.Composition;
  4using System.Diagnostics;
  5using System.IO;
  6using System.Linq;
  7using System.Net;
  8using System.Net.Http;
  9using System.Net.Http.Headers;
 10using System.Reactive.Concurrency;
 11using System.Reactive.Linq;
 12using System.Text;
 13using System.Text.RegularExpressions;
 14using System.Threading;
 15using System.Threading.Tasks;
 16using GitCommands.Utils;
 17using GitUI;
 18using GitUIPluginInterfaces;
 19using GitUIPluginInterfaces.BuildServerIntegration;
 20using JetBrains.Annotations;
 21using Microsoft.VisualStudio.Threading;
 22using Newtonsoft.Json.Linq;
 23
 24namespace JenkinsIntegration
 25{
 26    [MetadataAttribute]
 27    [AttributeUsage(AttributeTargets.Class)]
 28    public class JenkinsIntegrationMetadata : BuildServerAdapterMetadataAttribute
 29    {
 30        public JenkinsIntegrationMetadata(string buildServerType)
 31            : base(buildServerType)
 32        {
 33        }
 34
 35        public override string CanBeLoaded
 36        {
 37            get
 38            {
 39                if (EnvUtils.IsNet4FullOrHigher())
 40                {
 41                    return null;
 42                }
 43
 44                return ".Net 4 full framework required";
 45            }
 46        }
 47    }
 48
 49    [Export(typeof(IBuildServerAdapter))]
 50    [JenkinsIntegrationMetadata(PluginName)]
 51    [PartCreationPolicy(CreationPolicy.NonShared)]
 52    internal class JenkinsAdapter : IBuildServerAdapter
 53    {
 54        public const string PluginName = "Jenkins";
 55        private static readonly IBuildDurationFormatter _buildDurationFormatter = new BuildDurationFormatter();
 56        private IBuildServerWatcher _buildServerWatcher;
 57
 58        private HttpClient _httpClient;
 59
 60        private readonly Dictionary<string, JenkinsCacheInfo> _lastBuildCache = new Dictionary<string, JenkinsCacheInfo>();
 61        private readonly List<string> _projectsUrls = new List<string>();
 62        private Regex _ignoreBuilds;
 63
 64        public void Initialize(IBuildServerWatcher buildServerWatcher, ISettingsSource config, Func<ObjectId, bool> isCommitInRevisionGrid = null)
 65        {
 66            if (_buildServerWatcher != null)
 67            {
 68                throw new InvalidOperationException("Already initialized");
 69            }
 70
 71            _buildServerWatcher = buildServerWatcher;
 72
 73            var projectName = config.GetString("ProjectName", null);
 74            var hostName = config.GetString("BuildServerUrl", null);
 75
 76            if (!string.IsNullOrEmpty(hostName) && !string.IsNullOrEmpty(projectName))
 77            {
 78                var baseAddress = hostName.Contains("://")
 79                    ? new Uri(hostName, UriKind.Absolute)
 80                    : new Uri($"{Uri.UriSchemeHttp}://{hostName}:8080", UriKind.Absolute);
 81
 82                _httpClient = new HttpClient(new HttpClientHandler { UseDefaultCredentials = true })
 83                {
 84                    Timeout = TimeSpan.FromMinutes(2),
 85                    BaseAddress = baseAddress
 86                };
 87
 88                var buildServerCredentials = buildServerWatcher.GetBuildServerCredentials(this, true);
 89
 90                UpdateHttpClientOptions(buildServerCredentials);
 91
 92                string[] projectUrls = _buildServerWatcher.ReplaceVariables(projectName)
 93                    .Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
 94                foreach (var projectUrl in projectUrls.Select(s => baseAddress + "job/" + s.Trim() + "/"))
 95                {
 96                    AddGetBuildUrl(projectUrl);
 97                }
 98            }
 99
100            var ignoreBuilds = config.GetString("IgnoreBuildBranch", string.Empty);
101            _ignoreBuilds = !string.IsNullOrWhiteSpace(ignoreBuilds) ? new Regex(ignoreBuilds) : null;
102        }
103
104        /// <summary>
105        /// Gets a unique key which identifies this build server.
106        /// </summary>
107        public string UniqueKey => _httpClient.BaseAddress.Host;
108
109        private void AddGetBuildUrl(string projectUrl)
110        {
111            if (!_projectsUrls.Contains(projectUrl))
112            {
113                _projectsUrls.Add(projectUrl);
114                _lastBuildCache[projectUrl] = new JenkinsCacheInfo();
115            }
116        }
117
118        public class ResponseInfo
119        {
120            public string Url { get; set; }
121            public long Timestamp { get; set; }
122            public IEnumerable<JToken> JobDescription { get; set; }
123        }
124
125        public class JenkinsCacheInfo
126        {
127            public long Timestamp = -1;
128        }
129
130        private async Task<ResponseInfo> GetBuildInfoTaskAsync(string projectUrl, bool fullInfo, CancellationToken cancellationToken)
131        {
132            string t = null;
133            long timestamp = 0;
134            IEnumerable<JToken> s = Enumerable.Empty<JToken>();
135
136            try
137            {
138                t = await GetResponseAsync(FormatToGetJson(projectUrl, fullInfo), cancellationToken).ConfigureAwait(false);
139            }
140            catch
141            {
142                // Could be cancelled or failed. Explicitly assign 't' to reveal the intended behavior of following code
143                // for this case.
144                t = null;
145            }
146
147            if (!string.IsNullOrWhiteSpace(t) && !cancellationToken.IsCancellationRequested)
148            {
149                JObject jobDescription = JObject.Parse(t);
150                if (jobDescription["builds"] != null)
151                {
152                    // Freestyle jobs
153                    s = jobDescription["builds"];
154                }
155                else if (jobDescription["jobs"] != null)
156                {
157                    // Multi-branch pipeline
158                    s = jobDescription["jobs"]
159                        .SelectMany(j => j["builds"]);
160                    foreach (var j in jobDescription["jobs"])
161                    {
162                        try
163                        {
164                            if (j["lastBuild"] != null && j["lastBuild"]["timestamp"] != null)
165                            {
166                                var ts = j["lastBuild"]["timestamp"];
167                                timestamp = Math.Max(timestamp, ts.ToObject<long>());
168                            }
169                        }
170                        catch
171                        {
172                            // Ignore malformed build ids
173                        }
174                    }
175                }
176
177                // else: The server had no response (overloaded?) or a multi-branch pipeline is not configured
178                if (timestamp == 0 && jobDescription["lastBuild"]?["timestamp"] != null)
179                {
180                    timestamp = jobDescription["lastBuild"]["timestamp"].ToObject<long>();
181                }
182            }
183
184            return new ResponseInfo
185            {
186                Url = projectUrl,
187                Timestamp = timestamp,
188                JobDescription = s
189            };
190        }
191
192        public IObservable<BuildInfo> GetFinishedBuildsSince(IScheduler scheduler, DateTime? sinceDate = null)
193        {
194            // GetBuilds() will return the same builds as for GetRunningBuilds().
195            // Multiple calls will fetch same info multiple times and make debugging very confusing
196            // Similar as for AppVeyor
197            return Observable.Empty<BuildInfo>();
198        }
199
200        public IObservable<BuildInfo> GetRunningBuilds(IScheduler scheduler)
201        {
202            return GetBuilds(scheduler, null, true);
203        }
204
205        private IObservable<BuildInfo> GetBuilds(IScheduler scheduler, DateTime? sinceDate = null, bool? running = null)
206        {
207            return Observable.Create<BuildInfo>((observer, cancellationToken) =>
208                ThreadHelper.JoinableTaskFactory.RunAsync(async () =>
209                {
210                    await TaskScheduler.Default;
211                    return scheduler.Schedule(() => ObserveBuilds(sinceDate, running, observer, cancellationToken));
212                }).Task);
213        }
214
215        private void ObserveBuilds(DateTime? sinceDate, bool? running, IObserver<BuildInfo> observer, CancellationToken cancellationToken)
216        {
217            // Note that 'running' is ignored (attempt to fetch data when updated)
218            // Similar for 'sinceDate', not supported in Jenkins API
219            try
220            {
221                var allBuildInfos = new List<JoinableTask<ResponseInfo>>();
222                var latestBuildInfos = new List<JoinableTask<ResponseInfo>>();
223
224                foreach (var projectUrl in _projectsUrls)
225                {
226                    if (_lastBuildCache[projectUrl].Timestamp <= 0)
227                    {
228                        // This job must be updated, no need to to check the latest builds
229                        allBuildInfos.Add(ThreadHelper.JoinableTaskFactory.RunAsync(() => GetBuildInfoTaskAsync(projectUrl, true, cancellationToken)));
230                    }
231                    else
232                    {
233                        latestBuildInfos.Add(ThreadHelper.JoinableTaskFactory.RunAsync(() => GetBuildInfoTaskAsync(projectUrl, false, cancellationToken)));
234                    }
235                }
236
237                // Check the latest build on the server to the existing build cache
238                // The simple request will limit the load on the Jenkins server
239                // To fetch just new builds is possible too, but it will make the solution more complicated
240                // Similar, the build results could be cached so they are available when switching repos
241                foreach (var info in latestBuildInfos)
242                {
243                    if (!info.Task.IsFaulted)
244                    {
245                        if (info.Join().Timestamp > _lastBuildCache[info.Join().Url].Timestamp)
246                        {
247                            // The cache has at least one newer job, query the status
248                            allBuildInfos.Add(ThreadHelper.JoinableTaskFactory.RunAsync(() => GetBuildInfoTaskAsync(info.Task.CompletedResult().Url, true, cancellationToken)));
249                        }
250                    }
251                }
252
253                if (allBuildInfos.All(t => t.Task.IsCanceled))
254                {
255                    observer.OnCompleted();
256                    return;
257                }
258
259                foreach (var build in allBuildInfos)
260                {
261                    if (build.Task.IsFaulted)
262                    {
263                        Debug.Assert(build.Task.Exception != null, "build.Task.Exception != null");
264
265                        observer.OnError(build.Task.Exception);
266                        continue;
267                    }
268
269                    if (build.Task.IsCanceled || build.Join().Timestamp <= 0)
270                    {
271                        // No valid information received for the build
272                        continue;
273                    }
274
275                    _lastBuildCache[build.Join().Url].Timestamp = build.Join().Timestamp;
276
277                    // Present information in reverse, so the latest job is displayed (i.e. new inprogress on one commit)
278                    // (for multi-branch pipeline, ignore the corner case with multiple branches with inprogress builds on one commit)
279                    foreach (var buildDetails in build.Join().JobDescription.Reverse())
280                    {
281                        if (cancellationToken.IsCancellationRequested)
282                        {
283                            return;
284                        }
285
286                        try
287                        {
288                            var buildInfo = CreateBuildInfo((JObject)buildDetails);
289
290                            if (buildInfo != null)
291                            {
292                                observer.OnNext(buildInfo);
293
294                                if (buildInfo.Status == BuildInfo.BuildStatus.InProgress)
295                                {
296                                    // Need to make a full request next time
297                                    _lastBuildCache[build.Join().Url].Timestamp = 0;
298                                }
299                            }
300                        }
301                        catch
302                        {
303                            // Ignore unexpected responses
304                        }
305                    }
306                }
307
308                // Complete the job, it will be run again with Observe.Retry() (every 10th sec)
309                observer.OnCompleted();
310            }
311            catch (OperationCanceledException)
312            {
313                // Do nothing, the observer is already stopped
314            }
315            catch (Exception ex)
316            {
317                // Cancelling a sub-task is similar to cancelling this task
318                if (!(ex.InnerException is OperationCanceledException))
319                {
320                    observer.OnError(ex);
321                }
322            }
323        }
324
325        private const string _jenkinsTreeBuildInfo = "number,result,timestamp,url,actions[lastBuiltRevision[SHA1,branch[name]],totalCount,failCount,skipCount],building,duration";
326
327        [CanBeNull]
328        private BuildInfo CreateBuildInfo(JObject buildDescription)
329        {
330            var idValue = buildDescription["number"].ToObject<string>();
331            var statusValue = buildDescription["result"].ToObject<string>();
332            var startDateTicks = buildDescription["timestamp"].ToObject<long>();
333            var webUrl = buildDescription["url"].ToObject<string>();
334
335            var action = buildDescription["actions"];
336            var commitHashList = new List<ObjectId>();
337            string testResults = string.Empty;
338            foreach (var element in action)
339            {
340                if (element["lastBuiltRevision"] != null)
341                {
342                    commitHashList.Add(ObjectId.Parse(element["lastBuiltRevision"]["SHA1"].ToObject<string>()));
343                    var branches = element["lastBuiltRevision"]["branch"];
344                    if (_ignoreBuilds != null && branches != null)
345                    {
346                        // Ignore build events for specified branches
347                        foreach (var branch in branches)
348                        {
349                            var name = branch["name"];
350                            if (name != null)
351                            {
352                                var name2 = name.ToObject<string>();
353                                if (!string.IsNullOrWhiteSpace(name2) && _ignoreBuilds.IsMatch(name2))
354                                {
355                                    return null;
356                                }
357                            }
358                        }
359                    }
360                }
361
362                if (element["totalCount"] != null)
363                {
364                    int testCount = element["totalCount"].ToObject<int>();
365                    if (testCount != 0)
366                    {
367                        int failedTestCount = element["failCount"].ToObject<int>();
368                        int skippedTestCount = element["skipCount"].ToObject<int>();
369                        testResults = $"{testCount} tests ({failedTestCount} failed, {skippedTestCount} skipped)";
370                    }
371                }
372            }
373
374            var isRunning = buildDescription["building"].ToObject<bool>();
375            long? buildDuration;
376            if (isRunning)
377            {
378                buildDuration = null;
379            }
380            else
381            {
382                buildDuration = buildDescription["duration"].ToObject<long>();
383            }
384
385            var status = isRunning ? BuildInfo.BuildStatus.InProgress : ParseBuildStatus(statusValue);
386            var statusText = status.ToString("G");
387            var buildInfo = new BuildInfo
388            {
389                Id = idValue,
390                StartDate = TimestampToDateTime(startDateTicks),
391                Duration = buildDuration,
392                Status = status,
393                CommitHashList = commitHashList.ToArray(),
394                Url = webUrl
395            };
396            var durationText = _buildDurationFormatter.Format(buildInfo.Duration);
397            buildInfo.Description = $"#{idValue} {durationText} {testResults} {statusText}";
398            return buildInfo;
399        }
400
401        public static DateTime TimestampToDateTime(long timestamp)
402        {
403            return new DateTime(1970, 1, 1, 0, 0, 0, DateTime.Now.Kind).AddMilliseconds(timestamp);
404        }
405
406        private static AuthenticationHeaderValue CreateBasicHeader(string username, string password)
407        {
408            byte[] byteArray = Encoding.UTF8.GetBytes(string.Format("{0}:{1}", username, password));
409            return new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));
410        }
411
412        private static BuildInfo.BuildStatus ParseBuildStatus(string statusValue)
413        {
414            switch (statusValue)
415            {
416                case "SUCCESS":
417                    return BuildInfo.BuildStatus.Success;
418                case "FAILURE":
419                    return BuildInfo.BuildStatus.Failure;
420                case "UNSTABLE":
421                    return BuildInfo.BuildStatus.Unstable;
422                case "ABORTED":
423                    return BuildInfo.BuildStatus.Stopped;
424                default:
425                    return BuildInfo.BuildStatus.Unknown;
426            }
427        }
428
429        private async Task<Stream> GetStreamAsync(string restServicePath, CancellationToken cancellationToken)
430        {
431            cancellationToken.ThrowIfCancellationRequested();
432
433            var response = await _httpClient.GetAsync(restServicePath, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
434
435            return await GetStreamFromHttpResponseAsync(response);
436
437            async Task<Stream> GetStreamFromHttpResponseAsync(HttpResponseMessage resp)
438            {
439                bool unauthorized = resp.StatusCode == HttpStatusCode.Unauthorized;
440
441                if (resp.IsSuccessStatusCode)
442                {
443                    var httpContent = resp.Content;
444
445                    if (httpContent.Headers.ContentType.MediaType == "text/html")
446                    {
447                        // Jenkins responds with an HTML login page when guest access is denied.
448                        unauthorized = true;
449                    }
450                    else
451                    {
452                        return await httpContent.ReadAsStreamAsync();
453                    }
454                }
455                else if (resp.StatusCode == HttpStatusCode.NotFound)
456                {
457                    // The url does not exist, no jobs to retrieve
458                    return null;
459                }
460                else if (resp.StatusCode == HttpStatusCode.Forbidden)
461                {
462                    unauthorized = true;
463                }
464
465                if (unauthorized)
466                {
467                    var buildServerCredentials = _buildServerWatcher.GetBuildServerCredentials(this, false);
468
469                    if (buildServerCredentials != null)
470                    {
471                        UpdateHttpClientOptions(buildServerCredentials);
472
473                        return await GetStreamAsync(restServicePath, cancellationToken);
474                    }
475
476                    throw new OperationCanceledException(resp.ReasonPhrase);
477                }
478
479                throw new HttpRequestException(resp.ReasonPhrase);
480            }
481        }
482
483        private void UpdateHttpClientOptions(IBuildServerCredentials buildServerCredentials)
484        {
485            var useGuestAccess = buildServerCredentials == null || buildServerCredentials.UseGuestAccess;
486
487            _httpClient.DefaultRequestHeaders.Authorization = useGuestAccess
488                ? null : CreateBasicHeader(buildServerCredentials.Username, buildServerCredentials.Password);
489        }
490
491        private async Task<string> GetResponseAsync(string relativePath, CancellationToken cancellationToken)
492        {
493            using (var responseStream = await GetStreamAsync(relativePath, cancellationToken).ConfigureAwait(false))
494            {
495                using (var reader = new StreamReader(responseStream))
496                {
497                    return await reader.ReadToEndAsync();
498                }
499            }
500        }
501
502        private static string FormatToGetJson(string restServicePath, bool buildsInfo = false)
503        {
504            string buildTree = "lastBuild[timestamp]";
505            int depth = 1;
506            int postIndex = restServicePath.IndexOf('?');
507            if (postIndex >= 0)
508            {
509                int endLen = restServicePath.Length - postIndex;
510                if (restServicePath.EndsWith("/"))
511                {
512                    endLen--;
513                }
514
515                string post = restServicePath.Substring(postIndex, endLen);
516                if (post == "?m")
517                {
518                    // Multi pipeline project
519                    buildTree = "jobs[" + buildTree;
520                    if (buildsInfo)
521                    {
522                        depth = 2;
523                        buildTree += ",builds[" + _jenkinsTreeBuildInfo + "]";
524                    }
525
526                    buildTree += "]";
527                }
528                else
529                {
530                    // user defined format (will likely require changes in the code)
531                    buildTree = post;
532                }
533
534                restServicePath = restServicePath.Substring(0, postIndex);
535            }
536            else
537            {
538                // Freestyle project
539                if (buildsInfo)
540                {
541                    buildTree += ",builds[" + _jenkinsTreeBuildInfo + "]";
542                }
543            }
544
545            if (!restServicePath.EndsWith("/"))
546            {
547                restServicePath += "/";
548            }
549
550            restServicePath += "api/json?depth=" + depth + "&tree=" + buildTree;
551            return restServicePath;
552        }
553
554        public void Dispose()
555        {
556            GC.SuppressFinalize(this);
557
558            _httpClient?.Dispose();
559        }
560    }
561}