PageRenderTime 23ms CodeModel.GetById 2ms app.highlight 15ms RepoModel.GetById 1ms app.codeStats 1ms

/Plugins/BuildServerIntegration/JenkinsIntegration/JenkinsAdapter.cs

http://github.com/spdr870/gitextensions
C# | 554 lines | 450 code | 71 blank | 33 comment | 71 complexity | f77062f9412b8f4de8714773f2007eab MD5 | raw file
  1using System;
  2using System.Collections.Generic;
  3using System.ComponentModel.Composition;
  4using System.Diagnostics;
  5using System.IO;
  6using System.Linq;
  7using System.Net;
  8using System.Net.Http;
  9using System.Net.Http.Headers;
 10using System.Reactive.Concurrency;
 11using System.Reactive.Linq;
 12using System.Text;
 13using System.Text.RegularExpressions;
 14using System.Threading;
 15using System.Threading.Tasks;
 16using GitCommands.Utils;
 17using GitUI;
 18using GitUIPluginInterfaces;
 19using GitUIPluginInterfaces.BuildServerIntegration;
 20using JetBrains.Annotations;
 21using Microsoft.VisualStudio.Threading;
 22using Newtonsoft.Json.Linq;
 23
 24namespace JenkinsIntegration
 25{
 26    [MetadataAttribute]
 27    [AttributeUsage(AttributeTargets.Class)]
 28    public class JenkinsIntegrationMetadata : BuildServerAdapterMetadataAttribute
 29    {
 30        public JenkinsIntegrationMetadata(string buildServerType)
 31            : base(buildServerType)
 32        {
 33        }
 34
 35        public override string CanBeLoaded
 36        {
 37            get
 38            {
 39                if (EnvUtils.IsNet4FullOrHigher())
 40                {
 41                    return null;
 42                }
 43
 44                return ".Net 4 full framework required";
 45            }
 46        }
 47    }
 48
 49    [Export(typeof(IBuildServerAdapter))]
 50    [JenkinsIntegrationMetadata(PluginName)]
 51    [PartCreationPolicy(CreationPolicy.NonShared)]
 52    internal class JenkinsAdapter : IBuildServerAdapter
 53    {
 54        public const string PluginName = "Jenkins";
 55        private static readonly IBuildDurationFormatter _buildDurationFormatter = new BuildDurationFormatter();
 56        private IBuildServerWatcher _buildServerWatcher;
 57
 58        private HttpClient _httpClient;
 59
 60        private readonly Dictionary<string, JenkinsCacheInfo> _lastBuildCache = new Dictionary<string, JenkinsCacheInfo>();
 61        private readonly List<string> _projectsUrls = new List<string>();
 62        private Regex _ignoreBuilds;
 63
 64        public void Initialize(IBuildServerWatcher buildServerWatcher, ISettingsSource config, Func<ObjectId, bool> isCommitInRevisionGrid = null)
 65        {
 66            if (_buildServerWatcher != null)
 67            {
 68                throw new InvalidOperationException("Already initialized");
 69            }
 70
 71            _buildServerWatcher = buildServerWatcher;
 72
 73            var projectName = config.GetString("ProjectName", null);
 74            var hostName = config.GetString("BuildServerUrl", null);
 75
 76            if (!string.IsNullOrEmpty(hostName) && !string.IsNullOrEmpty(projectName))
 77            {
 78                var baseAddress = hostName.Contains("://")
 79                    ? new Uri(hostName, UriKind.Absolute)
 80                    : new Uri($"{Uri.UriSchemeHttp}://{hostName}:8080", UriKind.Absolute);
 81
 82                _httpClient = new HttpClient(new HttpClientHandler { UseDefaultCredentials = true })
 83                {
 84                    Timeout = TimeSpan.FromMinutes(2),
 85                    BaseAddress = baseAddress
 86                };
 87
 88                var buildServerCredentials = buildServerWatcher.GetBuildServerCredentials(this, true);
 89
 90                UpdateHttpClientOptions(buildServerCredentials);
 91
 92                string[] projectUrls = _buildServerWatcher.ReplaceVariables(projectName)
 93                    .Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
 94                foreach (var projectUrl in projectUrls.Select(s => baseAddress + "job/" + s.Trim() + "/"))
 95                {
 96                    AddGetBuildUrl(projectUrl);
 97                }
 98            }
 99
100            var ignoreBuilds = config.GetString("IgnoreBuildBranch", string.Empty);
101            _ignoreBuilds = ignoreBuilds.IsNotNullOrWhitespace() ? new Regex(ignoreBuilds) : null;
102        }
103
104        /// <summary>
105        /// Gets a unique key which identifies this build server.
106        /// </summary>
107        public string UniqueKey => _httpClient.BaseAddress.Host;
108
109        private void AddGetBuildUrl(string projectUrl)
110        {
111            if (!_projectsUrls.Contains(projectUrl))
112            {
113                _projectsUrls.Add(projectUrl);
114                _lastBuildCache[projectUrl] = new JenkinsCacheInfo();
115            }
116        }
117
118        public class ResponseInfo
119        {
120            public string Url { get; set; }
121            public long Timestamp { get; set; }
122            public IEnumerable<JToken> JobDescription { get; set; }
123        }
124
125        public class JenkinsCacheInfo
126        {
127            public long Timestamp = -1;
128        }
129
130        private async Task<ResponseInfo> GetBuildInfoTaskAsync(string projectUrl, bool fullInfo, CancellationToken cancellationToken)
131        {
132            string t = null;
133            long timestamp = 0;
134            IEnumerable<JToken> s = Enumerable.Empty<JToken>();
135
136            try
137            {
138                t = await GetResponseAsync(FormatToGetJson(projectUrl, fullInfo), cancellationToken).ConfigureAwait(false);
139            }
140            catch
141            {
142                // Could be cancelled or failed. Explicitly assign 't' to reveal the intended behavior of following code
143                // for this case.
144                t = null;
145            }
146
147            if (t.IsNotNullOrWhitespace() && !cancellationToken.IsCancellationRequested)
148            {
149                JObject jobDescription = JObject.Parse(t);
150                if (jobDescription["builds"] != null)
151                {
152                    // Freestyle jobs
153                    s = jobDescription["builds"];
154                }
155                else if (jobDescription["jobs"] != null)
156                {
157                    // Multi-branch pipeline
158                    s = jobDescription["jobs"]
159                        .SelectMany(j => j["builds"]);
160                    foreach (var j in jobDescription["jobs"])
161                    {
162                        var ts = j["lastBuild"]["timestamp"];
163                        if (ts != null)
164                        {
165                            timestamp = Math.Max(timestamp, ts.ToObject<long>());
166                        }
167                    }
168                }
169
170                // else: The server had no response (overloaded?) or a multi-branch pipeline is not configured
171                if (timestamp == 0 && jobDescription["lastBuild"]?["timestamp"] != null)
172                {
173                    timestamp = jobDescription["lastBuild"]["timestamp"].ToObject<long>();
174                }
175            }
176
177            return new ResponseInfo
178            {
179                Url = projectUrl,
180                Timestamp = timestamp,
181                JobDescription = s
182            };
183        }
184
185        public IObservable<BuildInfo> GetFinishedBuildsSince(IScheduler scheduler, DateTime? sinceDate = null)
186        {
187            // GetBuilds() will return the same builds as for GetRunningBuilds().
188            // Multiple calls will fetch same info multiple times and make debugging very confusing
189            // Similar as for AppVeyor
190            return Observable.Empty<BuildInfo>();
191        }
192
193        public IObservable<BuildInfo> GetRunningBuilds(IScheduler scheduler)
194        {
195            return GetBuilds(scheduler, null, true);
196        }
197
198        private IObservable<BuildInfo> GetBuilds(IScheduler scheduler, DateTime? sinceDate = null, bool? running = null)
199        {
200            return Observable.Create<BuildInfo>((observer, cancellationToken) =>
201                ThreadHelper.JoinableTaskFactory.RunAsync(async () =>
202                {
203                    await TaskScheduler.Default;
204                    return scheduler.Schedule(() => ObserveBuilds(sinceDate, running, observer, cancellationToken));
205                }).Task);
206        }
207
208        private void ObserveBuilds(DateTime? sinceDate, bool? running, IObserver<BuildInfo> observer, CancellationToken cancellationToken)
209        {
210            // Note that 'running' is ignored (attempt to fetch data when updated)
211            // Similar for 'sinceDate', not supported in Jenkins API
212            try
213            {
214                var allBuildInfos = new List<JoinableTask<ResponseInfo>>();
215                var latestBuildInfos = new List<JoinableTask<ResponseInfo>>();
216
217                foreach (var projectUrl in _projectsUrls)
218                {
219                    if (_lastBuildCache[projectUrl].Timestamp <= 0)
220                    {
221                        // This job must be updated, no need to to check the latest builds
222                        allBuildInfos.Add(ThreadHelper.JoinableTaskFactory.RunAsync(() => GetBuildInfoTaskAsync(projectUrl, true, cancellationToken)));
223                    }
224                    else
225                    {
226                        latestBuildInfos.Add(ThreadHelper.JoinableTaskFactory.RunAsync(() => GetBuildInfoTaskAsync(projectUrl, false, cancellationToken)));
227                    }
228                }
229
230                // Check the latest build on the server to the existing build cache
231                // The simple request will limit the load on the Jenkins server
232                // To fetch just new builds is possible too, but it will make the solution more complicated
233                // Similar, the build results could be cached so they are available when switching repos
234                foreach (var info in latestBuildInfos)
235                {
236                    if (!info.Task.IsFaulted)
237                    {
238                        if (info.Join().Timestamp > _lastBuildCache[info.Join().Url].Timestamp)
239                        {
240                            // The cache has at least one newer job, query the status
241                            allBuildInfos.Add(ThreadHelper.JoinableTaskFactory.RunAsync(() => GetBuildInfoTaskAsync(info.Task.CompletedResult().Url, true, cancellationToken)));
242                        }
243                    }
244                }
245
246                if (allBuildInfos.All(t => t.Task.IsCanceled))
247                {
248                    observer.OnCompleted();
249                    return;
250                }
251
252                foreach (var build in allBuildInfos)
253                {
254                    if (build.Task.IsFaulted)
255                    {
256                        Debug.Assert(build.Task.Exception != null, "build.Task.Exception != null");
257
258                        observer.OnError(build.Task.Exception);
259                        continue;
260                    }
261
262                    if (build.Task.IsCanceled || build.Join().Timestamp <= 0)
263                    {
264                        // No valid information received for the build
265                        continue;
266                    }
267
268                    _lastBuildCache[build.Join().Url].Timestamp = build.Join().Timestamp;
269
270                    // Present information in reverse, so the latest job is displayed (i.e. new inprogress on one commit)
271                    // (for multi-branch pipeline, ignore the corner case with multiple branches with inprogress builds on one commit)
272                    foreach (var buildDetails in build.Join().JobDescription.Reverse())
273                    {
274                        if (cancellationToken.IsCancellationRequested)
275                        {
276                            return;
277                        }
278
279                        try
280                        {
281                            var buildInfo = CreateBuildInfo((JObject)buildDetails);
282
283                            if (buildInfo != null)
284                            {
285                                observer.OnNext(buildInfo);
286
287                                if (buildInfo.Status == BuildInfo.BuildStatus.InProgress)
288                                {
289                                    // Need to make a full request next time
290                                    _lastBuildCache[build.Join().Url].Timestamp = 0;
291                                }
292                            }
293                        }
294                        catch
295                        {
296                            // Ignore unexpected responses
297                        }
298                    }
299                }
300
301                // Complete the job, it will be run again with Observe.Retry() (every 10th sec)
302                observer.OnCompleted();
303            }
304            catch (OperationCanceledException)
305            {
306                // Do nothing, the observer is already stopped
307            }
308            catch (Exception ex)
309            {
310                // Cancelling a sub-task is similar to cancelling this task
311                if (!(ex.InnerException is OperationCanceledException))
312                {
313                    observer.OnError(ex);
314                }
315            }
316        }
317
318        private const string _jenkinsTreeBuildInfo = "number,result,timestamp,url,actions[lastBuiltRevision[SHA1,branch[name]],totalCount,failCount,skipCount],building,duration";
319
320        [CanBeNull]
321        private BuildInfo CreateBuildInfo(JObject buildDescription)
322        {
323            var idValue = buildDescription["number"].ToObject<string>();
324            var statusValue = buildDescription["result"].ToObject<string>();
325            var startDateTicks = buildDescription["timestamp"].ToObject<long>();
326            var webUrl = buildDescription["url"].ToObject<string>();
327
328            var action = buildDescription["actions"];
329            var commitHashList = new List<ObjectId>();
330            string testResults = string.Empty;
331            foreach (var element in action)
332            {
333                if (element["lastBuiltRevision"] != null)
334                {
335                    commitHashList.Add(ObjectId.Parse(element["lastBuiltRevision"]["SHA1"].ToObject<string>()));
336                    var branches = element["lastBuiltRevision"]["branch"];
337                    if (_ignoreBuilds != null && branches != null)
338                    {
339                        // Ignore build events for specified branches
340                        foreach (var branch in branches)
341                        {
342                            var name = branch["name"];
343                            if (name != null)
344                            {
345                                var name2 = name.ToObject<string>();
346                                if (name2.IsNotNullOrWhitespace() && _ignoreBuilds.IsMatch(name2))
347                                {
348                                    return null;
349                                }
350                            }
351                        }
352                    }
353                }
354
355                if (element["totalCount"] != null)
356                {
357                    int testCount = element["totalCount"].ToObject<int>();
358                    if (testCount != 0)
359                    {
360                        int failedTestCount = element["failCount"].ToObject<int>();
361                        int skippedTestCount = element["skipCount"].ToObject<int>();
362                        testResults = $"{testCount} tests ({failedTestCount} failed, {skippedTestCount} skipped)";
363                    }
364                }
365            }
366
367            var isRunning = buildDescription["building"].ToObject<bool>();
368            long? buildDuration;
369            if (isRunning)
370            {
371                buildDuration = null;
372            }
373            else
374            {
375                buildDuration = buildDescription["duration"].ToObject<long>();
376            }
377
378            var status = isRunning ? BuildInfo.BuildStatus.InProgress : ParseBuildStatus(statusValue);
379            var statusText = status.ToString("G");
380            var buildInfo = new BuildInfo
381            {
382                Id = idValue,
383                StartDate = TimestampToDateTime(startDateTicks),
384                Duration = buildDuration,
385                Status = status,
386                CommitHashList = commitHashList.ToArray(),
387                Url = webUrl
388            };
389            var durationText = _buildDurationFormatter.Format(buildInfo.Duration);
390            buildInfo.Description = $"#{idValue} {durationText} {testResults} {statusText}";
391            return buildInfo;
392        }
393
394        public static DateTime TimestampToDateTime(long timestamp)
395        {
396            return new DateTime(1970, 1, 1, 0, 0, 0, DateTime.Now.Kind).AddMilliseconds(timestamp);
397        }
398
399        private static AuthenticationHeaderValue CreateBasicHeader(string username, string password)
400        {
401            byte[] byteArray = Encoding.UTF8.GetBytes(string.Format("{0}:{1}", username, password));
402            return new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));
403        }
404
405        private static BuildInfo.BuildStatus ParseBuildStatus(string statusValue)
406        {
407            switch (statusValue)
408            {
409                case "SUCCESS":
410                    return BuildInfo.BuildStatus.Success;
411                case "FAILURE":
412                    return BuildInfo.BuildStatus.Failure;
413                case "UNSTABLE":
414                    return BuildInfo.BuildStatus.Unstable;
415                case "ABORTED":
416                    return BuildInfo.BuildStatus.Stopped;
417                default:
418                    return BuildInfo.BuildStatus.Unknown;
419            }
420        }
421
422        private async Task<Stream> GetStreamAsync(string restServicePath, CancellationToken cancellationToken)
423        {
424            cancellationToken.ThrowIfCancellationRequested();
425
426            var response = await _httpClient.GetAsync(restServicePath, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
427
428            return await GetStreamFromHttpResponseAsync(response);
429
430            async Task<Stream> GetStreamFromHttpResponseAsync(HttpResponseMessage resp)
431            {
432                bool unauthorized = resp.StatusCode == HttpStatusCode.Unauthorized;
433
434                if (resp.IsSuccessStatusCode)
435                {
436                    var httpContent = resp.Content;
437
438                    if (httpContent.Headers.ContentType.MediaType == "text/html")
439                    {
440                        // Jenkins responds with an HTML login page when guest access is denied.
441                        unauthorized = true;
442                    }
443                    else
444                    {
445                        return await httpContent.ReadAsStreamAsync();
446                    }
447                }
448                else if (resp.StatusCode == HttpStatusCode.NotFound)
449                {
450                    // The url does not exist, no jobs to retrieve
451                    return null;
452                }
453                else if (resp.StatusCode == HttpStatusCode.Forbidden)
454                {
455                    unauthorized = true;
456                }
457
458                if (unauthorized)
459                {
460                    var buildServerCredentials = _buildServerWatcher.GetBuildServerCredentials(this, false);
461
462                    if (buildServerCredentials != null)
463                    {
464                        UpdateHttpClientOptions(buildServerCredentials);
465
466                        return await GetStreamAsync(restServicePath, cancellationToken);
467                    }
468
469                    throw new OperationCanceledException(resp.ReasonPhrase);
470                }
471
472                throw new HttpRequestException(resp.ReasonPhrase);
473            }
474        }
475
476        private void UpdateHttpClientOptions(IBuildServerCredentials buildServerCredentials)
477        {
478            var useGuestAccess = buildServerCredentials == null || buildServerCredentials.UseGuestAccess;
479
480            _httpClient.DefaultRequestHeaders.Authorization = useGuestAccess
481                ? null : CreateBasicHeader(buildServerCredentials.Username, buildServerCredentials.Password);
482        }
483
484        private async Task<string> GetResponseAsync(string relativePath, CancellationToken cancellationToken)
485        {
486            using (var responseStream = await GetStreamAsync(relativePath, cancellationToken).ConfigureAwait(false))
487            {
488                using (var reader = new StreamReader(responseStream))
489                {
490                    return await reader.ReadToEndAsync();
491                }
492            }
493        }
494
495        private static string FormatToGetJson(string restServicePath, bool buildsInfo = false)
496        {
497            string buildTree = "lastBuild[timestamp]";
498            int depth = 1;
499            int postIndex = restServicePath.IndexOf('?');
500            if (postIndex >= 0)
501            {
502                int endLen = restServicePath.Length - postIndex;
503                if (restServicePath.EndsWith("/"))
504                {
505                    endLen--;
506                }
507
508                string post = restServicePath.Substring(postIndex, endLen);
509                if (post == "?m")
510                {
511                    // Multi pipeline project
512                    buildTree = "jobs[" + buildTree;
513                    if (buildsInfo)
514                    {
515                        depth = 2;
516                        buildTree += ",builds[" + _jenkinsTreeBuildInfo + "]";
517                    }
518
519                    buildTree += "]";
520                }
521                else
522                {
523                    // user defined format (will likely require changes in the code)
524                    buildTree = post;
525                }
526
527                restServicePath = restServicePath.Substring(0, postIndex);
528            }
529            else
530            {
531                // Freestyle project
532                if (buildsInfo)
533                {
534                    buildTree += ",builds[" + _jenkinsTreeBuildInfo + "]";
535                }
536            }
537
538            if (!restServicePath.EndsWith("/"))
539            {
540                restServicePath += "/";
541            }
542
543            restServicePath += "api/json?depth=" + depth + "&tree=" + buildTree;
544            return restServicePath;
545        }
546
547        public void Dispose()
548        {
549            GC.SuppressFinalize(this);
550
551            _httpClient?.Dispose();
552        }
553    }
554}