PageRenderTime 59ms CodeModel.GetById 26ms RepoModel.GetById 0ms app.codeStats 0ms

/Plugins/BuildServerIntegration/JenkinsIntegration/JenkinsAdapter.cs

https://github.com/PKRoma/gitextensions
C# | 624 lines | 496 code | 87 blank | 41 comment | 68 complexity | 21aff29be91ef33fba8bd4dcd0b1b14d MD5 | raw file
Possible License(s): GPL-3.0
  1. using System;
  2. using System.Collections.Generic;
  3. using System.ComponentModel.Composition;
  4. using System.Diagnostics;
  5. using System.IO;
  6. using System.Linq;
  7. using System.Net;
  8. using System.Net.Http;
  9. using System.Net.Http.Headers;
  10. using System.Reactive.Concurrency;
  11. using System.Reactive.Linq;
  12. using System.Text;
  13. using System.Text.RegularExpressions;
  14. using System.Threading;
  15. using System.Threading.Tasks;
  16. using GitCommands.Utils;
  17. using GitUI;
  18. using GitUIPluginInterfaces;
  19. using GitUIPluginInterfaces.BuildServerIntegration;
  20. using Microsoft;
  21. using Microsoft.VisualStudio.Threading;
  22. using Newtonsoft.Json.Linq;
  23. namespace JenkinsIntegration
  24. {
  25. [MetadataAttribute]
  26. [AttributeUsage(AttributeTargets.Class)]
  27. public class JenkinsIntegrationMetadata : BuildServerAdapterMetadataAttribute
  28. {
  29. public JenkinsIntegrationMetadata(string buildServerType)
  30. : base(buildServerType)
  31. {
  32. }
  33. public override string? CanBeLoaded
  34. {
  35. get
  36. {
  37. if (EnvUtils.IsNet4FullOrHigher())
  38. {
  39. return null;
  40. }
  41. return ".NET Framework 4 or higher required";
  42. }
  43. }
  44. }
  45. [Export(typeof(IBuildServerAdapter))]
  46. [JenkinsIntegrationMetadata(PluginName)]
  47. [PartCreationPolicy(CreationPolicy.NonShared)]
  48. internal class JenkinsAdapter : IBuildServerAdapter
  49. {
  50. public const string PluginName = "Jenkins";
  51. private static readonly IBuildDurationFormatter _buildDurationFormatter = new BuildDurationFormatter();
  52. private IBuildServerWatcher? _buildServerWatcher;
  53. private HttpClient? _httpClient;
  54. // last known build per project
  55. private readonly Dictionary<string, long> _lastProjectBuildTime = new();
  56. private Regex? _ignoreBuilds;
  57. public void Initialize(IBuildServerWatcher buildServerWatcher, ISettingsSource config, Action openSettings, Func<ObjectId, bool>? isCommitInRevisionGrid = null)
  58. {
  59. if (_buildServerWatcher is not null)
  60. {
  61. throw new InvalidOperationException("Already initialized");
  62. }
  63. _buildServerWatcher = buildServerWatcher;
  64. var projectName = config.GetString("ProjectName", null);
  65. var hostName = config.GetString("BuildServerUrl", null);
  66. if (!string.IsNullOrEmpty(hostName) && !string.IsNullOrEmpty(projectName))
  67. {
  68. var baseAddress = hostName.Contains("://")
  69. ? new Uri(hostName, UriKind.Absolute)
  70. : new Uri($"{Uri.UriSchemeHttp}://{hostName}:8080", UriKind.Absolute);
  71. _httpClient = new HttpClient(new HttpClientHandler { UseDefaultCredentials = true })
  72. {
  73. Timeout = TimeSpan.FromMinutes(2),
  74. BaseAddress = baseAddress
  75. };
  76. var buildServerCredentials = buildServerWatcher.GetBuildServerCredentials(this, true);
  77. UpdateHttpClientOptions(buildServerCredentials);
  78. string[] projectUrls = _buildServerWatcher.ReplaceVariables(projectName)
  79. .Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
  80. foreach (var projectUrl in projectUrls.Select(s => baseAddress + "job/" + s.Trim() + "/"))
  81. {
  82. _lastProjectBuildTime[projectUrl] = -1;
  83. }
  84. }
  85. var ignoreBuilds = config.GetString("IgnoreBuildBranch", string.Empty);
  86. _ignoreBuilds = !string.IsNullOrWhiteSpace(ignoreBuilds) ? new Regex(ignoreBuilds) : null;
  87. }
  88. /// <summary>
  89. /// Gets a unique key which identifies this build server.
  90. /// </summary>
  91. public string UniqueKey
  92. {
  93. get
  94. {
  95. Validates.NotNull(_httpClient);
  96. return _httpClient.BaseAddress.Host;
  97. }
  98. }
  99. private class ResponseInfo
  100. {
  101. public string? Url { get; set; }
  102. public long Timestamp { get; set; }
  103. public IEnumerable<JToken>? JobDescription { get; set; }
  104. }
  105. private async Task<ResponseInfo> GetBuildInfoTaskAsync(string projectUrl, bool fullInfo, CancellationToken cancellationToken)
  106. {
  107. string? t;
  108. long timestamp = 0;
  109. IEnumerable<JToken> s = Enumerable.Empty<JToken>();
  110. try
  111. {
  112. t = await GetResponseAsync(FormatToGetJson(projectUrl, fullInfo), cancellationToken).ConfigureAwait(false);
  113. }
  114. catch
  115. {
  116. // Could be cancelled or failed. Explicitly assign 't' to reveal the intended behavior of following code
  117. // for this case.
  118. t = null;
  119. }
  120. if (!string.IsNullOrWhiteSpace(t) && !cancellationToken.IsCancellationRequested)
  121. {
  122. JObject jobDescription = JObject.Parse(t);
  123. if (jobDescription["builds"] is not null)
  124. {
  125. // Freestyle jobs
  126. s = jobDescription["builds"];
  127. }
  128. else if (jobDescription["jobs"] is not null)
  129. {
  130. // Multi-branch pipeline
  131. s = jobDescription["jobs"]
  132. .SelectMany(j => j["builds"]);
  133. foreach (var j in jobDescription["jobs"])
  134. {
  135. try
  136. {
  137. if (j["lastBuild"]?["timestamp"] is null)
  138. {
  139. continue;
  140. }
  141. var ts = j["lastBuild"]["timestamp"];
  142. timestamp = Math.Max(timestamp, ts.ToObject<long>());
  143. }
  144. catch
  145. {
  146. // Ignore malformed build ids
  147. }
  148. }
  149. }
  150. // else: The server had no response (overloaded?) or a multi-branch pipeline is not configured
  151. if (timestamp == 0 && jobDescription["lastBuild"]?["timestamp"] is not null)
  152. {
  153. timestamp = jobDescription["lastBuild"]["timestamp"].ToObject<long>();
  154. }
  155. }
  156. return new ResponseInfo
  157. {
  158. Url = projectUrl,
  159. Timestamp = timestamp,
  160. JobDescription = s
  161. };
  162. }
  163. public IObservable<BuildInfo> GetFinishedBuildsSince(IScheduler scheduler, DateTime? sinceDate = null)
  164. {
  165. // GetBuilds() will return the same builds as for GetRunningBuilds().
  166. // Multiple calls will fetch same info multiple times and make debugging very confusing
  167. // Similar as for AppVeyor
  168. // 'sinceDate' is not supported in Jenkins API
  169. return Observable.Empty<BuildInfo>();
  170. }
  171. public IObservable<BuildInfo> GetRunningBuilds(IScheduler scheduler)
  172. {
  173. return GetBuilds(scheduler);
  174. }
  175. private IObservable<BuildInfo> GetBuilds(IScheduler scheduler)
  176. {
  177. return Observable.Create<BuildInfo>((observer, cancellationToken) =>
  178. ThreadHelper.JoinableTaskFactory.RunAsync(async () =>
  179. {
  180. await TaskScheduler.Default;
  181. return scheduler.Schedule(() => ObserveBuilds(observer, cancellationToken));
  182. }).Task);
  183. }
  184. private void ObserveBuilds(IObserver<BuildInfo> observer, CancellationToken cancellationToken)
  185. {
  186. try
  187. {
  188. List<JoinableTask<ResponseInfo>> allBuildInfos = new();
  189. List<JoinableTask<ResponseInfo>> latestBuildInfos = new();
  190. foreach (var projectUrl in _lastProjectBuildTime.Keys)
  191. {
  192. if (_lastProjectBuildTime[projectUrl] <= 0)
  193. {
  194. // This project must be updated, no need to to check the latest builds
  195. allBuildInfos.Add(ThreadHelper.JoinableTaskFactory.RunAsync(() => GetBuildInfoTaskAsync(projectUrl, true, cancellationToken)));
  196. }
  197. else
  198. {
  199. latestBuildInfos.Add(ThreadHelper.JoinableTaskFactory.RunAsync(() => GetBuildInfoTaskAsync(projectUrl, false, cancellationToken)));
  200. }
  201. }
  202. // Check the latest build on the server to the existing build timestamp
  203. // The simple request will limit the load on the Jenkins server
  204. // To fetch just new builds is possible too, but it will make the solution more complicated
  205. // Similar, the build results could be cached so they are available when switching repos
  206. foreach (var info in latestBuildInfos.Where(info => !info.Task.IsFaulted))
  207. {
  208. ResponseInfo responseInfo = info.Join();
  209. Validates.NotNull(responseInfo.Url);
  210. if (responseInfo.Timestamp > _lastProjectBuildTime[responseInfo.Url])
  211. {
  212. // The info has at least one newer job, query the status
  213. allBuildInfos.Add(ThreadHelper.JoinableTaskFactory.RunAsync(() =>
  214. GetBuildInfoTaskAsync(responseInfo.Url, true, cancellationToken)));
  215. }
  216. }
  217. if (allBuildInfos.All(t => t.Task.IsCanceled))
  218. {
  219. observer.OnCompleted();
  220. return;
  221. }
  222. Dictionary<ObjectId, BuildInfo.BuildStatus> builds = new();
  223. foreach (var build in allBuildInfos)
  224. {
  225. if (build.Task.IsFaulted)
  226. {
  227. Debug.Assert(build.Task.Exception is not null, "build.Task.Exception is not null");
  228. observer.OnError(build.Task.Exception);
  229. continue;
  230. }
  231. var buildResponse = build.Join(cancellationToken);
  232. if (build.Task.IsCanceled || buildResponse.Timestamp <= 0)
  233. {
  234. // No valid information received for the build
  235. continue;
  236. }
  237. Validates.NotNull(buildResponse.Url);
  238. Validates.NotNull(buildResponse.JobDescription);
  239. _lastProjectBuildTime[buildResponse.Url] = buildResponse.Timestamp;
  240. foreach (var buildDetails in buildResponse.JobDescription)
  241. {
  242. if (cancellationToken.IsCancellationRequested)
  243. {
  244. return;
  245. }
  246. try
  247. {
  248. var buildInfo = CreateBuildInfo((JObject)buildDetails);
  249. if (buildInfo is null)
  250. {
  251. continue;
  252. }
  253. if (buildInfo.Status == BuildInfo.BuildStatus.InProgress)
  254. {
  255. // Need to make a full request next time
  256. _lastProjectBuildTime[buildResponse.Url] = 0;
  257. }
  258. // Normally, there is only one commit per Jenkins build
  259. // builds listed with latest first, likely most interesting
  260. // Update the observer if at least one of them is better (cannot be selective)
  261. var isBetter = false;
  262. foreach (var commit in buildInfo.CommitHashList)
  263. {
  264. if (!StatusIsBetter(buildInfo.Status, commit, builds))
  265. {
  266. continue;
  267. }
  268. isBetter = true;
  269. builds[commit] = buildInfo.Status;
  270. }
  271. if (isBetter)
  272. {
  273. observer.OnNext(buildInfo);
  274. }
  275. }
  276. catch
  277. {
  278. // Ignore unexpected responses
  279. }
  280. }
  281. }
  282. // Complete the job, it will be run again with Observe.Retry() (every 10th sec)
  283. observer.OnCompleted();
  284. }
  285. catch (OperationCanceledException)
  286. {
  287. // Do nothing, the observer is already stopped
  288. }
  289. catch (Exception ex)
  290. {
  291. // Cancelling a sub-task is similar to cancelling this task
  292. if (!(ex.InnerException is OperationCanceledException))
  293. {
  294. observer.OnError(ex);
  295. }
  296. }
  297. return;
  298. // Priority: Completed > InProgress > Aborted/Stopped > None
  299. static bool StatusIsBetter(BuildInfo.BuildStatus newStatus, ObjectId commit, Dictionary<ObjectId, BuildInfo.BuildStatus> builds)
  300. {
  301. // No existing status
  302. if (!builds.ContainsKey(commit))
  303. {
  304. return true;
  305. }
  306. var existingStatus = builds[commit];
  307. // Completed status is never replaced
  308. if (IsBuildCompleted(existingStatus))
  309. {
  310. return false;
  311. }
  312. // Existing status is lower than Completed, new complete replaces
  313. if (IsBuildCompleted(newStatus))
  314. {
  315. return true;
  316. }
  317. // Replace existing aborted/stopped if new is InProgress
  318. return existingStatus != BuildInfo.BuildStatus.InProgress
  319. && newStatus == BuildInfo.BuildStatus.InProgress;
  320. }
  321. static bool IsBuildCompleted(BuildInfo.BuildStatus status)
  322. {
  323. return status == BuildInfo.BuildStatus.Success
  324. || status == BuildInfo.BuildStatus.Failure
  325. || status == BuildInfo.BuildStatus.Unstable;
  326. }
  327. }
  328. private const string _jenkinsTreeBuildInfo = "number,result,timestamp,url,actions[lastBuiltRevision[SHA1,branch[name]],totalCount,failCount,skipCount],building,duration";
  329. private BuildInfo? CreateBuildInfo(JObject buildDescription)
  330. {
  331. var idValue = buildDescription["number"].ToObject<string>();
  332. var statusValue = buildDescription["result"].ToObject<string>();
  333. var startDateTicks = buildDescription["timestamp"].ToObject<long>();
  334. var webUrl = buildDescription["url"].ToObject<string>();
  335. var action = buildDescription["actions"];
  336. List<ObjectId> commitHashList = new();
  337. string testResults = string.Empty;
  338. foreach (var element in action)
  339. {
  340. if (element["lastBuiltRevision"] is not null)
  341. {
  342. commitHashList.Add(ObjectId.Parse(element["lastBuiltRevision"]["SHA1"].ToObject<string>()));
  343. var branches = element["lastBuiltRevision"]["branch"];
  344. if (_ignoreBuilds is not null && branches is not null)
  345. {
  346. // Ignore build events for specified branches
  347. foreach (var branch in branches)
  348. {
  349. var name = branch["name"];
  350. if (name is null)
  351. {
  352. continue;
  353. }
  354. var name2 = name.ToObject<string>();
  355. if (!string.IsNullOrWhiteSpace(name2) && _ignoreBuilds.IsMatch(name2))
  356. {
  357. return null;
  358. }
  359. }
  360. }
  361. }
  362. if (element["totalCount"] is null)
  363. {
  364. continue;
  365. }
  366. int testCount = element["totalCount"].ToObject<int>();
  367. if (testCount != 0)
  368. {
  369. int failedTestCount = element["failCount"].ToObject<int>();
  370. int skippedTestCount = element["skipCount"].ToObject<int>();
  371. testResults = $"{testCount} tests ({failedTestCount} failed, {skippedTestCount} skipped)";
  372. }
  373. }
  374. var isRunning = buildDescription["building"].ToObject<bool>();
  375. long? buildDuration;
  376. if (isRunning)
  377. {
  378. buildDuration = null;
  379. }
  380. else
  381. {
  382. buildDuration = buildDescription["duration"].ToObject<long>();
  383. }
  384. var status = isRunning ? BuildInfo.BuildStatus.InProgress : ParseBuildStatus(statusValue);
  385. var statusText = status.ToString("G");
  386. BuildInfo buildInfo = new()
  387. {
  388. Id = idValue,
  389. StartDate = TimestampToDateTime(startDateTicks),
  390. Duration = buildDuration,
  391. Status = status,
  392. CommitHashList = commitHashList.ToArray(),
  393. Url = webUrl
  394. };
  395. var durationText = _buildDurationFormatter.Format(buildInfo.Duration);
  396. buildInfo.Description = $"#{idValue} {durationText} {testResults} {statusText}";
  397. return buildInfo;
  398. }
  399. private static DateTime TimestampToDateTime(long timestamp)
  400. {
  401. return new DateTime(1970, 1, 1, 0, 0, 0, DateTime.Now.Kind).AddMilliseconds(timestamp);
  402. }
  403. private static AuthenticationHeaderValue CreateBasicHeader(string username, string password)
  404. {
  405. byte[] byteArray = Encoding.UTF8.GetBytes($"{username}:{password}");
  406. return new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));
  407. }
  408. private static BuildInfo.BuildStatus ParseBuildStatus(string statusValue)
  409. {
  410. return statusValue switch
  411. {
  412. "SUCCESS" => BuildInfo.BuildStatus.Success,
  413. "FAILURE" => BuildInfo.BuildStatus.Failure,
  414. "UNSTABLE" => BuildInfo.BuildStatus.Unstable,
  415. "ABORTED" => BuildInfo.BuildStatus.Stopped,
  416. // Jenkins status "NOT_BUILT"
  417. _ => BuildInfo.BuildStatus.Unknown
  418. };
  419. }
  420. private async Task<Stream?> GetStreamAsync(string restServicePath, CancellationToken cancellationToken)
  421. {
  422. cancellationToken.ThrowIfCancellationRequested();
  423. Validates.NotNull(_httpClient);
  424. var response = await _httpClient.GetAsync(restServicePath, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
  425. return await GetStreamFromHttpResponseAsync(response);
  426. async Task<Stream?> GetStreamFromHttpResponseAsync(HttpResponseMessage resp)
  427. {
  428. bool unauthorized = resp.StatusCode == HttpStatusCode.Unauthorized;
  429. if (resp.IsSuccessStatusCode)
  430. {
  431. var httpContent = resp.Content;
  432. if (httpContent.Headers.ContentType.MediaType == "text/html")
  433. {
  434. // Jenkins responds with an HTML login page when guest access is denied.
  435. unauthorized = true;
  436. }
  437. else
  438. {
  439. return await httpContent.ReadAsStreamAsync();
  440. }
  441. }
  442. else if (resp.StatusCode == HttpStatusCode.NotFound)
  443. {
  444. // The url does not exist, no jobs to retrieve
  445. return null;
  446. }
  447. else if (resp.StatusCode == HttpStatusCode.Forbidden)
  448. {
  449. unauthorized = true;
  450. }
  451. if (!unauthorized)
  452. {
  453. throw new HttpRequestException(resp.ReasonPhrase);
  454. }
  455. Validates.NotNull(_buildServerWatcher);
  456. var buildServerCredentials = _buildServerWatcher.GetBuildServerCredentials(this, false);
  457. if (buildServerCredentials is null)
  458. {
  459. throw new OperationCanceledException(resp.ReasonPhrase);
  460. }
  461. UpdateHttpClientOptions(buildServerCredentials);
  462. return await GetStreamAsync(restServicePath, cancellationToken);
  463. }
  464. }
  465. private void UpdateHttpClientOptions(IBuildServerCredentials? buildServerCredentials)
  466. {
  467. Validates.NotNull(_httpClient);
  468. if (buildServerCredentials is null || buildServerCredentials.UseGuestAccess)
  469. {
  470. _httpClient.DefaultRequestHeaders.Authorization = null;
  471. }
  472. else
  473. {
  474. Validates.NotNull(buildServerCredentials.Username);
  475. Validates.NotNull(buildServerCredentials.Password);
  476. _httpClient.DefaultRequestHeaders.Authorization = CreateBasicHeader(buildServerCredentials.Username, buildServerCredentials.Password);
  477. }
  478. }
  479. private async Task<string> GetResponseAsync(string relativePath, CancellationToken cancellationToken)
  480. {
  481. using var responseStream = await GetStreamAsync(relativePath, cancellationToken).ConfigureAwait(false);
  482. using StreamReader reader = new(responseStream);
  483. return await reader.ReadToEndAsync();
  484. }
  485. private static string FormatToGetJson(string restServicePath, bool buildsInfo = false)
  486. {
  487. string buildTree = "lastBuild[timestamp]";
  488. int depth = 1;
  489. int postIndex = restServicePath.IndexOf('?');
  490. if (postIndex >= 0)
  491. {
  492. int endLen = restServicePath.Length - postIndex;
  493. if (restServicePath.EndsWith("/"))
  494. {
  495. endLen--;
  496. }
  497. string post = restServicePath.Substring(postIndex, endLen);
  498. if (post == "?m")
  499. {
  500. // Multi pipeline project
  501. buildTree = "jobs[" + buildTree;
  502. if (buildsInfo)
  503. {
  504. depth = 2;
  505. buildTree += ",builds[" + _jenkinsTreeBuildInfo + "]";
  506. }
  507. buildTree += "]";
  508. }
  509. else
  510. {
  511. // user defined format (will likely require changes in the code)
  512. buildTree = post;
  513. }
  514. restServicePath = restServicePath.Substring(0, postIndex);
  515. }
  516. else
  517. {
  518. // Freestyle project
  519. if (buildsInfo)
  520. {
  521. buildTree += ",builds[" + _jenkinsTreeBuildInfo + "]";
  522. }
  523. }
  524. if (!restServicePath.EndsWith("/"))
  525. {
  526. restServicePath += "/";
  527. }
  528. restServicePath += "api/json?depth=" + depth + "&tree=" + buildTree;
  529. return restServicePath;
  530. }
  531. public void Dispose()
  532. {
  533. GC.SuppressFinalize(this);
  534. _httpClient?.Dispose();
  535. }
  536. }
  537. }