/eden/mononoke/lfs_import_lib/src/lib.rs

https://github.com/facebookexperimental/eden · Rust · 130 lines · 112 code · 9 blank · 9 comment · 4 complexity · 48968556c9782442acce2deb0cefd48c MD5 · raw file

  1. /*
  2. * Copyright (c) Facebook, Inc. and its affiliates.
  3. *
  4. * This software may be used and distributed according to the terms of the
  5. * GNU General Public License version 2.
  6. */
  7. use anyhow::{Error, Result};
  8. use blobrepo::BlobRepo;
  9. use bytes::Bytes;
  10. use cloned::cloned;
  11. use context::CoreContext;
  12. use filestore::{self, Alias, FetchKey, StoreRequest};
  13. use futures::{
  14. future::{loop_fn, Loop},
  15. Future, IntoFuture, Stream,
  16. };
  17. use futures_ext::{BoxFuture, FutureExt};
  18. use mercurial_types::blobs::LFSContent;
  19. use mononoke_types::ContentMetadata;
  20. use slog::info;
  21. use std::io::BufReader;
  22. use std::process::{Command, Stdio};
  23. use tokio::codec;
  24. use tokio_process::{Child, CommandExt};
  25. fn lfs_stream(
  26. lfs_helper: &str,
  27. lfs: &LFSContent,
  28. ) -> Result<(Child, impl Stream<Item = Bytes, Error = Error>)> {
  29. let cmd = Command::new(lfs_helper)
  30. .arg(format!("{}", lfs.oid().to_hex()))
  31. .arg(format!("{}", lfs.size()))
  32. .stdin(Stdio::null())
  33. .stdout(Stdio::piped())
  34. .spawn_async();
  35. cmd.map_err(|e| Error::new(e).context(format!("While starting lfs_helper: {:?}", lfs_helper)))
  36. .map(|mut cmd| {
  37. let stdout = cmd.stdout().take().expect("stdout was missing");
  38. let stdout = BufReader::new(stdout);
  39. let stream = codec::FramedRead::new(stdout, codec::BytesCodec::new())
  40. .map(|bytes_mut| bytes_ext::copy_from_old(bytes_mut.freeze()))
  41. .from_err();
  42. (cmd, stream)
  43. })
  44. }
  45. fn do_lfs_upload(
  46. ctx: CoreContext,
  47. blobrepo: BlobRepo,
  48. lfs_helper: String,
  49. lfs: LFSContent,
  50. ) -> BoxFuture<ContentMetadata, Error> {
  51. let blobstore = blobrepo.get_blobstore();
  52. filestore::get_metadata(
  53. &blobstore,
  54. ctx.clone(),
  55. &FetchKey::Aliased(Alias::Sha256(lfs.oid())),
  56. )
  57. .and_then({
  58. move |metadata| match metadata {
  59. Some(metadata) => {
  60. info!(
  61. ctx.logger(),
  62. "lfs_upload: reusing blob {:?}", metadata.sha256
  63. );
  64. Ok(metadata).into_future()
  65. }
  66. .left_future(),
  67. None => {
  68. info!(ctx.logger(), "lfs_upload: importing blob {:?}", lfs.oid());
  69. let req = StoreRequest::with_sha256(lfs.size(), lfs.oid());
  70. lfs_stream(&lfs_helper, &lfs)
  71. .into_future()
  72. .and_then(move |(child, stream)| {
  73. let upload_fut = filestore::store(
  74. blobrepo.get_blobstore(),
  75. blobrepo.filestore_config(),
  76. ctx.clone(),
  77. &req,
  78. stream,
  79. );
  80. // NOTE: We ignore the child exit code here. Since the Filestore validates the object
  81. // we're uploading by SHA256, that's indeed fine (it doesn't matter if the Child failed
  82. // if it gave us exactly the content we wanted).
  83. (upload_fut, child.from_err()).into_future().map({
  84. cloned!(ctx);
  85. move |(meta, _)| {
  86. info!(ctx.logger(), "lfs_upload: imported blob {:?}", meta.sha256);
  87. meta
  88. }
  89. })
  90. })
  91. }
  92. .right_future(),
  93. }
  94. })
  95. .boxify()
  96. }
  97. pub fn lfs_upload(
  98. ctx: CoreContext,
  99. blobrepo: BlobRepo,
  100. lfs_helper: String,
  101. lfs: LFSContent,
  102. ) -> impl Future<Item = ContentMetadata, Error = Error> {
  103. let max_attempts = 5;
  104. loop_fn(0, move |i| {
  105. do_lfs_upload(
  106. ctx.clone(),
  107. blobrepo.clone(),
  108. lfs_helper.clone(),
  109. lfs.clone(),
  110. )
  111. .then(move |r| {
  112. let loop_state = if r.is_ok() || i > max_attempts {
  113. Loop::Break(r)
  114. } else {
  115. Loop::Continue(i + 1)
  116. };
  117. Ok(loop_state)
  118. })
  119. })
  120. .and_then(|r| r)
  121. }