/registry/storage/driver/s3-aws/s3_test.go

https://gitlab.com/vpl/distribution · Go · 238 lines · 198 code · 36 blank · 4 comment · 79 complexity · e98534c3afd7180d8dbb6df7cea995bc MD5 · raw file

  1. package s3
  2. import (
  3. "io/ioutil"
  4. "os"
  5. "strconv"
  6. "testing"
  7. "github.com/aws/aws-sdk-go/aws"
  8. "github.com/aws/aws-sdk-go/service/s3"
  9. "github.com/docker/distribution/context"
  10. storagedriver "github.com/docker/distribution/registry/storage/driver"
  11. "github.com/docker/distribution/registry/storage/driver/testsuites"
  12. "gopkg.in/check.v1"
  13. )
  14. // Hook up gocheck into the "go test" runner.
  15. func Test(t *testing.T) { check.TestingT(t) }
  16. var s3DriverConstructor func(rootDirectory, storageClass string) (*Driver, error)
  17. var skipS3 func() string
  18. func init() {
  19. accessKey := os.Getenv("AWS_ACCESS_KEY")
  20. secretKey := os.Getenv("AWS_SECRET_KEY")
  21. bucket := os.Getenv("S3_BUCKET")
  22. encrypt := os.Getenv("S3_ENCRYPT")
  23. keyID := os.Getenv("S3_KEY_ID")
  24. secure := os.Getenv("S3_SECURE")
  25. region := os.Getenv("AWS_REGION")
  26. root, err := ioutil.TempDir("", "driver-")
  27. regionEndpoint := os.Getenv("REGION_ENDPOINT")
  28. if err != nil {
  29. panic(err)
  30. }
  31. defer os.Remove(root)
  32. s3DriverConstructor = func(rootDirectory, storageClass string) (*Driver, error) {
  33. encryptBool := false
  34. if encrypt != "" {
  35. encryptBool, err = strconv.ParseBool(encrypt)
  36. if err != nil {
  37. return nil, err
  38. }
  39. }
  40. secureBool := true
  41. if secure != "" {
  42. secureBool, err = strconv.ParseBool(secure)
  43. if err != nil {
  44. return nil, err
  45. }
  46. }
  47. parameters := DriverParameters{
  48. accessKey,
  49. secretKey,
  50. bucket,
  51. region,
  52. regionEndpoint,
  53. encryptBool,
  54. keyID,
  55. secureBool,
  56. minChunkSize,
  57. rootDirectory,
  58. storageClass,
  59. driverName + "-test",
  60. }
  61. return New(parameters)
  62. }
  63. // Skip S3 storage driver tests if environment variable parameters are not provided
  64. skipS3 = func() string {
  65. if accessKey == "" || secretKey == "" || region == "" || bucket == "" || encrypt == "" {
  66. return "Must set AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, S3_BUCKET, and S3_ENCRYPT to run S3 tests"
  67. }
  68. return ""
  69. }
  70. testsuites.RegisterSuite(func() (storagedriver.StorageDriver, error) {
  71. return s3DriverConstructor(root, s3.StorageClassStandard)
  72. }, skipS3)
  73. }
  74. func TestEmptyRootList(t *testing.T) {
  75. if skipS3() != "" {
  76. t.Skip(skipS3())
  77. }
  78. validRoot, err := ioutil.TempDir("", "driver-")
  79. if err != nil {
  80. t.Fatalf("unexpected error creating temporary directory: %v", err)
  81. }
  82. defer os.Remove(validRoot)
  83. rootedDriver, err := s3DriverConstructor(validRoot, s3.StorageClassStandard)
  84. if err != nil {
  85. t.Fatalf("unexpected error creating rooted driver: %v", err)
  86. }
  87. emptyRootDriver, err := s3DriverConstructor("", s3.StorageClassStandard)
  88. if err != nil {
  89. t.Fatalf("unexpected error creating empty root driver: %v", err)
  90. }
  91. slashRootDriver, err := s3DriverConstructor("/", s3.StorageClassStandard)
  92. if err != nil {
  93. t.Fatalf("unexpected error creating slash root driver: %v", err)
  94. }
  95. filename := "/test"
  96. contents := []byte("contents")
  97. ctx := context.Background()
  98. err = rootedDriver.PutContent(ctx, filename, contents)
  99. if err != nil {
  100. t.Fatalf("unexpected error creating content: %v", err)
  101. }
  102. defer rootedDriver.Delete(ctx, filename)
  103. keys, err := emptyRootDriver.List(ctx, "/")
  104. for _, path := range keys {
  105. if !storagedriver.PathRegexp.MatchString(path) {
  106. t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
  107. }
  108. }
  109. keys, err = slashRootDriver.List(ctx, "/")
  110. for _, path := range keys {
  111. if !storagedriver.PathRegexp.MatchString(path) {
  112. t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
  113. }
  114. }
  115. }
  116. func TestStorageClass(t *testing.T) {
  117. if skipS3() != "" {
  118. t.Skip(skipS3())
  119. }
  120. rootDir, err := ioutil.TempDir("", "driver-")
  121. if err != nil {
  122. t.Fatalf("unexpected error creating temporary directory: %v", err)
  123. }
  124. defer os.Remove(rootDir)
  125. standardDriver, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
  126. if err != nil {
  127. t.Fatalf("unexpected error creating driver with standard storage: %v", err)
  128. }
  129. rrDriver, err := s3DriverConstructor(rootDir, s3.StorageClassReducedRedundancy)
  130. if err != nil {
  131. t.Fatalf("unexpected error creating driver with reduced redundancy storage: %v", err)
  132. }
  133. standardFilename := "/test-standard"
  134. rrFilename := "/test-rr"
  135. contents := []byte("contents")
  136. ctx := context.Background()
  137. err = standardDriver.PutContent(ctx, standardFilename, contents)
  138. if err != nil {
  139. t.Fatalf("unexpected error creating content: %v", err)
  140. }
  141. defer standardDriver.Delete(ctx, standardFilename)
  142. err = rrDriver.PutContent(ctx, rrFilename, contents)
  143. if err != nil {
  144. t.Fatalf("unexpected error creating content: %v", err)
  145. }
  146. defer rrDriver.Delete(ctx, rrFilename)
  147. standardDriverUnwrapped := standardDriver.Base.StorageDriver.(*driver)
  148. resp, err := standardDriverUnwrapped.S3.GetObject(&s3.GetObjectInput{
  149. Bucket: aws.String(standardDriverUnwrapped.Bucket),
  150. Key: aws.String(standardDriverUnwrapped.s3Path(standardFilename)),
  151. })
  152. if err != nil {
  153. t.Fatalf("unexpected error retrieving standard storage file: %v", err)
  154. }
  155. defer resp.Body.Close()
  156. // Amazon only populates this header value for non-standard storage classes
  157. if resp.StorageClass != nil {
  158. t.Fatalf("unexpected storage class for standard file: %v", resp.StorageClass)
  159. }
  160. rrDriverUnwrapped := rrDriver.Base.StorageDriver.(*driver)
  161. resp, err = rrDriverUnwrapped.S3.GetObject(&s3.GetObjectInput{
  162. Bucket: aws.String(rrDriverUnwrapped.Bucket),
  163. Key: aws.String(rrDriverUnwrapped.s3Path(rrFilename)),
  164. })
  165. if err != nil {
  166. t.Fatalf("unexpected error retrieving reduced-redundancy storage file: %v", err)
  167. }
  168. defer resp.Body.Close()
  169. if resp.StorageClass == nil {
  170. t.Fatalf("unexpected storage class for reduced-redundancy file: %v", s3.StorageClassStandard)
  171. } else if *resp.StorageClass != s3.StorageClassReducedRedundancy {
  172. t.Fatalf("unexpected storage class for reduced-redundancy file: %v", *resp.StorageClass)
  173. }
  174. }
  175. func TestOverThousandBlobs(t *testing.T) {
  176. if skipS3() != "" {
  177. t.Skip(skipS3())
  178. }
  179. rootDir, err := ioutil.TempDir("", "driver-")
  180. if err != nil {
  181. t.Fatalf("unexpected error creating temporary directory: %v", err)
  182. }
  183. defer os.Remove(rootDir)
  184. standardDriver, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
  185. if err != nil {
  186. t.Fatalf("unexpected error creating driver with standard storage: %v", err)
  187. }
  188. ctx := context.Background()
  189. for i := 0; i < 1005; i++ {
  190. filename := "/thousandfiletest/file" + strconv.Itoa(i)
  191. contents := []byte("contents")
  192. err = standardDriver.PutContent(ctx, filename, contents)
  193. if err != nil {
  194. t.Fatalf("unexpected error creating content: %v", err)
  195. }
  196. }
  197. // cant actually verify deletion because read-after-delete is inconsistent, but can ensure no errors
  198. err = standardDriver.Delete(ctx, "/thousandfiletest")
  199. if err != nil {
  200. t.Fatalf("unexpected error deleting thousand files: %v", err)
  201. }
  202. }