PageRenderTime 33ms CodeModel.GetById 12ms RepoModel.GetById 0ms app.codeStats 1ms

/apis/s3/src/test/java/org/jclouds/s3/S3ClientLiveTest.java

https://github.com/regularfry/jclouds
Java | 501 lines | 384 code | 78 blank | 39 comment | 8 complexity | 4970533158065089ad0339b7a52803d2 MD5 | raw file
  1. /**
  2. * Licensed to jclouds, Inc. (jclouds) under one or more
  3. * contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. jclouds licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing,
  13. * software distributed under the License is distributed on an
  14. * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  15. * KIND, either express or implied. See the License for the
  16. * specific language governing permissions and limitations
  17. * under the License.
  18. */
  19. package org.jclouds.s3;
  20. import static org.jclouds.s3.internal.StubS3AsyncClient.TEST_ACL_EMAIL;
  21. import static org.jclouds.s3.internal.StubS3AsyncClient.TEST_ACL_ID;
  22. import static org.jclouds.s3.options.CopyObjectOptions.Builder.ifSourceETagDoesntMatch;
  23. import static org.jclouds.s3.options.CopyObjectOptions.Builder.ifSourceETagMatches;
  24. import static org.jclouds.s3.options.CopyObjectOptions.Builder.ifSourceModifiedSince;
  25. import static org.jclouds.s3.options.CopyObjectOptions.Builder.ifSourceUnmodifiedSince;
  26. import static org.jclouds.s3.options.CopyObjectOptions.Builder.overrideAcl;
  27. import static org.jclouds.s3.options.CopyObjectOptions.Builder.overrideMetadataWith;
  28. import static org.jclouds.s3.options.PutObjectOptions.Builder.withAcl;
  29. import static org.testng.Assert.assertEquals;
  30. import static org.testng.Assert.assertFalse;
  31. import static org.testng.Assert.assertTrue;
  32. import java.io.IOException;
  33. import java.net.URL;
  34. import java.util.Date;
  35. import java.util.Map;
  36. import java.util.concurrent.ExecutionException;
  37. import java.util.concurrent.TimeoutException;
  38. import org.jclouds.s3.domain.AccessControlList;
  39. import org.jclouds.s3.domain.CannedAccessPolicy;
  40. import org.jclouds.s3.domain.ObjectMetadata;
  41. import org.jclouds.s3.domain.S3Object;
  42. import org.jclouds.s3.domain.AccessControlList.CanonicalUserGrantee;
  43. import org.jclouds.s3.domain.AccessControlList.EmailAddressGrantee;
  44. import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI;
  45. import org.jclouds.s3.domain.AccessControlList.Permission;
  46. import org.jclouds.s3.options.PutObjectOptions;
  47. import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest;
  48. import org.jclouds.http.HttpResponseException;
  49. import org.jclouds.util.Strings2;
  50. import org.testng.annotations.Test;
  51. import com.google.common.base.Throwables;
  52. import com.google.common.collect.Maps;
  53. /**
  54. *
  55. * @author James Murty
  56. * @author Adrian Cole
  57. */
  58. @Test(groups = { "integration", "live" })
  59. public class S3ClientLiveTest extends BaseBlobStoreIntegrationTest {
  60. public S3Client getApi() {
  61. return (S3Client) context.getProviderSpecificContext().getApi();
  62. }
  63. /**
  64. * this method overrides containerName to ensure it isn't found
  65. */
  66. @Test(groups = { "integration", "live" })
  67. public void deleteContainerIfEmptyNotFound() throws Exception {
  68. assert getApi().deleteBucketIfEmpty("dbienf");
  69. }
  70. @Test(groups = { "integration", "live" })
  71. public void deleteContainerIfEmptyButHasContents() throws Exception {
  72. String containerName = getContainerName();
  73. try {
  74. addBlobToContainer(containerName, "test");
  75. assert !getApi().deleteBucketIfEmpty(containerName);
  76. } finally {
  77. returnContainer(containerName);
  78. }
  79. }
  80. protected URL getObjectURL(String containerName, String key) throws Exception {
  81. URL url = new URL(String.format("http://%s.%s/%s", containerName, context.getProviderSpecificContext()
  82. .getEndpoint().getHost(), key));
  83. return url;
  84. }
  85. public void testPutCannedAccessPolicyPublic() throws Exception {
  86. String containerName = getContainerName();
  87. try {
  88. String key = "hello";
  89. S3Object object = getApi().newS3Object();
  90. object.getMetadata().setKey(key);
  91. object.setPayload(TEST_STRING);
  92. getApi().putObject(containerName, object,
  93. withAcl(CannedAccessPolicy.PUBLIC_READ));
  94. URL url = this.getObjectURL(containerName, key);
  95. Strings2.toStringAndClose(url.openStream());
  96. } finally {
  97. returnContainer(containerName);
  98. }
  99. }
  100. public void testCopyCannedAccessPolicyPublic() throws Exception {
  101. String containerName = getContainerName();
  102. String destinationContainer = getContainerName();
  103. try {
  104. addBlobToContainer(containerName, sourceKey);
  105. validateContent(containerName, sourceKey);
  106. getApi().copyObject(containerName, sourceKey, destinationContainer, destinationKey,
  107. overrideAcl(CannedAccessPolicy.PUBLIC_READ));
  108. validateContent(destinationContainer, destinationKey);
  109. URL url = getObjectURL(destinationContainer, destinationKey);
  110. Strings2.toStringAndClose(url.openStream());
  111. } finally {
  112. returnContainer(containerName);
  113. returnContainer(destinationContainer);
  114. }
  115. }
  116. String sourceKey = "apples";
  117. String destinationKey = "pears";
  118. public void testPublicWriteOnObject() throws InterruptedException, ExecutionException, TimeoutException, IOException {
  119. final String publicReadWriteObjectKey = "public-read-write-acl";
  120. final String containerName = getContainerName();
  121. try {
  122. S3Object object = getApi().newS3Object();
  123. object.getMetadata().setKey(publicReadWriteObjectKey);
  124. object.setPayload("");
  125. // Public Read-Write object
  126. getApi()
  127. .putObject(containerName, object,
  128. new PutObjectOptions().withAcl(CannedAccessPolicy.PUBLIC_READ_WRITE));
  129. assertConsistencyAware(new Runnable() {
  130. public void run() {
  131. try {
  132. AccessControlList acl = getApi().getObjectACL(containerName, publicReadWriteObjectKey);
  133. assertEquals(acl.getGrants().size(), 3);
  134. assertEquals(acl.getPermissions(GroupGranteeURI.ALL_USERS).size(), 2);
  135. assertTrue(acl.getOwner() != null);
  136. String ownerId = acl.getOwner().getId();
  137. assertTrue(acl.hasPermission(ownerId, Permission.FULL_CONTROL));
  138. assertTrue(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ));
  139. assertTrue(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.WRITE));
  140. assertFalse(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ_ACP));
  141. assertFalse(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.WRITE_ACP));
  142. assertFalse(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.FULL_CONTROL));
  143. } catch (Exception e) {
  144. Throwables.propagateIfPossible(e);
  145. }
  146. }
  147. });
  148. } finally {
  149. returnContainer(containerName);
  150. }
  151. }
  152. public void testUpdateObjectACL() throws InterruptedException, ExecutionException, TimeoutException, IOException {
  153. String containerName = getContainerName();
  154. try {
  155. String objectKey = "private-acl";
  156. // Private object
  157. addBlobToContainer(containerName, objectKey);
  158. AccessControlList acl = getApi().getObjectACL(containerName, objectKey);
  159. String ownerId = acl.getOwner().getId();
  160. assertEquals(acl.getGrants().size(), 1);
  161. assertTrue(acl.hasPermission(ownerId, Permission.FULL_CONTROL));
  162. addGrantsToACL(acl);
  163. assertEquals(acl.getGrants().size(), 4);
  164. assertTrue(getApi().putObjectACL(containerName, objectKey, acl));
  165. // Confirm that the updated ACL has stuck.
  166. acl = getApi().getObjectACL(containerName, objectKey);
  167. checkGrants(acl);
  168. /*
  169. * Revoke all of owner's permissions!
  170. */
  171. acl.revokeAllPermissions(new CanonicalUserGrantee(ownerId));
  172. if (!ownerId.equals(TEST_ACL_ID))
  173. acl.revokeAllPermissions(new CanonicalUserGrantee(TEST_ACL_ID));
  174. assertEquals(acl.getGrants().size(), 1);
  175. // Only public read permission should remain...
  176. assertTrue(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ));
  177. // Update the object's ACL settings
  178. assertTrue(getApi().putObjectACL(containerName, objectKey, acl));
  179. // Confirm that the updated ACL has stuck
  180. acl = getApi().getObjectACL(containerName, objectKey);
  181. assertEquals(acl.getGrants().size(), 1);
  182. assertEquals(acl.getPermissions(ownerId).size(), 0);
  183. assertTrue(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ), acl.toString());
  184. } finally {
  185. returnContainer(containerName);
  186. }
  187. }
  188. public void testPrivateAclIsDefaultForObject() throws InterruptedException, ExecutionException, TimeoutException,
  189. IOException {
  190. String privateObjectKey = "private-acl";
  191. String containerName = getContainerName();
  192. try {
  193. // Private object
  194. addBlobToContainer(containerName, privateObjectKey);
  195. AccessControlList acl = getApi().getObjectACL(containerName, privateObjectKey);
  196. assertEquals(acl.getGrants().size(), 1);
  197. assertTrue(acl.getOwner() != null);
  198. String ownerId = acl.getOwner().getId();
  199. assertTrue(acl.hasPermission(ownerId, Permission.FULL_CONTROL));
  200. } finally {
  201. returnContainer(containerName);
  202. }
  203. }
  204. public void testPublicReadOnObject() throws InterruptedException, ExecutionException, TimeoutException, IOException {
  205. final String publicReadObjectKey = "public-read-acl";
  206. final String containerName = getContainerName();
  207. try {
  208. S3Object object = getApi().newS3Object();
  209. object.getMetadata().setKey(publicReadObjectKey);
  210. object.setPayload("");
  211. getApi().putObject(containerName, object, new PutObjectOptions().withAcl(CannedAccessPolicy.PUBLIC_READ));
  212. assertConsistencyAware(new Runnable() {
  213. public void run() {
  214. try {
  215. AccessControlList acl = getApi().getObjectACL(containerName, publicReadObjectKey);
  216. assertEquals(acl.getGrants().size(), 2);
  217. assertEquals(acl.getPermissions(GroupGranteeURI.ALL_USERS).size(), 1);
  218. assertTrue(acl.getOwner() != null);
  219. String ownerId = acl.getOwner().getId();
  220. assertTrue(acl.hasPermission(ownerId, Permission.FULL_CONTROL));
  221. assertTrue(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ));
  222. } catch (Exception e) {
  223. Throwables.propagateIfPossible(e);
  224. }
  225. }
  226. });
  227. } finally {
  228. returnContainer(containerName);
  229. }
  230. }
  231. protected String addBlobToContainer(String sourceContainer, String key) {
  232. S3Object sourceObject = getApi().newS3Object();
  233. sourceObject.getMetadata().setKey(key);
  234. sourceObject.getMetadata().getContentMetadata().setContentType("text/xml");
  235. sourceObject.setPayload(TEST_STRING);
  236. return getApi().putObject(sourceContainer, sourceObject);
  237. }
  238. protected S3Object validateObject(String sourceContainer, String key) throws InterruptedException,
  239. ExecutionException, TimeoutException, IOException {
  240. assertConsistencyAwareContainerSize(sourceContainer, 1);
  241. S3Object newObject = getApi().getObject(sourceContainer, key);
  242. assert newObject != null;
  243. assertEquals(Strings2.toStringAndClose(newObject.getPayload().getInput()), TEST_STRING);
  244. return newObject;
  245. }
  246. public void testMetadataWithCacheControlAndContentDisposition() throws Exception {
  247. String key = "hello";
  248. S3Object object = getApi().newS3Object();
  249. object.getMetadata().setKey(key);
  250. object.setPayload(TEST_STRING);
  251. object.getMetadata().setCacheControl("no-cache");
  252. object.getMetadata().getContentMetadata().setContentDisposition("attachment; filename=hello.txt");
  253. String containerName = getContainerName();
  254. try {
  255. getApi().putObject(containerName, object);
  256. S3Object newObject = validateObject(containerName, key);
  257. assertCacheControl(newObject, "no-cache");
  258. assertEquals(newObject.getMetadata().getContentMetadata().getContentDisposition(),
  259. "attachment; filename=hello.txt");
  260. } finally {
  261. returnContainer(containerName);
  262. }
  263. }
  264. protected void assertCacheControl(S3Object newObject, String string) {
  265. assert (newObject.getMetadata().getCacheControl().indexOf(string) != -1) : newObject.getMetadata()
  266. .getCacheControl();
  267. }
  268. protected void assertContentEncoding(S3Object newObject, String string) {
  269. assert (newObject.getPayload().getContentMetadata().getContentEncoding().indexOf(string) != -1) : newObject
  270. .getPayload().getContentMetadata().getContentEncoding();
  271. assert (newObject.getMetadata().getContentMetadata().getContentEncoding().indexOf(string) != -1) : newObject
  272. .getMetadata().getContentMetadata().getContentEncoding();
  273. }
  274. @Test(groups = { "integration", "live" })
  275. public void testMetadataContentEncoding() throws Exception {
  276. String key = "hello";
  277. S3Object object = getApi().newS3Object();
  278. object.getMetadata().setKey(key);
  279. object.setPayload(TEST_STRING);
  280. object.getMetadata().getContentMetadata().setContentEncoding("x-compress");
  281. String containerName = getContainerName();
  282. try {
  283. getApi().putObject(containerName, object);
  284. S3Object newObject = validateObject(containerName, key);
  285. assertContentEncoding(newObject, "x-compress");
  286. } finally {
  287. returnContainer(containerName);
  288. }
  289. }
  290. public void testCopyObject() throws Exception {
  291. String containerName = getContainerName();
  292. String destinationContainer = getContainerName();
  293. try {
  294. addToContainerAndValidate(containerName, sourceKey);
  295. getApi().copyObject(containerName, sourceKey, destinationContainer, destinationKey);
  296. validateContent(destinationContainer, destinationKey);
  297. } finally {
  298. returnContainer(containerName);
  299. returnContainer(destinationContainer);
  300. }
  301. }
  302. protected String addToContainerAndValidate(String containerName, String sourceKey) throws InterruptedException,
  303. ExecutionException, TimeoutException, IOException {
  304. String etag = addBlobToContainer(containerName, sourceKey);
  305. validateContent(containerName, sourceKey);
  306. return etag;
  307. }
  308. // TODO: fails on linux and windows
  309. public void testCopyIfModifiedSince() throws InterruptedException, ExecutionException, TimeoutException, IOException {
  310. String containerName = getContainerName();
  311. String destinationContainer = getContainerName();
  312. try {
  313. Date before = new Date();
  314. addToContainerAndValidate(containerName, sourceKey + "mod");
  315. Date after = new Date(System.currentTimeMillis() + 1000);
  316. getApi().copyObject(containerName, sourceKey + "mod", destinationContainer, destinationKey,
  317. ifSourceModifiedSince(before));
  318. validateContent(destinationContainer, destinationKey);
  319. try {
  320. getApi().copyObject(containerName, sourceKey + "mod", destinationContainer, destinationKey,
  321. ifSourceModifiedSince(after));
  322. } catch (HttpResponseException ex) {
  323. assertEquals(ex.getResponse().getStatusCode(), 412);
  324. }
  325. } finally {
  326. returnContainer(containerName);
  327. returnContainer(destinationContainer);
  328. }
  329. }
  330. // TODO: fails on linux and windows
  331. public void testCopyIfUnmodifiedSince() throws InterruptedException, ExecutionException, TimeoutException,
  332. IOException {
  333. String containerName = getContainerName();
  334. String destinationContainer = getContainerName();
  335. try {
  336. Date before = new Date();
  337. addToContainerAndValidate(containerName, sourceKey + "un");
  338. Date after = new Date(System.currentTimeMillis() + 1000);
  339. getApi().copyObject(containerName, sourceKey + "un", destinationContainer, destinationKey,
  340. ifSourceUnmodifiedSince(after));
  341. validateContent(destinationContainer, destinationKey);
  342. try {
  343. getApi().copyObject(containerName, sourceKey + "un", destinationContainer, destinationKey,
  344. ifSourceModifiedSince(before));
  345. } catch (HttpResponseException ex) {
  346. assertEquals(ex.getResponse().getStatusCode(), 412);
  347. }
  348. } finally {
  349. returnContainer(containerName);
  350. returnContainer(destinationContainer);
  351. }
  352. }
  353. public void testCopyIfMatch() throws InterruptedException, ExecutionException, TimeoutException, IOException {
  354. String containerName = getContainerName();
  355. String destinationContainer = getContainerName();
  356. try {
  357. String goodETag = addToContainerAndValidate(containerName, sourceKey);
  358. getApi().copyObject(containerName, sourceKey, destinationContainer, destinationKey,
  359. ifSourceETagMatches(goodETag));
  360. validateContent(destinationContainer, destinationKey);
  361. try {
  362. getApi().copyObject(containerName, sourceKey, destinationContainer, destinationKey,
  363. ifSourceETagMatches("setsds"));
  364. } catch (HttpResponseException ex) {
  365. assertEquals(ex.getResponse().getStatusCode(), 412);
  366. }
  367. } finally {
  368. returnContainer(containerName);
  369. returnContainer(destinationContainer);
  370. }
  371. }
  372. public void testCopyIfNoneMatch() throws IOException, InterruptedException, ExecutionException, TimeoutException {
  373. String containerName = getContainerName();
  374. String destinationContainer = getContainerName();
  375. try {
  376. String goodETag = addToContainerAndValidate(containerName, sourceKey);
  377. getApi().copyObject(containerName, sourceKey, destinationContainer, destinationKey,
  378. ifSourceETagDoesntMatch("asfasdf"));
  379. validateContent(destinationContainer, destinationKey);
  380. try {
  381. getApi().copyObject(containerName, sourceKey, destinationContainer, destinationKey,
  382. ifSourceETagDoesntMatch(goodETag));
  383. } catch (HttpResponseException ex) {
  384. assertEquals(ex.getResponse().getStatusCode(), 412);
  385. }
  386. } finally {
  387. returnContainer(containerName);
  388. returnContainer(destinationContainer);
  389. }
  390. }
  391. public void testCopyWithMetadata() throws InterruptedException, ExecutionException, TimeoutException, IOException {
  392. String containerName = getContainerName();
  393. String destinationContainer = getContainerName();
  394. try {
  395. addToContainerAndValidate(containerName, sourceKey);
  396. Map<String, String> metadata = Maps.newHashMap();
  397. metadata.put("adrian", "cole");
  398. getApi().copyObject(containerName, sourceKey, destinationContainer, destinationKey,
  399. overrideMetadataWith(metadata));
  400. validateContent(destinationContainer, destinationKey);
  401. ObjectMetadata objectMeta = getApi().headObject(destinationContainer, destinationKey);
  402. assertEquals(objectMeta.getUserMetadata(), metadata);
  403. } finally {
  404. returnContainer(containerName);
  405. returnContainer(destinationContainer);
  406. }
  407. }
  408. private void checkGrants(AccessControlList acl) {
  409. String ownerId = acl.getOwner().getId();
  410. assertEquals(acl.getGrants().size(), 4, acl.toString());
  411. assertTrue(acl.hasPermission(ownerId, Permission.FULL_CONTROL), acl.toString());
  412. assertTrue(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ), acl.toString());
  413. assertTrue(acl.hasPermission(ownerId, Permission.WRITE_ACP), acl.toString());
  414. // EmailAddressGrantee is replaced by a CanonicalUserGrantee, so we cannot test by email addr
  415. assertTrue(acl.hasPermission(TEST_ACL_ID, Permission.READ_ACP), acl.toString());
  416. }
  417. private void addGrantsToACL(AccessControlList acl) {
  418. String ownerId = acl.getOwner().getId();
  419. acl.addPermission(GroupGranteeURI.ALL_USERS, Permission.READ);
  420. acl.addPermission(new EmailAddressGrantee(TEST_ACL_EMAIL), Permission.READ_ACP);
  421. acl.addPermission(new CanonicalUserGrantee(ownerId), Permission.WRITE_ACP);
  422. }
  423. }