/core/src/com/cloud/storage/template/HttpTemplateDownloader.java
Java | 450 lines | 353 code | 67 blank | 30 comment | 73 complexity | 9b875c3836138a1d45757b9a3eaa5085 MD5 | raw file
- /**
- * Copyright (C) 2010 Cloud.com, Inc. All rights reserved.
- *
- * This software is licensed under the GNU General Public License v3 or later.
- *
- * It is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or any later version.
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- */
- package com.cloud.storage.template;
- import java.io.BufferedInputStream;
- import java.io.File;
- import java.io.IOException;
- import java.io.InputStream;
- import java.io.RandomAccessFile;
- import java.net.Inet6Address;
- import java.net.InetAddress;
- import java.net.URI;
- import java.net.URISyntaxException;
- import java.net.UnknownHostException;
- import java.util.Date;
- import org.apache.commons.httpclient.ChunkedInputStream;
- import org.apache.commons.httpclient.Credentials;
- import org.apache.commons.httpclient.Header;
- import org.apache.commons.httpclient.HttpClient;
- import org.apache.commons.httpclient.HttpException;
- import org.apache.commons.httpclient.HttpMethod;
- import org.apache.commons.httpclient.HttpMethodRetryHandler;
- import org.apache.commons.httpclient.HttpStatus;
- import org.apache.commons.httpclient.NoHttpResponseException;
- import org.apache.commons.httpclient.UsernamePasswordCredentials;
- import org.apache.commons.httpclient.auth.AuthScope;
- import org.apache.commons.httpclient.methods.GetMethod;
- import org.apache.commons.httpclient.params.HttpMethodParams;
- import org.apache.log4j.Logger;
- import com.cloud.storage.StorageLayer;
- import com.cloud.utils.exception.CloudRuntimeException;
- import com.cloud.utils.Pair;
- /**
- * Download a template file using HTTP
- * @author Chiradeep
- *
- */
- public class HttpTemplateDownloader implements TemplateDownloader {
- public static final Logger s_logger = Logger.getLogger(HttpTemplateDownloader.class.getName());
- private static final int CHUNK_SIZE = 1024*1024; //1M
- private String downloadUrl;
- private String toFile;
- public TemplateDownloader.Status status= TemplateDownloader.Status.NOT_STARTED;
- public String errorString = " ";
- private long remoteSize = 0;
- public long downloadTime = 0;
- public long totalBytes;
- private final HttpClient client;
- private GetMethod request;
- private boolean resume = false;
- private DownloadCompleteCallback completionCallback;
- StorageLayer _storage;
- boolean inited = true;
- private String toDir;
- private long MAX_TEMPLATE_SIZE_IN_BYTES;
- private final HttpMethodRetryHandler myretryhandler;
- public HttpTemplateDownloader (StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, String password) {
- this._storage = storageLayer;
- this.downloadUrl = downloadUrl;
- this.setToDir(toDir);
- this.status = TemplateDownloader.Status.NOT_STARTED;
-
- this.MAX_TEMPLATE_SIZE_IN_BYTES = maxTemplateSizeInBytes;
-
- this.totalBytes = 0;
- this.client = new HttpClient();
- myretryhandler = new HttpMethodRetryHandler() {
- public boolean retryMethod(
- final HttpMethod method,
- final IOException exception,
- int executionCount) {
- if (executionCount >= 2) {
- // Do not retry if over max retry count
- return false;
- }
- if (exception instanceof NoHttpResponseException) {
- // Retry if the server dropped connection on us
- return true;
- }
- if (!method.isRequestSent()) {
- // Retry if the request has not been sent fully or
- // if it's OK to retry methods that have been sent
- return true;
- }
- // otherwise do not retry
- return false;
- }
- };
-
- try {
- this.request = new GetMethod(downloadUrl);
- this.request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
- this.completionCallback = callback;
- //this.request.setFollowRedirects(false);
- File f = File.createTempFile("dnld", "tmp_", new File(toDir));
-
- if (_storage != null) {
- _storage.setWorldReadableAndWriteable(f);
- }
-
- toFile = f.getAbsolutePath();
- Pair<String, Integer> hostAndPort = validateUrl(downloadUrl);
-
- if ((user != null) && (password != null)) {
- client.getParams().setAuthenticationPreemptive(true);
- Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
- client.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
- s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second());
- } else {
- s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second());
- }
- } catch (IllegalArgumentException iae) {
- errorString = iae.getMessage();
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- inited = false;
- } catch (Exception ex){
- errorString = "Unable to start download -- check url? ";
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- s_logger.warn("Exception in constructor -- " + ex.toString());
- } catch (Throwable th) {
- s_logger.warn("throwable caught ", th);
- }
- }
-
- private Pair<String, Integer> validateUrl(String url) throws IllegalArgumentException {
- try {
- URI uri = new URI(url);
- if (!uri.getScheme().equalsIgnoreCase("http") && !uri.getScheme().equalsIgnoreCase("https") ) {
- throw new IllegalArgumentException("Unsupported scheme for url");
- }
- int port = uri.getPort();
- if (!(port == 80 || port == 443 || port == -1)) {
- throw new IllegalArgumentException("Only ports 80 and 443 are allowed");
- }
-
- if (port == -1 && uri.getScheme().equalsIgnoreCase("https")) {
- port = 443;
- } else if (port == -1 && uri.getScheme().equalsIgnoreCase("http")) {
- port = 80;
- }
-
- String host = uri.getHost();
- try {
- InetAddress hostAddr = InetAddress.getByName(host);
- if (hostAddr.isAnyLocalAddress() || hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress() || hostAddr.isMulticastAddress()) {
- throw new IllegalArgumentException("Illegal host specified in url");
- }
- if (hostAddr instanceof Inet6Address) {
- throw new IllegalArgumentException("IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")");
- }
- return new Pair<String, Integer>(host, port);
- } catch (UnknownHostException uhe) {
- throw new IllegalArgumentException("Unable to resolve " + host);
- }
- } catch (IllegalArgumentException iae) {
- s_logger.warn("Failed uri validation check: " + iae.getMessage());
- throw iae;
- } catch (URISyntaxException use) {
- s_logger.warn("Failed uri syntax check: " + use.getMessage());
- throw new IllegalArgumentException(use.getMessage());
- }
- }
-
- @Override
- public long download(boolean resume, DownloadCompleteCallback callback) {
- switch (status) {
- case ABORTED:
- case UNRECOVERABLE_ERROR:
- case DOWNLOAD_FINISHED:
- return 0;
- default:
- }
- int bytes=0;
- File file = new File(toFile);
- try {
-
- long localFileSize = 0;
- if (file.exists() && resume) {
- localFileSize = file.length();
- s_logger.info("Resuming download to file (current size)=" + localFileSize);
- }
-
- Date start = new Date();
- int responseCode=0;
-
- if (localFileSize > 0 ) {
- // require partial content support for resume
- request.addRequestHeader("Range", "bytes=" + localFileSize + "-");
- if (client.executeMethod(request) != HttpStatus.SC_PARTIAL_CONTENT) {
- errorString = "HTTP Server does not support partial get";
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- return 0;
- }
- } else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
- return 0; //FIXME: retry?
- }
-
- Header contentLengthHeader = request.getResponseHeader("Content-Length");
- boolean chunked = false;
- long remoteSize2 = 0;
- if (contentLengthHeader == null) {
- Header chunkedHeader = request.getResponseHeader("Transfer-Encoding");
- if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString=" Failed to receive length of download ";
- return 0; //FIXME: what status do we put here? Do we retry?
- } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())){
- chunked = true;
- }
- } else {
- remoteSize2 = Long.parseLong(contentLengthHeader.getValue());
- }
- if (remoteSize == 0) {
- remoteSize = remoteSize2;
- }
-
- if (remoteSize > MAX_TEMPLATE_SIZE_IN_BYTES) {
- s_logger.info("Remote size is too large: " + remoteSize + " , max=" + MAX_TEMPLATE_SIZE_IN_BYTES);
- status = Status.UNRECOVERABLE_ERROR;
- errorString = "Download file size is too large";
- return 0;
- }
-
- if (remoteSize == 0) {
- remoteSize = MAX_TEMPLATE_SIZE_IN_BYTES;
- }
-
- InputStream in = !chunked?new BufferedInputStream(request.getResponseBodyAsStream())
- : new ChunkedInputStream(request.getResponseBodyAsStream());
-
- RandomAccessFile out = new RandomAccessFile(file, "rwd");
- out.seek(localFileSize);
- s_logger.info("Starting download from " + getDownloadUrl() + " to " + toFile + " remoteSize=" + remoteSize + " , max size=" + MAX_TEMPLATE_SIZE_IN_BYTES);
-
- byte[] block = new byte[CHUNK_SIZE];
- long offset=0;
- boolean done=false;
- status = TemplateDownloader.Status.IN_PROGRESS;
- while (!done && status != Status.ABORTED && offset <= remoteSize) {
- if ( (bytes = in.read(block, 0, CHUNK_SIZE)) > -1) {
- out.write(block, 0, bytes);
- offset +=bytes;
- out.seek(offset);
- totalBytes += bytes;
- } else {
- done = true;
- }
- }
- Date finish = new Date();
- String downloaded = "(incomplete download)";
- if (totalBytes >= remoteSize) {
- status = TemplateDownloader.Status.DOWNLOAD_FINISHED;
- downloaded = "(download complete remote=" + remoteSize + "bytes)";
- }
- errorString = "Downloaded " + totalBytes + " bytes " + downloaded;
- downloadTime += finish.getTime() - start.getTime();
- out.close();
-
- return totalBytes;
- }catch (HttpException hte) {
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString = hte.getMessage();
- } catch (IOException ioe) {
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; //probably a file write error?
- errorString = ioe.getMessage();
- } finally {
- if (status == Status.UNRECOVERABLE_ERROR && file.exists() && !file.isDirectory()) {
- file.delete();
- }
- request.releaseConnection();
- if (callback != null) {
- callback.downloadComplete(status);
- }
- }
- return 0;
- }
- public String getDownloadUrl() {
- return downloadUrl;
- }
- public String getToFile() {
- File file = new File(toFile);
- return file.getAbsolutePath();
- }
- public TemplateDownloader.Status getStatus() {
- return status;
- }
- public long getDownloadTime() {
- return downloadTime;
- }
-
-
- public long getDownloadedBytes() {
- return totalBytes;
- }
- @Override
- @SuppressWarnings("fallthrough")
- public boolean stopDownload() {
- switch (getStatus()) {
- case IN_PROGRESS:
- if (request != null) {
- request.abort();
- }
- status = TemplateDownloader.Status.ABORTED;
- return true;
- case UNKNOWN:
- case NOT_STARTED:
- case RECOVERABLE_ERROR:
- case UNRECOVERABLE_ERROR:
- case ABORTED:
- status = TemplateDownloader.Status.ABORTED;
- case DOWNLOAD_FINISHED:
- File f = new File(toFile);
- if (f.exists()) {
- f.delete();
- }
- return true;
- default:
- return true;
- }
- }
- @Override
- public int getDownloadPercent() {
- if (remoteSize == 0) {
- return 0;
- }
-
- return (int)(100.0*totalBytes/remoteSize);
- }
- @Override
- public void run() {
- try {
- download(resume, completionCallback);
- } catch (Throwable t) {
- s_logger.warn("Caught exception during download "+ t.getMessage(), t);
- errorString = "Failed to install: " + t.getMessage();
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- }
-
- }
- @Override
- public void setStatus(TemplateDownloader.Status status) {
- this.status = status;
- }
- public boolean isResume() {
- return resume;
- }
- @Override
- public String getDownloadError() {
- return errorString;
- }
- @Override
- public String getDownloadLocalPath() {
- return getToFile();
- }
- public void setResume(boolean resume) {
- this.resume = resume;
- }
- public void setToDir(String toDir) {
- this.toDir = toDir;
- }
- public String getToDir() {
- return toDir;
- }
- public long getMaxTemplateSizeInBytes() {
- return this.MAX_TEMPLATE_SIZE_IN_BYTES;
- }
-
- public static void main(String[] args) {
- String url ="http:// dev.mysql.com/get/Downloads/MySQL-5.0/mysql-noinstall-5.0.77-win32.zip/from/http://mirror.services.wisc.edu/mysql/";
- try {
- URI uri = new java.net.URI(url);
- } catch (URISyntaxException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- TemplateDownloader td = new HttpTemplateDownloader(null, url,"/tmp/mysql", null, TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES, null, null);
- long bytes = td.download(true, null);
- if (bytes > 0) {
- System.out.println("Downloaded (" + bytes + " bytes)" + " in " + td.getDownloadTime()/1000 + " secs");
- } else {
- System.out.println("Failed download");
- }
- }
- @Override
- public void setDownloadError(String error) {
- errorString = error;
- }
- @Override
- public boolean isInited() {
- return inited;
- }
- }