/modules/elasticsearch/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java
Java | 400 lines | 231 code | 54 blank | 115 comment | 42 complexity | 175d08c8d50e80110d8f05f16dbe1070 MD5 | raw file
- /*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
- package org.elasticsearch.common.http.client;
- import org.elasticsearch.common.Nullable;
- import java.io.File;
- import java.io.FileNotFoundException;
- import java.io.FileOutputStream;
- import java.io.IOException;
- import java.io.InputStream;
- import java.io.OutputStream;
- import java.io.PrintStream;
- import java.net.HttpURLConnection;
- import java.net.URL;
- import java.net.URLConnection;
- /**
- * @author kimchy (shay.banon)
- */
- public class HttpDownloadHelper {
- private boolean useTimestamp = false;
- private boolean skipExisting = false;
- private long maxTime = 0;
- public boolean download(URL source, File dest, @Nullable DownloadProgress progress) throws IOException {
- if (dest.exists() && skipExisting) {
- return true;
- }
- //don't do any progress, unless asked
- if (progress == null) {
- progress = new NullProgress();
- }
- //set the timestamp to the file date.
- long timestamp = 0;
- boolean hasTimestamp = false;
- if (useTimestamp && dest.exists()) {
- timestamp = dest.lastModified();
- hasTimestamp = true;
- }
- GetThread getThread = new GetThread(source, dest, hasTimestamp, timestamp, progress);
- getThread.setDaemon(true);
- getThread.start();
- try {
- getThread.join(maxTime * 1000);
- } catch (InterruptedException ie) {
- // ignore
- }
- if (getThread.isAlive()) {
- String msg = "The GET operation took longer than " + maxTime
- + " seconds, stopping it.";
- getThread.closeStreams();
- throw new IOException(msg);
- }
- return getThread.wasSuccessful();
- }
- /**
- * Interface implemented for reporting
- * progress of downloading.
- */
- public interface DownloadProgress {
- /**
- * begin a download
- */
- void beginDownload();
- /**
- * tick handler
- */
- void onTick();
- /**
- * end a download
- */
- void endDownload();
- }
- /**
- * do nothing with progress info
- */
- public static class NullProgress implements DownloadProgress {
- /**
- * begin a download
- */
- public void beginDownload() {
- }
- /**
- * tick handler
- */
- public void onTick() {
- }
- /**
- * end a download
- */
- public void endDownload() {
- }
- }
- /**
- * verbose progress system prints to some output stream
- */
- public static class VerboseProgress implements DownloadProgress {
- private int dots = 0;
- // CheckStyle:VisibilityModifier OFF - bc
- PrintStream out;
- // CheckStyle:VisibilityModifier ON
- /**
- * Construct a verbose progress reporter.
- *
- * @param out the output stream.
- */
- public VerboseProgress(PrintStream out) {
- this.out = out;
- }
- /**
- * begin a download
- */
- public void beginDownload() {
- out.print("Downloading ");
- dots = 0;
- }
- /**
- * tick handler
- */
- public void onTick() {
- out.print(".");
- if (dots++ > 50) {
- out.flush();
- dots = 0;
- }
- }
- /**
- * end a download
- */
- public void endDownload() {
- out.println("DONE");
- out.flush();
- }
- }
- private class GetThread extends Thread {
- private final URL source;
- private final File dest;
- private final boolean hasTimestamp;
- private final long timestamp;
- private final DownloadProgress progress;
- private boolean success = false;
- private IOException ioexception = null;
- private InputStream is = null;
- private OutputStream os = null;
- private URLConnection connection;
- private int redirections = 0;
- GetThread(URL source, File dest, boolean h, long t, DownloadProgress p) {
- this.source = source;
- this.dest = dest;
- hasTimestamp = h;
- timestamp = t;
- progress = p;
- }
- public void run() {
- try {
- success = get();
- } catch (IOException ioex) {
- ioexception = ioex;
- }
- }
- private boolean get() throws IOException {
- connection = openConnection(source);
- if (connection == null) {
- return false;
- }
- boolean downloadSucceeded = downloadFile();
- //if (and only if) the use file time option is set, then
- //the saved file now has its timestamp set to that of the
- //downloaded file
- if (downloadSucceeded && useTimestamp) {
- updateTimeStamp();
- }
- return downloadSucceeded;
- }
- private boolean redirectionAllowed(URL aSource, URL aDest) throws IOException {
- // Argh, github does this...
- // if (!(aSource.getProtocol().equals(aDest.getProtocol()) || ("http"
- // .equals(aSource.getProtocol()) && "https".equals(aDest
- // .getProtocol())))) {
- // String message = "Redirection detected from "
- // + aSource.getProtocol() + " to " + aDest.getProtocol()
- // + ". Protocol switch unsafe, not allowed.";
- // throw new IOException(message);
- // }
- redirections++;
- if (redirections > 5) {
- String message = "More than " + 5 + " times redirected, giving up";
- throw new IOException(message);
- }
- return true;
- }
- private URLConnection openConnection(URL aSource) throws IOException {
- // set up the URL connection
- URLConnection connection = aSource.openConnection();
- // modify the headers
- // NB: things like user authentication could go in here too.
- if (hasTimestamp) {
- connection.setIfModifiedSince(timestamp);
- }
- if (connection instanceof HttpURLConnection) {
- ((HttpURLConnection) connection).setInstanceFollowRedirects(false);
- ((HttpURLConnection) connection).setUseCaches(true);
- ((HttpURLConnection) connection).setConnectTimeout(5000);
- }
- // connect to the remote site (may take some time)
- connection.connect();
- // First check on a 301 / 302 (moved) response (HTTP only)
- if (connection instanceof HttpURLConnection) {
- HttpURLConnection httpConnection = (HttpURLConnection) connection;
- int responseCode = httpConnection.getResponseCode();
- if (responseCode == HttpURLConnection.HTTP_MOVED_PERM ||
- responseCode == HttpURLConnection.HTTP_MOVED_TEMP ||
- responseCode == HttpURLConnection.HTTP_SEE_OTHER) {
- String newLocation = httpConnection.getHeaderField("Location");
- String message = aSource
- + (responseCode == HttpURLConnection.HTTP_MOVED_PERM ? " permanently"
- : "") + " moved to " + newLocation;
- URL newURL = new URL(newLocation);
- if (!redirectionAllowed(aSource, newURL)) {
- return null;
- }
- return openConnection(newURL);
- }
- // next test for a 304 result (HTTP only)
- long lastModified = httpConnection.getLastModified();
- if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED
- || (lastModified != 0 && hasTimestamp && timestamp >= lastModified)) {
- // not modified so no file download. just return
- // instead and trace out something so the user
- // doesn't think that the download happened when it
- // didn't
- return null;
- }
- // test for 401 result (HTTP only)
- if (responseCode == HttpURLConnection.HTTP_UNAUTHORIZED) {
- String message = "HTTP Authorization failure";
- throw new IOException(message);
- }
- }
- //REVISIT: at this point even non HTTP connections may
- //support the if-modified-since behaviour -we just check
- //the date of the content and skip the write if it is not
- //newer. Some protocols (FTP) don't include dates, of
- //course.
- return connection;
- }
- private boolean downloadFile() throws FileNotFoundException, IOException {
- IOException lastEx = null;
- for (int i = 0; i < 3; i++) {
- // this three attempt trick is to get round quirks in different
- // Java implementations. Some of them take a few goes to bind
- // property; we ignore the first couple of such failures.
- try {
- is = connection.getInputStream();
- break;
- } catch (IOException ex) {
- lastEx = ex;
- }
- }
- if (is == null) {
- throw new IOException("Can't get " + source + " to " + dest, lastEx);
- }
- os = new FileOutputStream(dest);
- progress.beginDownload();
- boolean finished = false;
- try {
- byte[] buffer = new byte[1024 * 100];
- int length;
- while (!isInterrupted() && (length = is.read(buffer)) >= 0) {
- os.write(buffer, 0, length);
- progress.onTick();
- }
- finished = !isInterrupted();
- } finally {
- try {
- os.close();
- } catch (IOException e) {
- // ignore
- }
- try {
- is.close();
- } catch (IOException e) {
- // ignore
- }
- // we have started to (over)write dest, but failed.
- // Try to delete the garbage we'd otherwise leave
- // behind.
- if (!finished) {
- dest.delete();
- }
- }
- progress.endDownload();
- return true;
- }
- private void updateTimeStamp() {
- long remoteTimestamp = connection.getLastModified();
- if (remoteTimestamp != 0) {
- dest.setLastModified(remoteTimestamp);
- }
- }
- /**
- * Has the download completed successfully?
- *
- * <p>Re-throws any exception caught during executaion.</p>
- */
- boolean wasSuccessful() throws IOException {
- if (ioexception != null) {
- throw ioexception;
- }
- return success;
- }
- /**
- * Closes streams, interrupts the download, may delete the
- * output file.
- */
- void closeStreams() {
- interrupt();
- try {
- os.close();
- } catch (IOException e) {
- // ignore
- }
- try {
- is.close();
- } catch (IOException e) {
- // ignore
- }
- if (!success && dest.exists()) {
- dest.delete();
- }
- }
- }
- }