/edu/uncc/parsets/util/BatchConvert.java

  1package edu.uncc.parsets.util;
  2
  3import java.io.File;
  4import java.util.ArrayList;
  5import java.util.List;
  6
  7import edu.uncc.parsets.data.JSONExport;
  8import edu.uncc.parsets.data.LocalDB;
  9import edu.uncc.parsets.data.LocalDBDataSet;
 10import edu.uncc.parsets.data.old.CSVDataSet;
 11import edu.uncc.parsets.data.old.CSVParser;
 12
 13/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
 14 * Copyright (c) 2009, Robert Kosara, Caroline Ziemkiewicz,
 15 *                     and others (see Authors.txt for full list)
 16 * All rights reserved.
 17 * 
 18 * Redistribution and use in source and binary forms, with or without
 19 * modification, are permitted provided that the following conditions are met:
 20 * 
 21 *    * Redistributions of source code must retain the above copyright
 22 *      notice, this list of conditions and the following disclaimer.
 23 *    * Redistributions in binary form must reproduce the above copyright
 24 *      notice, this list of conditions and the following disclaimer in the
 25 *      documentation and/or other materials provided with the distribution.
 26 *    * Neither the name of UNC Charlotte nor the names of its contributors
 27 *      may be used to endorse or promote products derived from this software
 28 *      without specific prior written permission.
 29 *      
 30 * THIS SOFTWARE IS PROVIDED BY ITS AUTHORS ''AS IS'' AND ANY
 31 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 32 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 33 * DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY
 34 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 35 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 36 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 37 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 39 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 40\* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 41
 42public class BatchConvert {
 43
 44	private static final String TEMPDBFILENAME = "temp.db";
 45
 46	private static final String USAGE = "Usage: convert <srcdir> <destdir>";
 47
 48	public static final String BASEURL = "http://data.eagereyes.org/";
 49	
 50	public static void batchConvert(String args[]) {
 51		if (args.length != 3 || !args[0].equals("convert")) {
 52			System.err.println(USAGE);
 53			return;
 54		}
 55		File tempDBFile = new File(TEMPDBFILENAME);
 56		if (tempDBFile.exists())
 57			if (!tempDBFile.delete())
 58				System.err.println("Could not delete temporary database "+TEMPDBFILENAME);
 59		
 60		File srcDir = new File(args[1]);
 61		File dstDir = new File(args[2]);
 62		
 63		LocalDB tempDB = new LocalDB(TEMPDBFILENAME);
 64		
 65		List<File> csvFiles = scanDir(srcDir, dstDir);
 66		for (File f : csvFiles) {
 67			CSVParser parser = new CSVParser(f.getPath(), null);
 68			Thread t = parser.analyzeCSVFile();
 69			try {
 70				t.join();
 71			} catch (InterruptedException e) {
 72				e.printStackTrace();
 73			}
 74			CSVDataSet csvData = parser.getDataSet();
 75			System.out.println("Importing "+csvData.getName()+" ...");
 76			String newPath = rebase(f.getPath(), dstDir.getPath())+".json.gz";
 77			csvData.setURL(BASEURL+newPath);
 78			parser.streamToDB(tempDB);
 79		}
 80		
 81		for (LocalDBDataSet ds : tempDB.getDataSets()) {
 82			String newPath = ds.getURL().substring(BASEURL.length());
 83			System.out.println("Exporting "+ds.getName()+" to "+newPath);
 84			JSONExport.exportDataSet(ds, newPath);
 85		}
 86		
 87		System.out.println("Exporting index ...");
 88		String indexName = JSONExport.exportDBIndex(tempDB, dstDir.getPath()+File.separatorChar+"index");
 89		System.out.println("Index: "+indexName);
 90		System.out.println("Done.");
 91		
 92		if (!tempDBFile.delete())
 93			System.err.println("Could not delete temporary database "+TEMPDBFILENAME);
 94	}
 95
 96	private static String rebase(String original, String newBase) {
 97		StringBuilder newPath = new StringBuilder(newBase);
 98		newPath.append(original.substring(original.indexOf(File.separatorChar), original.lastIndexOf(".")));
 99		return newPath.toString();
100	}
101
102	private static List<File> scanDir(File srcDir, File dstDir) {
103		if (!dstDir.exists())
104			if (!dstDir.mkdir())
105				System.err.println("Could not create directory "+dstDir.getPath());
106		List<File> files = new ArrayList<File>();
107		for (File f : srcDir.listFiles()) {
108			if (f.isDirectory()) {
109				if (!f.getName().startsWith(".")) {
110					File newDest = new File(dstDir.getPath()+File.separatorChar+f.getName());
111					files.addAll(scanDir(f, newDest));
112				}
113			} else if (f.getName().endsWith(".csv")) {
114				files.add(f);
115//				System.out.println(f.getPath());
116			}
117		}
118		return files;
119	}	
120}