/src/FloCK/FCSLoader/FCSLoader.java
Java | 665 lines | 498 code | 97 blank | 70 comment | 119 complexity | 8267c09a5dc67d245fd5f4870cba2afc MD5 | raw file
Possible License(s): GPL-3.0
- /**
- * Copyright (C) 2011 Tjibbe Donker
- *
- * This file originally came from the FloCK program
- * (http://theory.bio.uu.nl/tjibbe/flock/) and is now
- * part of the IFCSoft project (http://ifcsoft.com)
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
- package FloCK.FCSLoader;
- //import FloCK.FCSLoader.stats.*;
- //import fcs.matrix.*;
- import java.io.FileReader;
- import java.io.FileWriter;
- import java.io.IOException;
- import java.io.FileInputStream ;
- import java.io.File ;
- import java.io.InputStream ;
- import java.io.DataInputStream ;
- import javax.swing.*;
- import java.awt.event.*;
- //import components.*;
- import Jama.Matrix;
- import java.io.BufferedInputStream;
- public class FCSLoader {
- int progress = 0; //Added by Kyle Thayer for a reporting progress
- boolean BigDEBUG = false;
- boolean memDEBUG = false;
- boolean DEBUG = false;
- public File FCSFile;
- int length=2;
- int StartText=0;
- int StopText=0;
- int StartData=0;
- int StopData=0;
- int NumTextElements=1;
- int NumBytes=1;
- int ByteOrd[];
- char DataType='*';
- String CreatorMachine;
- String DateCreated;
- String DateExport;
- String VersionS;
- float VersionF;
- String ChannelNames[];
- boolean LogNorm[];
- boolean NeedCompensation = false;
- Matrix CompensationMatrix;
- Matrix invCompensationMatrix;
- int NumMChan;
- int CompensationChannels[];
- int NormScale = 1024;
- static JFrame frame;
-
- volatile boolean busy;
-
- public FCSLoader(){
- length=2;
- StartText=0;
- StopText=0;
- StartData=0;
- StopData=0;
- NumTextElements=1;
- NumBytes=1;
- ByteOrd = new int[4];
- }
-
- public void resetDataSet(){
- length=2;
- StartText=0;
- StopText=0;
- StartData=0;
- StopData=0;
- NumTextElements=1;
- NumBytes=1;
- ByteOrd = new int[4];
- }
- public float arr2float (byte[] arr, int start) {
- //http://www.captain.at/howto-java-convert-binary-data.php
- int i = 0;
- int len = 4;
- byte[] tmp = new byte[len];
-
- for (i = start; i < (start + len); i++)
- tmp[ByteOrd[i-start]] = arr[i];
-
- int accum = 0;
- i = 0;
- for ( int shiftBy = 0; shiftBy < 32; shiftBy += 8 ) {
- accum |= ( (long)( tmp[i] & 0xff ) ) << shiftBy; i++;
- }
- return Float.intBitsToFloat(accum);
- }
- public int arr2int (byte[] arr, int len) {
- // int len=2;
- int myMin=4;
- int myMax=0;
- int myInt=0;
- byte[] tmp = new byte[len];
- for(int i=0;i<len;i++){
- if(myMin>ByteOrd[i]) myMin=ByteOrd[i];
- if(myMax<ByteOrd[i]) myMax=ByteOrd[i];
- }
- if(myMax-myMin==len-1) {
- for(int i=0;i<len;i++)
- myInt+=((arr[ByteOrd[i]-myMin]&0xff)<<(8*i));
- } else myInt=-1;
-
- return myInt;
- }
- class CompareScreen extends JPanel implements ActionListener{
- JButton okButton;
-
- public CompareScreen(){
- okButton = new JButton("OK");
- add(okButton);
- okButton.addActionListener(this);
- }
-
- public void actionPerformed(ActionEvent e) {
-
-
- if(e.getSource()==okButton){
- synchronized(this){
- notifyAll();
- }
- }
- }
- }
- void CreateCompareScreen() {
- //Create and set up the window.
- JFrame NumClusInf = new JFrame("Cluster Number Information");
- NumClusInf.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
- //Create and set up the content pane.
- CompareScreen ComparePane = new CompareScreen();
- ComparePane.setOpaque(true); //content panes must be opaque
- NumClusInf.setContentPane(ComparePane);
- //Display the window.
- NumClusInf.pack();
- NumClusInf.setVisible(true);
- }
- public Stats ReadFCS(String Filename)throws IOException{
- return ReadFCS(Filename,true);
- }
-
- // Convert the string containing the spillover matrix to a real matrix
- public void LoadSpillmatrix(String loadedString){
-
- int CurIndex=0;
- int TransToMatrix[] = new int[ChannelNames.length];
- for(int j=0;j<ChannelNames.length;j++) TransToMatrix[j]=-1;
- int NewIndex=loadedString.indexOf(",",CurIndex);
- String str = loadedString.substring(CurIndex,NewIndex);
- NumMChan=(int)Integer.valueOf(str.trim()).intValue() ;
- double vals[][] = new double[NumMChan][NumMChan];
- if(DEBUG) System.out.println(loadedString);
- if(DEBUG) System.out.println("Number of channels in matrix: "+NumMChan);
- for(int i=0;i<NumMChan;i++){
- CurIndex=NewIndex;
- NewIndex=loadedString.indexOf(",",CurIndex+1);
- str = loadedString.substring(CurIndex+1,NewIndex).trim();
- int NameLink=-1;
- for(int j=0;j<ChannelNames.length;j++){
- if(ChannelNames[j].indexOf(str,0)!=-1)NameLink=j;
- }
- TransToMatrix[NameLink]=i;
- if(DEBUG) System.out.println(str+" at position "+NameLink);
- }
- for(int i=0;i<NumMChan;i++){
- for(int j=0;j<NumMChan;j++){
- CurIndex=NewIndex;
- NewIndex=loadedString.indexOf(",",CurIndex+1);
- if(NewIndex<0) NewIndex=loadedString.length();
- if(DEBUG) System.out.print(CurIndex+" to "+NewIndex);
- str = loadedString.substring(CurIndex+1,NewIndex).trim();
-
- vals[j][i]=Double.valueOf(str.trim()).doubleValue();
- //if(i!=j) vals[i][j]=-vals[i][j];
- if(DEBUG) System.out.println(" gives "+vals[i][j]);
- }}
- CompensationChannels=TransToMatrix;
- CompensationMatrix=new Matrix(vals);
- //System.out.println(CompensationMatrix);
- if(DEBUG){
- System.out.println("..");
- System.out.println(" TODO: Make it dump matrix");
- //CompensationMatrix.print();
- System.out.println("..");
- }
- invCompensationMatrix= CompensationMatrix.inverse();
-
- if(DEBUG){
- System.out.println(" TODO: Make it dump matrix");
- System.out.println("..");
- }
- if(DEBUG) for(int j=0;j<ChannelNames.length;j++)System.out.println(j+": "+TransToMatrix[j]);
- }
- //////////////////////////////////////////////////////////////////////////////////
- // //
- // This is the main FCS file reader: //
- // //
- //////////////////////////////////////////////////////////////////////////////////
- public Stats ReadFCS(String Filename, boolean renorm) throws IOException {
- FileReader inputStream = null;
- FileWriter outputStream = null;
- FCSFile = new File(Filename);
- int baseLines[];
- boolean mBool=false;
- int Events,MyChannels;
- Stats DA=new Stats(0,0); ;
- resetDataSet();
-
- System.gc();
- Events=0;
- try {
- inputStream = new FileReader(Filename);
- File file = new File(Filename);
- InputStream is = new BufferedInputStream(new FileInputStream(file)); //Added buffering to make it load files much, much faster
- DataInputStream dis = new DataInputStream( is );
- float c =0;
- char sep;
- String str;
- //Read the "FCS"
- char incomingData[] = new char[58];
- inputStream.read(incomingData);
-
- str = new String(incomingData,0,3);
- str.toUpperCase();
- if(str.indexOf("FCS")==-1){
- if(DEBUG){
- System.out.println("!--Warning--!");
- System.out.println("First check failed, Probably not an FCS file");
- System.out.println("Fatal error, exiting");
- System.out.println("!-----------!");
- }
- mBool=false;
- } else {
- str = new String(incomingData,3,3);
- if(DEBUG) System.out.println(str);
- c=(float)Float.valueOf(str).floatValue();
- VersionF=c;
- VersionS=str;
- if(c!=2.0)
- if(DEBUG){
- System.out.println("!--Warning--!");
- System.out.println("FCS version"+(int)c+"not supported");
- System.out.println("This might cause problems");
- System.out.println("!-----------!");
- }
-
- //Read Start of text
- str = new String(incomingData,10,8);
- StartText=(int)Integer.valueOf(str.trim()).intValue();
- if(DEBUG) System.out.println(StartText);
- //Read End of text
- str = new String(incomingData,18,8);
- StopText=(int)Integer.valueOf(str.trim()).intValue() ;
- if(DEBUG) System.out.println(StopText);
- str = new String(incomingData,26,8);
- StartData=(int)Integer.valueOf(str.trim()).intValue() ;
- if(DEBUG) System.out.println(StartData);
- //Read End of text
- str = new String(incomingData,34,8);
- StopData=(int)Integer.valueOf(str.trim()).intValue() ;
- if(DEBUG) System.out.println(StopData);
- /////////////////////////////////////////////////
- // Start reading the text part of the FCS file //
- /////////////////////////////////////////////////
-
- //Read text from file
- char TextPart[] = new char[(StopText-StartText)];
- inputStream.skip(StartText-58);
- inputStream.read(TextPart);
- //Put it into String *Is this needed?*
- int l=TextPart.length;
- str = new String(TextPart,0,l);
- if(DEBUG) System.out.println(l);
-
- //Determine the separator and count the number of records
- sep=TextPart[0];
- if(DEBUG) System.out.println("Separator was defined as: "+sep);
- int NumBreak=0;
- for(int i=0; i<l; i++){
- if(TextPart[i]==sep) NumBreak++;
- }
- NumTextElements = (int)(NumBreak/2);
- if(DEBUG) System.out.println(NumTextElements);
-
- //Translating text part to array
- String TextArray[][] = new String[NumTextElements][2];
- int CurIndex = 0;
- int NewIndex;
- int SecondIndex;
-
- for(int i=0;i<NumTextElements;i++){
- NewIndex=str.indexOf(sep,CurIndex+1);
- SecondIndex=str.indexOf(sep,NewIndex+1);
- TextArray[i][0]=str.substring(CurIndex+1,NewIndex);
-
- if(SecondIndex!=-1){
- TextArray[i][1]=str.substring(NewIndex+1,SecondIndex);
- } else {
- TextArray[i][1]=str.substring(NewIndex+1);
- }
- CurIndex=SecondIndex;
- }
- //Reading the needed variables from array
- //First the variables independent of number of channels
-
- for(int b=0;b<4;b++)
- ByteOrd[b]=b;
- MyChannels=-1;
- DataType='I';
- String TMPstr;
- for(int i=0;i<NumTextElements;i++){
- if(StartData==0){
- if(TextArray[i][0].indexOf("BEGINDATA")!=-1){StartData=(int)Integer.valueOf(TextArray[i][1].trim()).intValue();}
- if(TextArray[i][0].indexOf("ENDDATA")!=-1){StopData=(int)Integer.valueOf(TextArray[i][1].trim()).intValue();}
- }
- if(TextArray[i][0].indexOf("$PAR")==0){ // ==0 was !=-1
- MyChannels=(int)Integer.valueOf(TextArray[i][1].trim()).intValue();
- if(DEBUG) System.out.println("Number of channels found: "+MyChannels);
- }
- if(TextArray[i][0].indexOf("DATATYPE")!=-1){DataType=TextArray[i][1].charAt(0);}
- if((TextArray[i][0].indexOf("P1B")!=-1)&&(TextArray[i][0].indexOf("P1BS")==-1)){
- NumBytes=(int)((int)Integer.valueOf(TextArray[i][1].trim()).intValue()/8);
- }
- if(TextArray[i][0].indexOf("CREATOR")!=-1) CreatorMachine=TextArray[i][1];
- if(TextArray[i][0].indexOf("$DATE")!=-1) DateCreated=TextArray[i][1];
- if(TextArray[i][0].indexOf("EXPORT TIME")!=-1) DateExport=TextArray[i][1];
- if(TextArray[i][0].indexOf("TOT")!=-1){Events=(int)Integer.valueOf(TextArray[i][1].trim()).intValue();}
- if(TextArray[i][0].indexOf("APPLY COMPENSATION")!=-1&&TextArray[i][1].toUpperCase().indexOf("TRUE")!=-1)
- {NeedCompensation=true;}
- if(TextArray[i][0].indexOf("BYTEORD")!=-1){
- if(TextArray[i][1].indexOf(",")!=-1){
- for(int b=0;b<4;b++)
- ByteOrd[b]=(int)Integer.valueOf(TextArray[i][1].substring((b*2),(b*2)+1)).intValue()-1;
- } else {
- for(int b=0;b<4;b++)
- ByteOrd[b]=b;
- }
- if(DEBUG) System.out.println("ByteOrd = "+ByteOrd[0]+ByteOrd[1]+ByteOrd[2]+ByteOrd[3]);
- }
- }
- //If the number of channels (given by $PAR) is not found, use this
- //emergency procedure to detect the number of channels.
- if(MyChannels==-1){
- MyChannels=1;
- boolean foundC=true;
- while(foundC==true){
- TMPstr = "$P"+Integer.toString(MyChannels)+"N";
- foundC=false;
- for(int i=0;i<NumTextElements;i++){
- if(TextArray[i][0].indexOf(TMPstr)!=-1){
- foundC=true;
- MyChannels++;
- }
- }
- }
- MyChannels--;
- }
-
-
- //Try to free some memory because after a short while we're going to load the data
- //This Needs some tweaking
-
- Object[] options = { "Yes", "No" };
- System.gc();
- Runtime rt = Runtime.getRuntime();
-
- long avMem=(int)((rt.maxMemory()-rt.totalMemory())*0.75);
- long needMem=(Events*(1+(MyChannels*6)+(27*4)));//+(MyChannels*(63+(25*20)))+1024; //63+(25*20)
- if(memDEBUG){
- System.out.println("----------------Before Loading--------------------------");
- System.out.println("Total memory allocated to VM: " + (int)(rt.totalMemory()/1024));
- System.out.println("Max memory allocated to VM: " + (int)(rt.maxMemory()/1024));
- System.out.println("Memory currently available: " + (int)(rt.freeMemory()/1024));
- System.out.println("Memory currently available: " + (int)(avMem/1024));
-
- System.out.println("Collecting garbage");
- System.gc();
- System.out.println("Memory currently available: " + (int)(rt.freeMemory()/1024));
- System.out.println("Estimated memory needed: "+(int)(needMem/1024));
- System.out.println("--------------------------------------------------------");
- } else {System.gc();}
-
- needMem=0; //temporary solution
- if(avMem<needMem)
- {
- int myNumEvents=(int)((avMem-1024-(MyChannels*(63+(20*25))))/(1+(MyChannels*6)+(27*4)));
-
- JOptionPane.showMessageDialog(null, "File too large.\n Memory available: "+
- Long.toString(avMem)+
- " KB, Needed: "+
- Long.toString(needMem)+
- " KB\nSetting events to "+
- Integer.toString(myNumEvents),
- "Memory Test",
- JOptionPane.WARNING_MESSAGE
- );
- Events=myNumEvents;
- }
- // Events = 300;//TEMPORARY
- int skippoints=0;
- int zeropoints=0;
- //int NewCN=MyChannels;
- /////////////////Loading channel dependent variables///////////////
- // This next bit loads the:
- // - Names
- // - Scales
- // - decades
- // - offsets
- // - lognorms
- // - ranges
- ///////////////////////////////////////////////////////////////////
- System.out.println("Using "+Events+" events and "+MyChannels+" channels");
- DA = new Stats(MyChannels,Events);
- LogNorm = new boolean[MyChannels];
- DA.logScale = new boolean[MyChannels];
- DA.decades = new float[MyChannels];
- DA.offsets = new float[MyChannels];
-
- for(int j=1;j<MyChannels+1;j++) LogNorm[j-1]=false;
- baseLines = new int[MyChannels];
-
- String tChannelNames[] = new String[MyChannels];
- boolean tLogNorm[] = new boolean[MyChannels];
- boolean tlogScale[] = new boolean[MyChannels];
- float tdecades[] = new float[MyChannels];
- float toffsets[] = new float[MyChannels];
- int tranges[] = new int[MyChannels];
- int tbaseLines[] = new int[MyChannels];
-
- for(int j=1;j<MyChannels+1;j++) tLogNorm[j-1]=false;
-
- for(int i=0;i<NumTextElements;i++){
- if(TextArray[i][0].indexOf("$P")!=-1){
- for(int j=1;j<MyChannels+1;j++) {
- TMPstr = "$P"+Integer.toString(j)+"N";
- if(TextArray[i][0].indexOf(TMPstr)!=-1){
- tChannelNames[j-1]=TextArray[i][1];
- }
- TMPstr = "$P"+Integer.toString(j)+"E";
- if(TextArray[i][0].indexOf(TMPstr)!=-1){
-
- if(TextArray[i][1].indexOf(",")!=-1){
- tdecades[j-1]=(float)Float.valueOf(TextArray[i][1].substring(0,TextArray[i][1].indexOf(","))).floatValue();
- toffsets[j-1]=(float)Float.valueOf(TextArray[i][1].substring(TextArray[i][1].indexOf(",")+1)).floatValue();
- if(tdecades[j-1]>0) tlogScale[j-1]=true; else tlogScale[j-1]=false;
- } else {
- if(DEBUG) System.out.println("No seperation between decades and offset found");
- }
- if(DEBUG) System.out.println("$P"+j+"E found: "+tdecades[j-1]+toffsets[j-1]);
- }
- TMPstr = "$P"+Integer.toString(j)+"R";
- if(TextArray[i][0].indexOf(TMPstr)!=-1){
- tranges[j-1]=(int)Integer.valueOf(TextArray[i][1]).intValue();
- }
- }
- }
- if(TextArray[i][0].indexOf("P")!=-1){
- for(int j=1;j<MyChannels+1;j++) {
- TMPstr = "P"+Integer.toString(j)+"DISPLAY";
- if(TextArray[i][0].indexOf(TMPstr)!=-1){
- if(TextArray[i][1].indexOf("LOG")!=-1){
- if(DEBUG) System.out.println("Log transform");
- tLogNorm[j-1]=true;
- tlogScale[j-1]=true;
- }
- }
- TMPstr = "P"+Integer.toString(j)+"BS";
- if(TextArray[i][0].indexOf(TMPstr)!=-1)
- tbaseLines[j-1]=(int)Integer.valueOf(TextArray[i][1]).intValue();
- }
- }
- }
- for(int i=0;i<NumTextElements;i++){
- if(TextArray[i][0].indexOf("$P")!=-1){
- for(int j=1;j<MyChannels+1;j++) {
- TMPstr = "$P"+Integer.toString(j)+"S";
- if(TextArray[i][0].indexOf(TMPstr)!=-1){
- tChannelNames[j-1]=tChannelNames[j-1]+" ( "+TextArray[i][1]+" )";
- }
- }
- }
- }
- for(int j=0;j<MyChannels;j++)
- if(tLogNorm[j]&&tdecades[j]==0){
- tdecades[j]=(float)(Math.log(tranges[j])/Math.log(10));
- if(DEBUG) System.out.println("Channel: "+j+", decades: "+tdecades[j]);
- }
- // If the number of channels are equal between datasets, the order is assumed to be the same
- // If not, then use the matcher to order the channels to match the previous dataset.
-
- int[] ChannelOrder;
- ChannelOrder = new int[MyChannels];
- System.out.println("ChannelOrderLength = "+MyChannels);
- for(int i=0;i<MyChannels;i++)
- ChannelOrder[i]=i;
- ChannelOrder = new int[MyChannels];
- for(int i=0;i<MyChannels;i++)
- ChannelOrder[i]=i;
-
-
- LogNorm = tLogNorm;
- DA.logScale = tlogScale;
- DA.decades = tdecades;
- DA.offsets = toffsets;
- DA.ranges = tranges;
- baseLines = tbaseLines;
- ChannelNames = tChannelNames;
- //}
- //Now that the names are known, load the Spillover Matrix
- for(int i=0;i<NumTextElements;i++){
- if(TextArray[i][0].indexOf("SPILL")!=-1){LoadSpillmatrix(TextArray[i][1]);}
- }
- if(BigDEBUG)
- for(int i=0;i<TextArray.length;i++){
- System.out.println(TextArray[i][0]+": "+TextArray[i][1]);
- }
- if(DEBUG) {
- for(int i=0;i<ChannelOrder.length;i++) System.out.println("nr "+i+" is "+ChannelOrder[i]);
- System.out.println("DataType is "+DataType);
- System.out.println("Data Part from "+StartData+" to "+StopData+" (Total "+DA.NumDataPoints+" datapoints in "+DA.NumEvents+" events and "+DA.NumChannels+" channels)");
- }
- // Start loading the data
- is.skip(StartData);
- float f=0;
- int myI=0;
- byte[] ch = new byte[NumBytes];
- int CountE=0;
- int CountT=0;
- for(int j=0;j<DA.NumEvents;j++){
- progress++; //Added by Kyle Thayer for a reporting progress
- CountE++;
- for(int i=0;i<MyChannels;i++){
- DA.DataArray[i][j]=0;
- }
- for(int i=0;i<MyChannels;i++){
- is.read(ch);
- if(ChannelOrder[i]<MyChannels&&ChannelOrder[i]>-1){
- if(DataType=='F') myI=(int)arr2float(ch, 0);
- if(DataType=='I') myI=arr2int(ch,NumBytes);
- DA.DataArray[ChannelOrder[i]][j]=myI;
- }
- CountT++;
- }
- mBool=true;
- }
-
- if(DEBUG) System.out.println("Data loaded. Events: "+CountE+", Total: "+CountT);
- if(memDEBUG){
- rt = Runtime.getRuntime();
- System.out.println("----------------After Loading----------------------------");
- System.out.println("Total memory allocated to VM: " + (int)(rt.totalMemory()/(1024)));
- System.out.println("Max memory allocated to VM: " + (int)(rt.maxMemory()/(1024)));
- System.out.println("Memory currently available: " + (int)(rt.freeMemory()/(1024)));
- System.out.println("Collecting garbage");
- System.gc();
- System.out.println("Memory currently available: " + (int)(rt.freeMemory()/(1024)));
- System.out.println("Estimated memory needed: "+(int)(needMem/1024));
- System.out.println("--------------------------------------------------------");
- }
- DA.MinAndMax(false);
- if(DEBUG)
- for(int i=0;i<MyChannels;i++){
- System.out.println("Channel "+i+": "+DA.min[i]+" to "+DA.max[i]);
- }
- DA.ChannelNames=ChannelNames;
- DA.CreatorMachine=CreatorMachine;
- DA.DateCreated=DateCreated;
- DA.DateExport=DateExport;
- DA.VersionS=VersionS;
- DA.VersionF=VersionF;
- DA.DataType=DataType;
-
- DA.CompOverRating();
- DA.NegativesToZero();
- DA.MinAndMax(false);
- DA.loaded=true;
- DA.ClusterNumbers=new char[DA.NumEvents];
- }
- } finally {
- if (inputStream != null) {
- inputStream.close();
- }
- if (outputStream != null) {
- outputStream.close();
- }
- }
- if(!mBool) DA = new Stats(0,0);
-
- return DA;
- }
- public int getProgress(){ //Added by Kyle Thayer for a reporting progress
- return progress;
- }
- }