//to run in eclipse, add jars below:
//- all jars in $HADOOP_HOME
//- all in $HADOOP_HOME/lib

//USAGE:
//- convert batch query result folders to result files

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;

class SubFolders2NewFiles {
	
	public static void main(String[] args) {
		Configuration conf = new Configuration();
		conf.set("fs.default.name", "hdfs://master:9000");
		String srcPath ="hdfs://master:9000/ol_out/";
		String dstPath="hdfs://master:9000/ol_matrix/";
		try {
		    FileSystem fs = FileSystem.get(conf);
		    FileStatus[] status = fs.listStatus(new Path(srcPath));
		    int fid=1;
		    for(int i = 0 ;i < status.length ; i ++)
		    {
				Path srcfilePath = status[i].getPath();
				Path dstfilePath = new Path(dstPath + "part_"+fid);
				fid++;
				FileUtil.copyMerge(fs, srcfilePath, fs, dstfilePath, false, conf, null);
				System.out.println(srcfilePath+"  processed");
		    }
		}
		catch (IOException ioe) {
			ioe.printStackTrace();
		}
    }
}