Leave a Comment
11:25:58.779 [main] DEBUG o.a.h.h.p.d.s.DataTransferSaslUtil - DataTransferProtocol not using SaslPropertiesResolver, no QOP found in configuration for dfs.data.transfer.protection Exception in thread "main" java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCodeLoader.buildSupportsSnappy()Z at org.apache.hadoop.util.NativeCodeLoader.buildSupportsSnappy(Native Method) at org.apache.hadoop.io.compress.SnappyCodec.checkNativeCodeLoaded(SnappyCodec.java:63) at org.apache.hadoop.io.compress.SnappyCodec.getDecompressorType(SnappyCodec.java:192) at org.apache.hadoop.io.compress.CodecPool.getDecompressor(CodecPool.java:176) at org.apache.hadoop.io.compress.CompressionCodec$Util.createInputStreamWithCodecPool(CompressionCodec.java:157) at org.apache.hadoop.io.compress.SnappyCodec.createInputStream(SnappyCodec.java:162) at hadoopSnappyTest.readSnappy(hadoopSnappyTest.java:41) at hadoopSnappyTest.main(hadoopSnappyTest.java:19)
Sample Snappy read code:
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.SnappyCodec;
import org.apache.hadoop.util.ReflectionUtils;
import java.io.*;
/**
* Created by shiyanghuang on 17/11/10.
*/
public class hadoopSnappyTest {
public static void main(String[] args) {
String home = null;
if (args.length >= 1) {
home = args[0];
}
try {
readSnappy(home);
} catch (IOException e) {
e.printStackTrace();
}
}
private static void readSnappy(String home) throws IOException {
Configuration conf = new Configuration();
if (home == null) {
home = "/etc/hadoop/conf/";
}
conf.addResource(new Path(home + "core-site.xml")); // Replace with actual path
conf.addResource(new Path(home + "hdfs-site.xml")); // Replace with actual path
conf.addResource(new Path(home + "mapred-site.xml")); // Replace with actual path
conf.addResource(new Path(home + "yarn-site.xml")); // Replace with actual path
FileSystem fs = null;
fs = FileSystem.get(conf);
Path path = new Path("/snappy/text.txt.snappy");
CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(SnappyCodec.class, conf);
InputStream inStream = codec.createInputStream(fs.open(path));
InputStreamReader inRead = new InputStreamReader(inStream);
BufferedReader br = new BufferedReader(inRead);
String line=null;
while ((line = br.readLine()) != null){
System.out.println(line);
}
}
}
Run the code with params:
java -cp .:/path/to/hadoop/lib -Djava.library.path=/path/to/libsnappy.so hadoopSnappyTest /etc/hadoop/conf/