Leave a Comment
11:25:58.779 [main] DEBUG o.a.h.h.p.d.s.DataTransferSaslUtil - DataTransferProtocol not using SaslPropertiesResolver, no QOP found in configuration for dfs.data.transfer.protection Exception in thread "main" java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCodeLoader.buildSupportsSnappy()Z at org.apache.hadoop.util.NativeCodeLoader.buildSupportsSnappy(Native Method) at org.apache.hadoop.io.compress.SnappyCodec.checkNativeCodeLoaded(SnappyCodec.java:63) at org.apache.hadoop.io.compress.SnappyCodec.getDecompressorType(SnappyCodec.java:192) at org.apache.hadoop.io.compress.CodecPool.getDecompressor(CodecPool.java:176) at org.apache.hadoop.io.compress.CompressionCodec$Util.createInputStreamWithCodecPool(CompressionCodec.java:157) at org.apache.hadoop.io.compress.SnappyCodec.createInputStream(SnappyCodec.java:162) at hadoopSnappyTest.readSnappy(hadoopSnappyTest.java:41) at hadoopSnappyTest.main(hadoopSnappyTest.java:19)
Sample Snappy read code:
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.SnappyCodec; import org.apache.hadoop.util.ReflectionUtils; import java.io.*; /** * Created by shiyanghuang on 17/11/10. */ public class hadoopSnappyTest { public static void main(String[] args) { String home = null; if (args.length >= 1) { home = args[0]; } try { readSnappy(home); } catch (IOException e) { e.printStackTrace(); } } private static void readSnappy(String home) throws IOException { Configuration conf = new Configuration(); if (home == null) { home = "/etc/hadoop/conf/"; } conf.addResource(new Path(home + "core-site.xml")); // Replace with actual path conf.addResource(new Path(home + "hdfs-site.xml")); // Replace with actual path conf.addResource(new Path(home + "mapred-site.xml")); // Replace with actual path conf.addResource(new Path(home + "yarn-site.xml")); // Replace with actual path FileSystem fs = null; fs = FileSystem.get(conf); Path path = new Path("/snappy/text.txt.snappy"); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(SnappyCodec.class, conf); InputStream inStream = codec.createInputStream(fs.open(path)); InputStreamReader inRead = new InputStreamReader(inStream); BufferedReader br = new BufferedReader(inRead); String line=null; while ((line = br.readLine()) != null){ System.out.println(line); } } }
Run the code with params:
java -cp .:/path/to/hadoop/lib -Djava.library.path=/path/to/libsnappy.so hadoopSnappyTest /etc/hadoop/conf/