java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCodeLoader.buildSupportsSnappy

11:25:58.779 [main] DEBUG o.a.h.h.p.d.s.DataTransferSaslUtil - DataTransferProtocol not using SaslPropertiesResolver, no QOP found in configuration for dfs.data.transfer.protection
Exception in thread "main" java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCodeLoader.buildSupportsSnappy()Z
	at org.apache.hadoop.util.NativeCodeLoader.buildSupportsSnappy(Native Method)
	at org.apache.hadoop.io.compress.SnappyCodec.checkNativeCodeLoaded(SnappyCodec.java:63)
	at org.apache.hadoop.io.compress.SnappyCodec.getDecompressorType(SnappyCodec.java:192)
	at org.apache.hadoop.io.compress.CodecPool.getDecompressor(CodecPool.java:176)
	at org.apache.hadoop.io.compress.CompressionCodec$Util.createInputStreamWithCodecPool(CompressionCodec.java:157)
	at org.apache.hadoop.io.compress.SnappyCodec.createInputStream(SnappyCodec.java:162)
	at hadoopSnappyTest.readSnappy(hadoopSnappyTest.java:41)
	at hadoopSnappyTest.main(hadoopSnappyTest.java:19)

Sample Snappy read code:

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.SnappyCodec;
import org.apache.hadoop.util.ReflectionUtils;

import java.io.*;

/**
 * Created by shiyanghuang on 17/11/10.
 */
public class hadoopSnappyTest {
    public static void main(String[] args) {
        String home = null;
        if (args.length >= 1) {
            home = args[0];
        }
        try {
            readSnappy(home);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private static void readSnappy(String home) throws IOException {
        Configuration conf = new Configuration();
        if (home == null) {
            home = "/etc/hadoop/conf/";
        }
        conf.addResource(new Path(home + "core-site.xml")); // Replace with actual path
        conf.addResource(new Path(home + "hdfs-site.xml")); // Replace with actual path
        conf.addResource(new Path(home + "mapred-site.xml")); // Replace with actual path
        conf.addResource(new Path(home + "yarn-site.xml")); // Replace with actual path

        FileSystem fs = null;
        fs = FileSystem.get(conf);

        Path path = new Path("/snappy/text.txt.snappy");
        CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(SnappyCodec.class, conf);
        InputStream inStream = codec.createInputStream(fs.open(path));
        InputStreamReader inRead = new InputStreamReader(inStream);
        BufferedReader br = new BufferedReader(inRead);
        String line=null;
        while ((line = br.readLine()) != null){
            System.out.println(line);
        }
    }
}

Run the code with params:

java -cp .:/path/to/hadoop/lib -Djava.library.path=/path/to/libsnappy.so hadoopSnappyTest /etc/hadoop/conf/

Leave a Reply

Your email address will not be published. Required fields are marked *