Java读写avro例子

https://www.cnblogs.com/fillPv/p/5009737.html

avro是一个数据序列化框架,可以高效得进行序列化和反序列化,支持C, C++, C#, Java, PHP, Python, 和Ruby语言。现在使用Java来读写。

环境搭建

  1、下载avro-1.7.7.jar and avro-tools-1.7.7.jar两个jar包,放到指定文件目录。下载地址 http://www.trieuvan.com/apache/avro/avro-1.7.7/java/

    我放到了D:\soft\avro 文件夹,在改目录下新建java文件夹,用来存放生成的Java代码

   2、该目录下新建user.avsc文件,内容是:  

user.avsc

{
  "type": "record",
  "name": "User",
  "namespace":"com.test.avro",
  "aliases": ["User1","User2"],
  "fields" : [
    {"name": "name", "type": "string"},
    {"name": "age", "type": "int","default":10},
    {"name": "email", "type": "string"}
  ]
}

打开cmd,进入到该目录,执行命令生成User类,注意命令后面有个”.”,表示生成的代码放在本目录下。

java -jar avro-tools-1.7.7.jar compile schema user.avsc java .
---result---
Input files to compile:
  user.avsc

在该文件夹下的Java文件下的../example/avro/目录下就会生成User.java文件。

使用eclipse新建maven项目,在pom.xml加入avro的依赖。 

<dependency>
  <groupId>org.apache.avro</groupId>
  <artifactId>avro</artifactId>
  <version>1.7.7</version>
</dependency>

生成的User.java文件的内容如下。

  把生成的User.java类复制到工程中,注意这个User.java里面生成的User类及其内部类的包名默认是user.avsc文件中的namespace的值,

User.java

package com.paloaltonetworks.schemaregistry.serializers;

/**
 * Autogenerated by Avro
 *
 * DO NOT EDIT DIRECTLY
 */
@SuppressWarnings("all")
@org.apache.avro.specific.AvroGenerated
public class User extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
    public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"example.avro\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"age\",\"type\":\"int\"},{\"name\":\"email\",\"type\":\"string\"}]}");
    public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
    @Deprecated public java.lang.CharSequence name;
    @Deprecated public java.lang.Integer age;
    @Deprecated public java.lang.CharSequence email;

    /**
     * Default constructor.  Note that this does not initialize fields
     * to their default values from the schema.  If that is desired then
     * one should use <code>newBuilder()</code>. 
     */
    public User() {}

    /**
     * All-args constructor.
     */
    public User(java.lang.CharSequence name, java.lang.Integer age, java.lang.CharSequence email) {
        this.name = name;
        this.age = age;
        this.email = email;
    }

    public org.apache.avro.Schema getSchema() { return SCHEMA$; }
    // Used by DatumWriter.  Applications should not call. 
    public java.lang.Object get(int field$) {
        switch (field$) {
            case 0: return name;
            case 1: return age;
            case 2: return email;
            default: throw new org.apache.avro.AvroRuntimeException("Bad index");
        }
    }
    // Used by DatumReader.  Applications should not call. 
    @SuppressWarnings(value="unchecked")
    public void put(int field$, java.lang.Object value$) {
        switch (field$) {
            case 0: name = (java.lang.CharSequence)value$; break;
            case 1: age = (java.lang.Integer)value$; break;
            case 2: email = (java.lang.CharSequence)value$; break;
            default: throw new org.apache.avro.AvroRuntimeException("Bad index");
        }
    }

    /**
     * Gets the value of the 'name' field.
     */
    public java.lang.CharSequence getName() {
        return name;
    }

    /**
     * Sets the value of the 'name' field.
     * @param value the value to set.
     */
    public void setName(java.lang.CharSequence value) {
        this.name = value;
    }

    /**
     * Gets the value of the 'age' field.
     */
    public java.lang.Integer getAge() {
        return age;
    }

    /**
     * Sets the value of the 'age' field.
     * @param value the value to set.
     */
    public void setAge(java.lang.Integer value) {
        this.age = value;
    }

    /**
     * Gets the value of the 'email' field.
     */
    public java.lang.CharSequence getEmail() {
        return email;
    }

    /**
     * Sets the value of the 'email' field.
     * @param value the value to set.
     */
    public void setEmail(java.lang.CharSequence value) {
        this.email = value;
    }

    /** Creates a new User RecordBuilder */
    public static User.Builder newBuilder() {
        return new User.Builder();
    }

    /** Creates a new User RecordBuilder by copying an existing Builder */
    public static User.Builder newBuilder(User.Builder other) {
        return new User.Builder(other);
    }

    /** Creates a new User RecordBuilder by copying an existing User instance */
    public static User.Builder newBuilder(User other) {
        return new User.Builder(other);
    }

    /**
     * RecordBuilder for User instances.
     */
    public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<User>
            implements org.apache.avro.data.RecordBuilder<User> {

        private java.lang.CharSequence name;
        private java.lang.Integer age;
        private java.lang.CharSequence email;

        /** Creates a new Builder */
        private Builder() {
            super(User.SCHEMA$);
        }

        /** Creates a Builder by copying an existing Builder */
        private Builder(User.Builder other) {
            super(other);
            if (isValidValue(fields()[0], other.name)) {
                this.name = data().deepCopy(fields()[0].schema(), other.name);
                fieldSetFlags()[0] = true;
            }
            if (isValidValue(fields()[1], other.age)) {
                this.age = data().deepCopy(fields()[1].schema(), other.age);
                fieldSetFlags()[1] = true;
            }
            if (isValidValue(fields()[2], other.email)) {
                this.email = data().deepCopy(fields()[2].schema(), other.email);
                fieldSetFlags()[2] = true;
            }
        }

        /** Creates a Builder by copying an existing User instance */
        private Builder(User other) {
            super(User.SCHEMA$);
            if (isValidValue(fields()[0], other.name)) {
                this.name = data().deepCopy(fields()[0].schema(), other.name);
                fieldSetFlags()[0] = true;
            }
            if (isValidValue(fields()[1], other.age)) {
                this.age = data().deepCopy(fields()[1].schema(), other.age);
                fieldSetFlags()[1] = true;
            }
            if (isValidValue(fields()[2], other.email)) {
                this.email = data().deepCopy(fields()[2].schema(), other.email);
                fieldSetFlags()[2] = true;
            }
        }

        /** Gets the value of the 'name' field */
        public java.lang.CharSequence getName() {
            return name;
        }

        /** Sets the value of the 'name' field */
        public User.Builder setName(java.lang.CharSequence value) {
            validate(fields()[0], value);
            this.name = value;
            fieldSetFlags()[0] = true;
            return this;
        }

        /** Checks whether the 'name' field has been set */
        public boolean hasName() {
            return fieldSetFlags()[0];
        }

        /** Clears the value of the 'name' field */
        public User.Builder clearName() {
            name = null;
            fieldSetFlags()[0] = false;
            return this;
        }

        /** Gets the value of the 'age' field */
        public java.lang.Integer getAge() {
            return age;
        }

        /** Sets the value of the 'age' field */
        public User.Builder setAge(java.lang.Integer value) {
            validate(fields()[1], value);
            this.age = value;
            fieldSetFlags()[1] = true;
            return this;
        }

        /** Checks whether the 'age' field has been set */
        public boolean hasAge() {
            return fieldSetFlags()[1];
        }

        /** Clears the value of the 'age' field */
        public User.Builder clearAge() {
            email = null;
            fieldSetFlags()[1] = false;
            return this;
        }

        /** Gets the value of the 'email' field */
        public java.lang.CharSequence getEmail() {
            return email;
        }

        /** Sets the value of the 'email' field */
        public User.Builder setEmail(java.lang.CharSequence value) {
            validate(fields()[2], value);
            this.email = value;
            fieldSetFlags()[2] = true;
            return this;
        }

        /** Checks whether the 'email' field has been set */
        public boolean hasEmail() {
            return fieldSetFlags()[2];
        }

        /** Clears the value of the 'email' field */
        public User.Builder clearEmail() {
            email = null;
            fieldSetFlags()[2] = false;
            return this;
        }

        @Override
        public User build() {
            try {
                User record = new User();
                record.name = fieldSetFlags()[0] ? this.name : (java.lang.CharSequence) defaultValue(fields()[0]);
                record.age = fieldSetFlags()[1] ? this.age : (java.lang.Integer) defaultValue(fields()[1]);
                record.email = fieldSetFlags()[2] ? this.email : (java.lang.CharSequence) defaultValue(fields()[2]);
                return record;
            } catch (Exception e) {
                throw new org.apache.avro.AvroRuntimeException(e);
            }
        }
    }
}

Main.java

import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.io.BinaryEncoder;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.specific.SpecificDatumWriter;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;

public class Main {
    public static void main(String[] args) throws IOException {
         // 声明并初始化User对象

        String path = "/Users/shahuang/Documents/git/wf3lab/schemaregistry/javaclient/data/user2.avro"; // avro文件存放目录

        // 方式一
        User user1 = new User();
        user1.setName("zhangsan");
        user1.setAge(21);
        user1.setEmail("email");

        // 方式二 使用构造函数
        // Alternate constructor
        User user2 = new User("Ben", 7, "red");

        // 方式三,使用Build方式
        // Construct via builder
        User user3 = User.newBuilder()
                .setName("Charlie")
                .setEmail("blue")
                .setAge(22)
                .build();

        // byte[] data = datafwriter(user1);
        byte[] data = Bencode(user1);

        for (byte x : data) {
            System.out.print((char)x);
        }

        DatumReader<User> reader = new SpecificDatumReader<User>(User.class);
        DataFileReader<User> dataFileReader = new DataFileReader<User>(new File(path), reader);
        User user = null;
        while (dataFileReader.hasNext()) {
            user = dataFileReader.next();
            System.out.println(user);
        }
    }

    // 只serialize data到byte[]
    public static byte[] datafwriter(User user1) throws IOException {
        DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);
        DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);
        ByteArrayOutputStream bout = new ByteArrayOutputStream();

        dataFileWriter.create(user1.getSchema(),  bout);
        // 把生成的user对象写入到avro文件
        dataFileWriter.append(user1);
        dataFileWriter.close();
        byte[] data = bout.toByteArray();
        return data;
    }

    // serialize data and add schema at front
    public static byte[] Bencode(User msg) throws IOException {
        DatumWriter<User> writer;
        ByteArrayOutputStream baos;
        writer = new SpecificDatumWriter<User>(User.getClassSchema());
        baos = new ByteArrayOutputStream();
        baos.reset();
        BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(baos, null);

        writer.write(msg, encoder);
        encoder.flush();
        baos.flush();
        return baos.toByteArray();
    }
}

Result

{"name": "zhangsan", "age": 21, "email": "email"}
{"name": "Ben", "age": 7, "email": "red"}
{"name": "Charlie", "age": 22, "email": "blue"}

Leave a Reply

Your email address will not be published. Required fields are marked *