spark记录（20）自定义累加器Accumulator

自定义累加器

/**
 * 自定义累加器需要继承AccumulatorV2<IN,OUT>类
 * 并且要指定要累加的类型
 */
public class MyAccumulator extends AccumulatorV2<MyKey,MyKey> implements Serializable {

    private MyKey info = new MyKey(0, 0);

    public MyKey getInfo() {
        return info;
    }

    public void setInfo(MyKey info) {
        this.info = info;
    }

    //判断是否是初始状态，直接与原始状态的值比较
    @Override
    public boolean isZero() {
        return info.getPersonAgeSum()==0 && info.getPersonNum()==0;
    }

    //复制一个新的累加器
    @Override
    public AccumulatorV2<MyKey, MyKey> copy() {
        MyAccumulator myAccumulator = new MyAccumulator();
        myAccumulator.info = this.info;
        return myAccumulator;
    }

    //初始化不同的partition分区中的累加类型
    @Override
    public void reset() {
        info = new MyKey(0, 0);
    }

    //进行累加时以何种规则进行累加
    @Override
    public void add(MyKey v) {
        info.setPersonNum(info.getPersonNum() + v.getPersonNum());
        info.setPersonAgeSum(info.getPersonAgeSum() + v.getPersonAgeSum());
    }

    //合并不同partition分区中accumulator中储存的状态值
    @Override
    public void merge(AccumulatorV2<MyKey, MyKey> other) {
        MyKey value = other.value();
        info.setPersonNum(info.getPersonNum()+value.getPersonNum());
        info.setPersonAgeSum(info.getPersonAgeSum()+value.getPersonAgeSum());
    }

    //最后返回的状态值
    @Override
    public MyKey value() {
        return info;
    }
}

自定义key

public class MyKey implements Serializable {
    private Integer personNum;
    private Integer personAgeSum;

    public MyKey() {
    }

    public MyKey(Integer personNum, Integer personAgeSum) {
        this.personNum = personNum;
        this.personAgeSum = personAgeSum;
    }

    public Integer getPersonNum() {
        return personNum;
    }

    public void setPersonNum(Integer personNum) {
        this.personNum = personNum;
    }

    public Integer getPersonAgeSum() {
        return personAgeSum;
    }

    public void setPersonAgeSum(Integer personAgeSum) {
        this.personAgeSum = personAgeSum;
    }

    @Override
    public String toString() {
        return "MyKey{" +
                "personNum=" + personNum +
                ", personAgeSum=" + personAgeSum +
                '}';
    }
}

 

来源：https://www.cnblogs.com/kpsmile/p/11801722.html