1)MyRedisAppendSink
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.api.java.io.redis;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.typeutils.RowTypeInfo;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.table.sinks.AppendStreamTableSink;
import org.apache.flink.table.sinks.BatchTableSink;
import org.apache.flink.table.sinks.TableSink;
import org.apache.flink.table.util.TableConnectorUtil;
import org.apache.flink.types.Row;
import org.apache.flink.util.InstantiationUtil;
import java.io.IOException;
@SuppressWarnings("rawtypes")
public class MyRedisAppendSink implements AppendStreamTableSink<Row>, BatchTableSink<Row> {
private final MyRedisOutputFormat outputFormat;
private String[] fieldNames;
private TypeInformation[] fieldTypes;
MyRedisAppendSink(MyRedisOutputFormat outputFormat) {
this.outputFormat = outputFormat;
}
public static MyRedisAppendSinkBuilder builder() {
return new MyRedisAppendSinkBuilder();
}
@Override
public void emitDataStream(DataStream<Row> dataStream) {
dataStream.addSink(new MyRedisSinkFunction(outputFormat))
.name(TableConnectorUtil.generateRuntimeName(this.getClass(), fieldNames));
}
@Override
public void emitDataSet(DataSet<Row> dataSet) {
dataSet.output(outputFormat);
}
@Override
public TypeInformation<Row> getOutputType() {
return new RowTypeInfo(fieldTypes, fieldNames);
}
@Override
public String[] getFieldNames() {
return fieldNames;
}
@Override
public TypeInformation<?>[] getFieldTypes() {
return fieldTypes;
}
@Override
public TableSink<Row> configure(String[] fieldNames, TypeInformation<?>[] fieldTypes) {
//直接拷贝
MyRedisAppendSink copy;
try {
copy = new MyRedisAppendSink(InstantiationUtil.clone(outputFormat));
} catch (IOException | ClassNotFoundException e) {
throw new RuntimeException(e);
}
copy.fieldNames = fieldNames;
copy.fieldTypes = fieldTypes;
return copy;
}
@VisibleForTesting
MyRedisOutputFormat getOutputFormat() {
return outputFormat;
}
}
2)MyRedisAppendSinkBuilder
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.api.java.io.redis;
import com.google.common.base.Preconditions;
/**
* A builder to configure and build the RedisAppendSink.
*/
public class MyRedisAppendSinkBuilder {
private String host;
private String port;
private String password;
private String command;
private String param;
public MyRedisAppendSinkBuilder setParam(String p) {
this.param = p;
return this;
}
public MyRedisAppendSinkBuilder setHost(String param) {
this.host = param;
return this;
}
public MyRedisAppendSinkBuilder setPort(String param) {
this.port = param;
return this;
}
public MyRedisAppendSinkBuilder setPassword(String param) {
this.password = param;
return this;
}
public MyRedisAppendSinkBuilder setCommand(String c) {
this.command = c;
return this;
}
/**
* Finalizes the configuration and checks validity.
*
* @return Configured JDBCOutputFormat
*/
public MyRedisAppendSink build() {
Preconditions.checkNotNull(host, "host are not specified.");
Preconditions.checkNotNull(port, "port are not specified.");
Preconditions.checkNotNull(command, "command are not specified.");
Preconditions.checkNotNull(param, "param are not specified.");
MyRedisOutputFormat format = MyRedisOutputFormat.buildJDBCOutputFormat().setHost(host)
.setPort(port).setPassword(password).setCommand(command).setParam(param).finish();
return new MyRedisAppendSink(format);
}
}
3)MyRedisOutputFormat
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.api.java.io.redis;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.apache.flink.api.common.io.RichOutputFormat;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.types.Row;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.dianping.lion.client.ConfigCache;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisCluster;
import java.io.IOException;
import java.text.NumberFormat;
import java.util.HashSet;
import java.util.Set;
/**
* MyOutputFormat to write Rows into a JDBC database.
* The OutputFormat has to be configured using the supplied OutputFormatBuilder.
*
* @see Row
* @see DriverManager
* @Test:亿条测试通过
*/
public class MyRedisOutputFormat extends RichOutputFormat<Row> {
/** */
private static final long serialVersionUID = -8399998414680248843L;
private static final Logger LOG = LoggerFactory
.getLogger(MyRedisOutputFormat.class);
//private int[] typesArray;
private String host;
private String port;
private String password;
private String command;
private int keyCount = -1;
private String param;
//
private JedisCluster jedisCluster;
public MyRedisOutputFormat() {
}
@Override
public void configure(Configuration parameters) {
LOG.info("configure function invoked , thread -[{}]", Thread.currentThread().getName());
}
/**
* Connects to the target database and initializes the prepared statement.
*
* @param taskNumber The number of the parallel instance.
* @throws IOException Thrown, if the output could not be opened due to an
* I/O problem.
*/
@Override
public void open(int taskNumber, int numTasks) throws IOException {
//
Set<HostAndPort> hostSet = new HashSet<HostAndPort>();
//
String lionHost = ConfigCache.getInstance().getProperty(host.trim());
String[] hostArray = lionHost.split(",");
Integer portInt = ConfigCache.getInstance().getIntProperty(port.trim());
//
for (String host : hostArray) {
hostSet.add(new HostAndPort(host, portInt));
}
//
GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig();
poolConfig.setMaxTotal(3);
poolConfig.setMaxIdle(3);
poolConfig.setMinIdle(3);
poolConfig.setMaxWaitMillis(3000);
//没有密码
if (null == this.password || 0 == this.password.trim().length()) {
jedisCluster = new JedisCluster(hostSet, 6000, 10, poolConfig);
} else {//设置了密码
jedisCluster = new JedisCluster(hostSet, 6000, 6000, 10, password.trim(), poolConfig);
}
LOG.info(
"open function invoked , taskNumber-[{}], numTasks-[{}], host-[{}] , port-[{}] , thread -[{}]",
taskNumber, numTasks, host, port, Thread.currentThread().getName());
}
/**
* Adds a record to the prepared statement.
*
* <p>When this method is called, the output format is guaranteed to be opened.
*
* <p>WARNING: this may fail when no column types specified (because a best effort approach is attempted in order to
* insert a null value but it's not guaranteed that the JDBC driver handles PreparedStatement.setObject(pos, null))
*
* @param row The records to add to the output.
* @see PreparedStatement
* @throws IOException Thrown, if the records could not be added due to an I/O problem.
*/
private String field2Str(Object obj) {
if (null == obj) {
return "null";
}
//先写这么多,后面有需要再补充
if (obj instanceof java.lang.Byte) {
return Byte.toString((Byte) obj);
} else if (obj instanceof java.lang.Integer) {
return Integer.toString((Integer) obj);
} else if (obj instanceof java.lang.Long) {
return Long.toString((Long) obj);
} else if (obj instanceof java.lang.Float) {
return Float.toString((Float) obj);
} else if (obj instanceof java.lang.Double) {
return Double.toString((Double) obj);
} else if (obj instanceof java.sql.Timestamp) {
return "" + ((java.sql.Timestamp) obj).getTime();
} else {
return obj.toString();
}
}
@Override
public void writeRecord(Row row) throws IOException {
//参数替换
String paramTemplate = this.param;
for (int index = 1; index <= row.getArity(); index++) {
String str;
Object field = row.getField(index - 1);
//LOG.info("field type ->" + field.getClass());
str = field2Str(field);
//替换
paramTemplate = paramTemplate.replaceAll("[$]\\{" + (index) + "\\}", str);
}
String[] evalParams = paramTemplate.split(" ");
//执行
//https://help.aliyun.com/document_detail/55684.html?spm=5176.8961142.0.0.2d7d5b8bnK2I3W#h2-url-4
//具体写法参考了文章https://github.com/StackExchange/StackExchange.Redis/issues/305
Object obj = this.jedisCluster.eval(this.command, this.keyCount, evalParams);
LOG.info("command {} keycount {} evalParams {} result {}", command, keyCount, evalParams,
obj);
}
void flush() {
}
// public int[] getTypesArray() {
// return typesArray;
// }
/**
* Executes closes all resources of this instance.
*
* @throws IOException Thrown, if the input could not be closed properly.
*/
@Override
public void close() throws IOException {
if (null != jedisCluster) {
try {
jedisCluster.close();
jedisCluster = null;
} catch (Exception e) {
}
}
}
public static MyRedisOutputFormatBuilder buildJDBCOutputFormat() {
return new MyRedisOutputFormatBuilder();
}
/**
* Builder for a {@link JDBCOutputFormat}.
*/
public static class MyRedisOutputFormatBuilder {
private final MyRedisOutputFormat format;
protected MyRedisOutputFormatBuilder() {
this.format = new MyRedisOutputFormat();
}
public MyRedisOutputFormatBuilder setHost(String param) {
format.host = param;
return this;
}
public MyRedisOutputFormatBuilder setPort(String param) {
format.port = param;
return this;
}
public MyRedisOutputFormatBuilder setPassword(String param) {
format.password = param;
return this;
}
public MyRedisOutputFormatBuilder setParam(String p) {
format.param = p;
return this;
}
public MyRedisOutputFormatBuilder setCommand(String c) {
if (null == c || c.trim().length() == 0) {
throw new IllegalArgumentException("No command supplied.");
}
format.command = c;
//这里先确定key的个数,如果每次都重新算就浪费时间
int keyCount = 0;
for (int count = 1; count > 0; count++) {
int index = c.indexOf("KEYS[" + count + "]");
if (-1 != index) {
keyCount = count;
} else {
break;
}
}
format.keyCount = keyCount;
return this;
}
/**
* Finalizes the configuration and checks validity.
*
* @return Configured MyJDBCOutputFormat
*/
public MyRedisOutputFormat finish() {
if (format.host == null || 0 == format.host.trim().length()) {
throw new IllegalArgumentException("No host supplied.");
}
if (format.port == null || 0 == format.port.trim().length()) {
throw new IllegalArgumentException("No port supplied.");
}
if (format.command == null || 0 == format.command.trim().length()) {
throw new IllegalArgumentException("No command supplied.");
}
return format;
}
}
}
4)MyRedisSinkFunction
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.api.java.io.redis;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.state.FunctionInitializationContext;
import org.apache.flink.runtime.state.FunctionSnapshotContext;
import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.flink.types.Row;
class MyRedisSinkFunction extends RichSinkFunction<Row> implements CheckpointedFunction {
/** */
private static final long serialVersionUID = 1048743703192892444L;
final MyRedisOutputFormat outputFormat;
MyRedisSinkFunction(MyRedisOutputFormat outputFormat) {
this.outputFormat = outputFormat;
}
@Override
public void invoke(Row value) throws Exception {
outputFormat.writeRecord(value);
}
@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
outputFormat.flush();
}
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
}
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
RuntimeContext ctx = getRuntimeContext();
outputFormat.setRuntimeContext(ctx);
outputFormat.open(ctx.getIndexOfThisSubtask(), ctx.getNumberOfParallelSubtasks());
}
@Override
public void close() throws Exception {
outputFormat.close();
super.close();
}
}
来源：oschina
链接：https://my.oschina.net/u/1382024/blog/2874281