+0
Daily
26,473
stars
14,184
forks
896
watchers
29,939
prs
0
issues
0
releases
README
Apache Flink
Apache Flink is an open source stream processing framework with powerful stream- and batch-processing capabilities.
Learn more about Flink at https://flink.apache.org/
Features
A streaming-first runtime that supports both batch processing and data streaming programs
Elegant and fluent APIs in Java
A runtime that supports very high throughput and low event latency at the same time
Support for event time and out-of-order processing in the DataStream API, based on the Dataflow Model
Flexible windowing (time, count, sessions, custom triggers) across different time semantics (event time, processing time)
Fault-tolerance with exactly-once processing guarantees
Natural back-pressure in streaming programs
Libraries for Graph processing (batch), Machine Learning (batch), and Complex Event Processing (streaming)
Custom memory management for efficient and robust switching between in-memory and out-of-core data processing algorithms
Compatibility layers for Apache Hadoop MapReduce
Integration with YARN, HDFS, HBase, and other components of the Apache Hadoop ecosystem
Streaming Example
java
// pojo class WordWithCount
public class WordWithCount {
public String word;
public int count;
public WordWithCount() {}
public WordWithCount(String word, int count) {
this.word = word;
this.count = count;
}
}
// main method
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStreamSource text = env.socketTextStream(host, port);
DataStream windowCounts = text
.flatMap(
(FlatMapFunction) (line, collector)
- Arrays.stream(line.split("\\s")).forEach(collector::collect)
).returns(String.class)
.map(word - new WordWithCount(word, 1)).returns(TypeInformation.of(WordWithCount.class))
.keyBy(wordWithCnt - wordWithCnt.word)
.window(TumblingProcessingTimeWindows.of(Duration.ofSeconds(5)))
.sum("count").returns(TypeInformation.of(WordWithCount.class));
windowCounts.print();
env.execute();
}
Batch Example
java
// pojo class WordWithCount
public class WordWithCount {
public String word;
public int count;
public WordWithCount() {}
public WordWithCount(String word, int count) {
this.word = word;
this.count = count;
}
}
// main method
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRuntimeMode(RuntimeExecutionMode.BATCH);
FileSource source = FileSource.forRecordStreamFormat(new TextLineInputFormat(), new Path("MyInput.txt")).build();
DataStreamSource text = env.fromSource(source, WatermarkStrategy.noWatermarks(), "MySource");
DataStream windowCounts = text
.flatMap((FlatMapFunction) (line, collector) - Arrays
.stream(line.split("\\s"))
.forEach(collector::collect)).returns(String.class)
.map(word - new WordWithCount(word, 1)).returns(TypeInformation.of(WordWithCount.class))
.keyB