Skip to content

Commit

Permalink
Merge pull request #20 from PastorGL/feature_exp-functions
Browse files Browse the repository at this point in the history 
Expression functions in TDL
  • Loading branch information
@PastorGL
PastorGL authored Jan 31, 2025
2 parents 2a7ff40 + f797a0a commit de1be52
Show file tree
Hide file tree
Showing 5 changed files with 288 additions and 162 deletions.
56 changes: 36 additions & 20 deletions datacooker-commons/src/main/antlr4/io/github/pastorgl/datacooker/scripting/TDL4.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ type_columns
;

key_item
: K_KEY attr_expr
: K_KEY expression
;

copy_stmt
Expand All @@ -49,7 +49,7 @@ params_expr
;

param
: S_AT L_IDENTIFIER S_EQ attr_expr
: S_AT L_IDENTIFIER S_EQ expression
;

select_stmt
Expand All @@ -64,19 +64,15 @@ limit_expr
;

what_expr
: attr_expr ( K_AS type_alias? alias )?
: expression ( K_AS type_alias? alias )?
;

alias
: L_IDENTIFIER
;

expression
: ( is_op | between_op | in_op | kw_op | sym_op | var_name | literal | func_call | array )+
;

attr_expr
: ( is_op | between_op | in_op | kw_op | sym_op | var_name | literal | func_attr | array | attr )+
: ( is_op | between_op | in_op | kw_op | sym_op | var_name | literal | func_call | array | attr )+
;

func_call
Expand All @@ -85,12 +81,6 @@ func_call
| S_OPEN_PAR expression S_CLOSE_PAR
;

func_attr
: func S_OPEN_PAR attr_expr ( S_COMMA attr_expr )* S_CLOSE_PAR
| func S_OPEN_PAR S_CLOSE_PAR
| S_OPEN_PAR attr_expr S_CLOSE_PAR
;

type_alias
: T_POINT | T_POLYGON | T_SEGMENT | T_TRACK | T_VALUE
;
Expand Down Expand Up @@ -119,7 +109,7 @@ ds_name
;

where_expr
: type_alias? attr_expr
: type_alias? expression
;

call_stmt
Expand Down Expand Up @@ -163,18 +153,26 @@ ds_alias
;

let_stmt
: K_LET var_name S_EQ expression
| K_LET var_name S_EQ sub_query
: K_LET? var_name S_EQ expression
| K_LET? var_name S_EQ sub_query
;

sub_query
: K_SELECT K_DISTINCT? what_expr K_FROM ds_parts ( K_WHERE where_expr )? ( K_LIMIT limit_expr )?
;

let_func
: K_LET? var_name S_EQ expression
;

loop_stmt
: K_LOOP var_name S_IN? expression K_BEGIN statements ( K_ELSE statements )? K_END K_LOOP?
;

loop_func
: K_LOOP var_name S_IN? expression K_BEGIN func_stmts ( K_ELSE func_stmts )? K_END K_LOOP?
;

attr
: L_IDENTIFIER
;
Expand All @@ -183,6 +181,10 @@ if_stmt
: K_IF expression K_THEN statements ( K_ELSE statements )? K_END K_IF?
;

if_func
: K_IF expression K_THEN func_stmts ( K_ELSE func_stmts )? K_END K_IF?
;

analyze_stmt
: K_ANALYZE K_DS? ds_name S_STAR? key_item? K_PARTITION?
;
Expand All @@ -192,12 +194,26 @@ options_stmt
;

create_proc
: ( K_CREATE ( S_OR K_REPLACE )? )? K_PROCEDURE func ( S_OPEN_PAR proc_param ( S_COMMA proc_param )* S_CLOSE_PAR )? K_AS? K_BEGIN statements K_END K_PROCEDURE?
: ( K_CREATE ( S_OR K_REPLACE )? )? K_PROCEDURE func ( S_OPEN_PAR proc_param ( S_COMMA proc_param )* S_CLOSE_PAR )?
K_AS? K_BEGIN statements K_END K_PROCEDURE?
;

create_func
: ( K_CREATE ( S_OR K_REPLACE )? )? K_FUNCTION func ( S_OPEN_PAR proc_param ( S_COMMA proc_param )* S_CLOSE_PAR )? K_AS? K_RETURN? expression
| ( K_CREATE ( S_OR K_REPLACE )? )? K_FUNCTION func ( S_OPEN_PAR proc_param ( S_COMMA proc_param )* S_CLOSE_PAR )? K_RECORD K_AS? K_RETURN? attr_expr
: ( K_CREATE ( S_OR K_REPLACE )? )? K_FUNCTION func ( S_OPEN_PAR proc_param ( S_COMMA proc_param )* S_CLOSE_PAR )?
K_RECORD?
K_AS? ( K_RETURN? expression | K_BEGIN func_stmts K_END K_FUNCTION? )
;

func_stmts
: ( func_stmt S_SCOL )*
;

func_stmt
: let_func | loop_func | if_func | return_func
;

return_func
: K_RETURN expression
;

proc_param
Expand Down
26 changes: 13 additions & 13 deletions datacooker-commons/src/main/java/io/github/pastorgl/datacooker/data/DataContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ public StreamInfo alterDataStream(String dsName, StreamConverter converter, Map<
while (it.hasNext()) {
Tuple2<Object, DataRecord<?>> rec = it.next();

ret.add(new Tuple2<>(Expressions.evalAttr(rec._1, rec._2, expr, vc), rec._2));
ret.add(new Tuple2<>(Expressions.eval(rec._1, rec._2, expr, vc), rec._2));
}

return ret.iterator();
Expand Down Expand Up @@ -677,7 +677,7 @@ public JavaPairRDD<Object, DataRecord<?>> select(
while (it.hasNext()) {
Tuple2<Object, DataRecord<?>> rec = it.next();

if (Expressions.boolAttr(rec._1, rec._2, _where.expression, vc)) {
if (Expressions.bool(rec._1, rec._2, _where.expression, vc)) {
if (star) {
ret.add(rec);
} else {
Expand All @@ -691,7 +691,7 @@ public JavaPairRDD<Object, DataRecord<?>> select(
}

for (int i = 0; i < size; i++) {
Object value = Expressions.evalAttr(rec._1, rec._2, _what.get(i).expression, vc);
Object value = Expressions.eval(rec._1, rec._2, _what.get(i).expression, vc);
if ((value != null) && value.getClass().isArray()) {
Object[] arr = (Object[]) value;
for (int j = 0; j < arr.length; j++) {
Expand Down Expand Up @@ -724,7 +724,7 @@ public JavaPairRDD<Object, DataRecord<?>> select(
Tuple2<Object, DataRecord<?>> next = it.next();

SegmentedTrack st = (SegmentedTrack) next._2;
if (_qTrack && !Expressions.boolAttr(next._1, st, _where.expression, vc)) {
if (_qTrack && !Expressions.bool(next._1, st, _where.expression, vc)) {
continue;
}

Expand All @@ -735,7 +735,7 @@ public JavaPairRDD<Object, DataRecord<?>> select(
SelectItem selectItem = _what.get(i);

if (ObjLvl.TRACK.equals(selectItem.category)) {
Object value = Expressions.evalAttr(next._1, st, selectItem.expression, vc);
Object value = Expressions.eval(next._1, st, selectItem.expression, vc);
if ((value != null) && value.getClass().isArray()) {
Object[] arr = (Object[]) value;
for (int j = 0; j < arr.length; j++) {
Expand All @@ -756,7 +756,7 @@ public JavaPairRDD<Object, DataRecord<?>> select(
if (_qSegment) {
List<Geometry> segList = new ArrayList<>();
for (Geometry g : st) {
if (Expressions.boolAttr(next._1, (TrackSegment) g, _where.expression, vc)) {
if (Expressions.bool(next._1, (TrackSegment) g, _where.expression, vc)) {
segList.add(g);
}
}
Expand All @@ -775,7 +775,7 @@ public JavaPairRDD<Object, DataRecord<?>> select(
SelectItem selectItem = _what.get(i);

if (ObjLvl.SEGMENT.equals(selectItem.category)) {
Object value = Expressions.evalAttr(next._1, g, selectItem.expression, vc);
Object value = Expressions.eval(next._1, g, selectItem.expression, vc);
if ((value != null) && value.getClass().isArray()) {
Object[] arr = (Object[]) value;
for (int k = 0; k < arr.length; k++) {
Expand Down Expand Up @@ -804,7 +804,7 @@ public JavaPairRDD<Object, DataRecord<?>> select(

List<Geometry> points = new ArrayList<>();
for (Geometry gg : seg) {
if (Expressions.boolAttr(next._1, (PointEx) gg, _where.expression, vc)) {
if (Expressions.bool(next._1, (PointEx) gg, _where.expression, vc)) {
points.add(gg);
}
}
Expand Down Expand Up @@ -834,7 +834,7 @@ public JavaPairRDD<Object, DataRecord<?>> select(
SelectItem selectItem = _what.get(i);

if (ObjLvl.POINT.equals(selectItem.category)) {
Object value = Expressions.evalAttr(next._1, gg, selectItem.expression, vc);
Object value = Expressions.eval(next._1, gg, selectItem.expression, vc);
if ((value != null) && value.getClass().isArray()) {
Object[] arr = (Object[]) value;
for (int l = 0; l < arr.length; l++) {
Expand Down Expand Up @@ -899,8 +899,8 @@ public Collection<?> subQuery(boolean distinct, DataStream input, int[] partitio
while (it.hasNext()) {
Tuple2<Object, DataRecord<?>> rec = it.next();

if (Expressions.boolAttr(rec._1, rec._2, _query, vc)) {
ret.add(Expressions.evalAttr(rec._1, rec._2, _what, vc));
if (Expressions.bool(rec._1, rec._2, _query, vc)) {
ret.add(Expressions.eval(rec._1, rec._2, _what, vc));
}
}

Expand Down Expand Up @@ -945,7 +945,7 @@ public void analyze(Map<String, DataStream> dataStreams, List<Expressions.ExprIt
if (keyExpession.isEmpty()) {
id = r._1;
} else {
id = Expressions.evalAttr(r._1, r._2, keyExpession, vc);
id = Expressions.eval(r._1, r._2, keyExpession, vc);
}

ret.add(new Tuple2<>(id, null));
Expand Down Expand Up @@ -996,7 +996,7 @@ public void analyze(Map<String, DataStream> dataStreams, List<Expressions.ExprIt
if (keyExpession.isEmpty()) {
id = r._1;
} else {
id = Expressions.evalAttr(r._1, r._2, keyExpession, vc);
id = Expressions.eval(r._1, r._2, keyExpession, vc);
}

ids.compute(id, (k, v) -> v == null ? 1L : v + 1L);
Expand Down
14 changes: 3 additions & 11 deletions datacooker-commons/src/main/java/io/github/pastorgl/datacooker/scripting/Expressions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,28 +384,20 @@ public String toString() {
};
}

public static boolean boolLoose(List<ExprItem<?>> item, VariablesContext vc) {
return boolAttr(null, null, item, vc);
}

public static boolean boolAttr(Object key, DataRecord<?> rec, List<ExprItem<?>> item, VariablesContext vc) {
public static boolean bool(Object key, DataRecord<?> rec, List<ExprItem<?>> item, VariablesContext vc) {
if ((item == null) || item.isEmpty()) {
return true;
}

Object r = evalAttr(key, rec, item, vc);
Object r = eval(key, rec, item, vc);
if (r == null) {
return false;
}

return Boolean.parseBoolean(String.valueOf(r));
}

public static Object evalLoose(List<ExprItem<?>> item, VariablesContext vc) {
return evalAttr(null, null, item, vc);
}

public static Object evalAttr(Object key, DataRecord<?> rec, List<ExprItem<?>> item, VariablesContext vc) {
public static Object eval(Object key, DataRecord<?> rec, List<ExprItem<?>> item, VariablesContext vc) {
if (item.isEmpty()) {
return null;
}
Expand Down
Loading

0 comments on commit de1be52

Please sign in to comment.