Skip to content

Commit

Permalink
Give up on idea to compile predicate at compile time, sanitizing opti…
Browse files Browse the repository at this point in the history 
…onals is enough
  • Loading branch information
@grouzen
grouzen committed Aug 30, 2024
1 parent ec39d6c commit 539741f
Show file tree
Hide file tree
Showing 17 changed files with 246 additions and 248 deletions.
8 changes: 5 additions & 3 deletions .../apache/parquet/core/filter/package.scala → ...o/apache/parquet/core/filter/syntax.scala
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
package me.mnedokushev.zio.apache.parquet.core
package me.mnedokushev.zio.apache.parquet.core.filter

import me.mnedokushev.zio.apache.parquet.core.filter.internal.CompilePredicateMacro
import me.mnedokushev.zio.apache.parquet.core.filter.CompiledPredicate
import org.apache.parquet.filter2.predicate.FilterPredicate

package object filter {
package object syntax {

implicit class NullableColumnSyntax[F, S, A](val column: Lens[F, S, Option[A]]) {
def nullable(implicit typeTag: TypeTag[A]): Column.Named[A, column.Identity] =
Column.Named(column.path)
}

def compile[A](predicate: Predicate[A]): Either[String, FilterPredicate] = macro CompilePredicateMacro.compileImpl[A]
def compile[A](predicate: Predicate[A]): CompiledPredicate =
macro CompilePredicateMacro.compileImpl[A]

// NOTE: no compile-time check on parent/child columns relation due to lack of singleton types in scala 2.12
def concat[A, B: TypeTag, F](parent: Column[A], child: Column.Named[B, F]): Column[B] =
Expand Down
12 changes: 7 additions & 5 deletions .../apache/parquet/core/filter/package.scala → ...o/apache/parquet/core/filter/syntax.scala
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
package me.mnedokushev.zio.apache.parquet.core
package me.mnedokushev.zio.apache.parquet.core.filter

import me.mnedokushev.zio.apache.parquet.core.filter.internal.{ ColumnPathConcatMacro, CompilePredicateMacro }
import org.apache.parquet.filter2.predicate.FilterPredicate
import me.mnedokushev.zio.apache.parquet.core.filter.internal.{ ColumnPathConcatMacro, SanitizeOptionalsMacro }
import me.mnedokushev.zio.apache.parquet.core.filter.CompiledPredicate
import me.mnedokushev.zio.apache.parquet.core.Lens

package object filter {
package object syntax {

implicit class NullableColumnSyntax[F, S, A](val column: Lens[F, S, Option[A]]) {
def nullable(implicit typeTag: TypeTag[A]): Column.Named[A, column.Identity] =
Column.Named(column.path)
}

def compile[A](predicate: Predicate[A]): Either[String, FilterPredicate] = macro CompilePredicateMacro.compileImpl[A]
def predicate[A](predicate: Predicate[A]): CompiledPredicate =
macro SanitizeOptionalsMacro.sanitizeImpl[A]

def concat[A, B, F](
parent: Column[A],
Expand Down
6 changes: 3 additions & 3 deletions ...lter/internal/CompilePredicateMacro.scala → ...ter/internal/SanitizeOptionalsMacro.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ import me.mnedokushev.zio.apache.parquet.core.filter.Predicate

import scala.reflect.macros.blackbox

class CompilePredicateMacro(val c: blackbox.Context) extends MacroUtils(c) {
class SanitizeOptionalsMacro(val c: blackbox.Context) extends MacroUtils(c) {
import c.universe._

def compileImpl[A](predicate: Expr[Predicate[A]])(ptt: c.WeakTypeTag[A]): Tree = {
def sanitizeImpl[A](predicate: Expr[Predicate[A]])(ptt: c.WeakTypeTag[A]): Tree = {

// Example of a tree for A type:
// RefinedType(
Expand Down Expand Up @@ -58,7 +58,7 @@ class CompilePredicateMacro(val c: blackbox.Context) extends MacroUtils(c) {
""".stripMargin
)
else
q"_root_.me.mnedokushev.zio.apache.parquet.core.filter.Filter.compile($predicate)"
q"_root_.me.mnedokushev.zio.apache.parquet.core.filter.Predicate.compile0($predicate)"
}

}
2 changes: 1 addition & 1 deletion ...cala-3/me/mnedokushev/zio/apache/parquet/core/filter/internal/ColumnPathConcatMacro.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ object ColumnPathConcatMacro {
if (parentFields.contains(childField)) {
val concatExpr = '{ ${ parent }.path + "." + ${ child }.path }

'{ me.mnedokushev.zio.apache.parquet.core.filter.Column.Named[B, F]($concatExpr)($childTypeTag) }
'{ me.mnedokushev.zio.apache.parquet.core.filter.Column.Named[B, F]($concatExpr)(using $childTypeTag) }
} else
report.errorAndAbort(s"Parent column doesn't contain a column named '$childField'")
}
Expand Down
37 changes: 5 additions & 32 deletions ...lter/internal/CompilePredicateMacro.scala → ...ter/internal/SanitizeOptionalsMacro.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,13 @@ package me.mnedokushev.zio.apache.parquet.core.filter.internal

import me.mnedokushev.zio.apache.parquet.core.filter.Predicate
import org.apache.parquet.filter2.predicate.FilterPredicate

import scala.quoted.*
import me.mnedokushev.zio.apache.parquet.core.filter.CompiledPredicate

object CompilePredicateMacro {

private class PredicateFromExpr[A] extends FromExpr[Predicate[A]] {

override def unapply(x: Expr[Predicate[A]])(using Quotes): Option[Predicate[A]] = {
import quotes.reflect.*

println(x.asTerm.show(using Printer.TreeStructure))

x.asTerm.tpe.widenTermRefByName match {
case ConstantType(c) => Some(c.value.asInstanceOf[Predicate[A]])
case _ => None
}
}

}
object SanitizeOptionalsMacro {

// TODO: tests
def compileImpl[A: Type](predicate: Expr[Predicate[A]])(using Quotes): Expr[FilterPredicate] = {
def sanitizeImpl[A: Type](predicate: Expr[Predicate[A]])(using Quotes): Expr[CompiledPredicate] = {
import quotes.reflect.*

// Example of a type representation of A type:
Expand Down Expand Up @@ -58,20 +43,8 @@ object CompilePredicateMacro {
| Predicate tree: ${predicate.show}
""".stripMargin
)
else {
val pred0 = predicate.valueOrAbort(using PredicateFromExpr[A])

_root_.me.mnedokushev.zio.apache.parquet.core.filter.Filter.compile0(pred0) match {
case Right(filter) =>
val clazz = filter.getClass()
Ref(defn.Predef_classOf)
.appliedToType(TypeRepr.typeConstructorOf(clazz))
.asExpr
.asInstanceOf[Expr[FilterPredicate]]
case Left(error) =>
report.errorAndAbort(s"Error while compiling filter predicate: $error")
}
}
else
'{ _root_.me.mnedokushev.zio.apache.parquet.core.filter.Predicate.compile0($predicate) }

}

Expand Down
10 changes: 7 additions & 3 deletions .../apache/parquet/core/filter/package.scala → ...o/apache/parquet/core/filter/syntax.scala
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,19 @@
package me.mnedokushev.zio.apache.parquet.core

package me.mnedokushev.zio.apache.parquet.core.filter

import me.mnedokushev.zio.apache.parquet.core.filter.internal.ColumnPathConcatMacro
import me.mnedokushev.zio.apache.parquet.core.filter.internal.SanitizeOptionalsMacro
import me.mnedokushev.zio.apache.parquet.core.Lens

package object filter {
package object syntax {

extension [F, S, A](column: Lens[F, S, Option[A]]) {
def nullable(implicit typeTag: TypeTag[A]): Column.Named[A, column.Identity] =
Column.Named(column.path)
}

inline def predicate[A](inline predicate: Predicate[A]): CompiledPredicate =
${ SanitizeOptionalsMacro.sanitizeImpl[A]('predicate) }

inline def concat[A, B, F](inline parent: Column[A], inline child: Column.Named[B, F])(using
ctt: TypeTag[B]
): Column[B] =
Expand Down
2 changes: 1 addition & 1 deletion ...re/src/main/scala/me/mnedokushev/zio/apache/parquet/core/filter/ExprAccessorBuilder.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package me.mnedokushev.zio.apache.parquet.core.filter

import zio.schema.{ AccessorBuilder, Schema }

final class ExprAccessorBuilder(typeTags: Map[String, TypeTag[_]]) extends AccessorBuilder {
final class ExprAccessorBuilder(typeTags: Map[String, TypeTag[?]]) extends AccessorBuilder {

override type Lens[F, S, A] = Column.Named[A, F]

Expand Down
99 changes: 2 additions & 97 deletions modules/core/src/main/scala/me/mnedokushev/zio/apache/parquet/core/filter/Filter.scala
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
package me.mnedokushev.zio.apache.parquet.core.filter

import me.mnedokushev.zio.apache.parquet.core.{ Lens, Prism, Traversal }
import org.apache.parquet.filter2.predicate.{ FilterApi, FilterPredicate, Operators }
import zio.prelude._
import zio.schema._
import me.mnedokushev.zio.apache.parquet.core.filter.internal.CompilePredicateMacro

trait Filter {

Expand Down Expand Up @@ -35,100 +32,8 @@ object Filter {
schema.makeAccessors(accessorBuilder)
}

inline def compile[A](inline predicate: Predicate[A]): FilterPredicate =
${ CompilePredicateMacro.compileImpl[A]('predicate) }

private[zio] def compile0[A](predicate: Predicate[A]): Either[String, FilterPredicate] = {

def error(op: Operator) =
Left(s"Operator $op is not supported by $predicate")

def binarySet[T <: Comparable[T], C <: Operators.Column[T] with Operators.SupportsEqNotEq](
column: C,
values: java.util.Set[T],
op: Operator.Binary.Set[_]
) =
op match {
case Operator.Binary.Set.In() =>
Right(FilterApi.in(column, values))
case Operator.Binary.Set.NotIn() =>
Right(FilterApi.notIn(column, values))
}

predicate match {
case Predicate.Unary(predicate0, op) =>
op match {
case Operator.Unary.Not() =>
compile0(predicate0).map(FilterApi.not)
}
case Predicate.Logical(left, right, op) =>
(compile0(left) <*> compile0(right)).map { case (left0, right0) =>
op match {
case Operator.Logical.And() =>
FilterApi.and(left0, right0)
case Operator.Logical.Or() =>
FilterApi.or(left0, right0)
}
}
case Predicate.Binary(column, value, op) =>
column.typeTag match {
case typeTag: TypeTag.EqNotEq[_] =>
val typeTag0 = typeTag.cast[A]
val column0 = typeTag0.column(column.path)
val value0 = typeTag0.value(value)

op match {
case Operator.Binary.Eq() =>
Right(FilterApi.eq(column0, value0))
case Operator.Binary.NotEq() =>
Right(FilterApi.notEq(column0, value0))
case _ =>
error(op)
}
case typeTag: TypeTag.LtGt[_] =>
val typeTag0 = typeTag.cast[A]
val column0 = typeTag0.column(column.path)
val value0 = typeTag0.value(value)

op match {
case Operator.Binary.Eq() =>
Right(FilterApi.eq(column0, value0))
case Operator.Binary.NotEq() =>
Right(FilterApi.notEq(column0, value0))
case Operator.Binary.LessThen() =>
Right(FilterApi.lt(column0, value0))
case Operator.Binary.LessEq() =>
Right(FilterApi.ltEq(column0, value0))
case Operator.Binary.GreaterThen() =>
Right(FilterApi.gt(column0, value0))
case Operator.Binary.GreaterEq() =>
Right(FilterApi.gtEq(column0, value0))
case _ =>
error(op)
}
case _ =>
error(op)
}
case Predicate.BinarySet(column, values, op) =>
column.typeTag match {
case typeTag: TypeTag.EqNotEq[_] =>
val typeTag0 = typeTag.cast[A]
val column0 = typeTag0.column(column.path)
val values0 = typeTag0.values(values)

binarySet(column0, values0, op)
case typeTag: TypeTag.LtGt[_] =>
val typeTag0 = typeTag.cast[A]
val column0 = typeTag0.column(column.path)
val values0 = typeTag0.values(values)

binarySet(column0, values0, op)
case _ =>
error(op)
}
}

}
// inline def compile[A](inline predicate: Predicate[A]): FilterPredicate =
// ${ CompilePredicateMacro.compileImpl[A]('predicate) }

def not[A](pred: Predicate[A]): Predicate[A] =
Predicate.Unary(pred, Operator.Unary.Not[A]())
Expand Down
2 changes: 1 addition & 1 deletion ...s/core/src/main/scala/me/mnedokushev/zio/apache/parquet/core/filter/OperatorSupport.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ sealed trait OperatorSupport[A]

object OperatorSupport {

trait Optional[A, S[_] <: OperatorSupport[_]] {
trait Optional[A, S[_] <: OperatorSupport[?]] {
val operatorSupport: S[A]
}

Expand Down
Loading

0 comments on commit 539741f

Please sign in to comment.