DRAFT: ADBDEV-6630 (for CI only)

.abi-check/7.2.0_arenadata7/postgres.symbols.ignore

@@ -0,0 +1,127 @@
+OptimizerMemoryContext
+optimize_query
+transformGroupedWindows
+optopt@@GLIBC_2.2.5
+optopt
+optimizer_log
+optimizer_log_failure
+optimizer_trace_fallback
+optimizer_minidump
+optimizer_cost_model
+optimizer_metadata_caching
+optimizer_mdcache_size
+optimizer_print_query
+optimizer_print_plan
+optimizer_print_xform
+optimizer_print_memo_after_exploration
+optimizer_print_memo_after_implementation
+optimizer_print_memo_after_optimization
+optimizer_print_job_scheduler
+optimizer_print_expression_properties
+optimizer_print_group_properties
+optimizer_print_optimization_context
+optimizer_print_optimization_stats
+optimizer_print_xform_results
+optimizer_xforms
+optimizer_search_strategy_path
+optimizer_enable_nljoin
+optimizer_enable_indexjoin
+optimizer_enable_motions_coordinatoronly_queries
+optimizer_enable_motions
+optimizer_enable_motion_broadcast
+optimizer_enable_motion_gather
+optimizer_enable_motion_redistribute
+optimizer_enable_sort
+optimizer_enable_materialize
+optimizer_enable_partition_propagation
+optimizer_enable_partition_selection
+optimizer_enable_outerjoin_rewrite
+optimizer_enable_multiple_distinct_aggs
+optimizer_enable_hashjoin_redistribute_broadcast_children
+optimizer_enable_broadcast_nestloop_outer_child
+optimizer_discard_redistribute_hashjoin
+optimizer_enable_streaming_material
+optimizer_enable_gather_on_segment_for_dml
+optimizer_enable_assert_maxonerow
+optimizer_enable_constant_expression_evaluation
+optimizer_enable_bitmapscan
+optimizer_enable_outerjoin_to_unionall_rewrite
+optimizer_enable_ctas
+optimizer_enable_dml
+optimizer_enable_dml_constraints
+optimizer_enable_direct_dispatch
+optimizer_enable_coordinator_only_queries
+optimizer_enable_hashjoin
+optimizer_enable_dynamictablescan
+optimizer_enable_dynamicindexscan
+optimizer_enable_dynamicindexonlyscan
+optimizer_enable_dynamicbitmapscan
+optimizer_enable_indexscan
+optimizer_enable_indexonlyscan
+optimizer_enable_tablescan
+optimizer_enable_eageragg
+optimizer_enable_orderedagg
+optimizer_expand_fulljoin
+optimizer_enable_hashagg
+optimizer_enable_groupagg
+optimizer_enable_mergejoin
+optimizer_enable_redistribute_nestloop_loj_inner_child
+optimizer_force_comprehensive_join_implementation
+optimizer_enable_replicated_table
+optimizer_enable_foreign_table
+optimizer_enable_right_outer_join
+optimizer_enumerate_plans
+optimizer_sample_plans
+optimizer_plan_id
+optimizer_samples_number
+optimizer_jit_above_cost
+optimizer_jit_inline_above_cost
+optimizer_jit_optimize_above_cost
+optimizer_extract_dxl_stats
+optimizer_extract_dxl_stats_all_nodes
+optimizer_print_missing_stats
+optimizer_damping_factor_filter
+optimizer_damping_factor_join
+optimizer_damping_factor_groupby
+optimizer_dpe_stats
+optimizer_enable_derive_stats_all_groups
+optimizer_segments
+optimizer_penalize_broadcast_threshold
+optimizer_cost_threshold
+optimizer_nestloop_factor
+optimizer_sort_factor
+optimizer_array_expansion_threshold
+optimizer_join_order_threshold
+optimizer_join_order
+optimizer_join_arity_for_associativity_commutativity
+optimizer_cte_inlining_bound
+optimizer_push_group_by_below_setop_threshold
+optimizer_xform_bind_threshold
+optimizer_skew_factor
+optimizer_force_multistage_agg
+optimizer_force_three_stage_scalar_dqa
+optimizer_force_expanded_distinct_aggs
+optimizer_force_agg_skew_avoidance
+optimizer_penalize_skew
+optimizer_prune_computed_columns
+optimizer_push_requirements_from_consumer_to_producer
+optimizer_enforce_subplans
+optimizer_apply_left_outer_to_union_all_disregarding_stats
+optimizer_use_external_constant_expression_evaluation_for_ints
+optimizer_remove_order_below_dml
+optimizer_multilevel_partitioning
+optimizer_parallel_union
+optimizer_array_constraints
+optimizer_cte_inlining
+optimizer_enable_space_pruning
+optimizer_enable_associativity
+optimizer_enable_range_predicate_dpe
+optimizer_enable_push_join_below_union_all
+optimizer_use_gpdb_allocators
+optimizer_partition_selection_log
+optimizer_enable_query_parameter
+ConfigureNamesBool_gp
+ConfigureNamesEnum_gp
+ConfigureNamesInt_gp
+ConfigureNamesReal_gp
+ConfigureNamesString_gp

.abi-check/7.2.0_arenadata7/postgres.types.ignore

@@ -0,0 +1 @@
+struct PlannedStmt

.travis.yml

@@ -121,9 +121,9 @@ matrix:
                 --enable-orca \
                 --without-zstd --disable-gpfdist \
                 CC="ccache clang-11" CXX="ccache clang++-11" CFLAGS=-O0 CXXFLAGS=-O0
-              bear --append make -s -C src/backend/gpopt
+              bear --append -- make -s -C gpcontrib/orca
               )
-              CLANG_TIDY=clang-tidy-11 src/tools/tidy chk-gpopt ../vpath.debug
+              CLANG_TIDY=clang-tidy-11 gpcontrib/orca/tools/tidy chk-gpopt ../vpath.debug
             - |
               (
               set -e
@@ -135,13 +135,13 @@ matrix:
                 --enable-orca \
                 --without-zstd --disable-gpfdist \
                 CC="ccache clang-11" CXX="ccache clang++-11" CFLAGS=-O0 CXXFLAGS=-O0
-              bear --append make -s -C src/backend/gpopt
+              bear --append -- make -s -C gpcontrib/orca
               )
-              CLANG_TIDY=clang-tidy-11 src/tools/tidy chk-gpopt ../vpath.release
-            - CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=Debug -Hsrc/backend/gporca -Bbuild.debug
-            - CLANG_TIDY=clang-tidy-11 src/tools/tidy chk-orca build.debug
-            - CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -Hsrc/backend/gporca -Bbuild.relwithdebinfo
-            - CLANG_TIDY=clang-tidy-11 src/tools/tidy chk-orca build.relwithdebinfo
+              CLANG_TIDY=clang-tidy-11 gpcontrib/orca/tools/tidy chk-gpopt ../vpath.release
+            - CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=Debug -Hgpcontrib/orca/gporca -Bbuild.debug
+            - CLANG_TIDY=clang-tidy-11 gpcontrib/orca/tools/tidy chk-orca build.debug
+            - CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -Hgpcontrib/orca/gporca -Bbuild.relwithdebinfo
+            - CLANG_TIDY=clang-tidy-11 gpcontrib/orca/tools/tidy chk-orca build.relwithdebinfo
 
         - name: check-format
           stage: lint
@@ -157,9 +157,9 @@ matrix:
           before_script: ~
           after_script: ~
           script:
-            - env CLANG_FORMAT=clang-format-11 src/tools/fmt gen
+            - env CLANG_FORMAT=clang-format-11 gpcontrib/orca/tools/fmt gen
             - git diff --exit-code
-            - env CLANG_FORMAT=clang-format-11 src/tools/fmt chk
+            - env CLANG_FORMAT=clang-format-11 gpcontrib/orca/tools/fmt chk
 
 stages:
   - lint

arenadata/Dockerfile.linter

@@ -9,4 +9,4 @@ COPY . /opt/gpdb_src
 
 ENV CLANG_FORMAT=clang-format-11
 
-ENTRYPOINT src/tools/fmt gen && git diff --exit-code && src/tools/fmt chk
+ENTRYPOINT gpcontrib/orca/tools/fmt gen && git diff --exit-code && gpcontrib/orca/tools/fmt chk

concourse/scripts/unit_tests_gporca.bash

@@ -7,7 +7,7 @@ function build_xerces
 {
     OUTPUT_DIR="gpdb_src/gpAux/ext/${BLD_ARCH}"
     mkdir -p xerces_patch/concourse
-    cp -r gpdb_src/src/backend/gporca/concourse/xerces-c xerces_patch/concourse
+    cp -r gpdb_src/gpcontrib/orca/gporca/concourse/xerces-c xerces_patch/concourse
     /usr/bin/python3 xerces_patch/concourse/xerces-c/build_xerces.py --output_dir=${OUTPUT_DIR}
     rm -rf build
 }
@@ -18,7 +18,7 @@ function test_orca
         return
     fi
     OUTPUT_DIR="../../../../gpAux/ext/${BLD_ARCH}"
-    pushd ${GPDB_SRC_PATH}/src/backend/gporca
+    pushd ${GPDB_SRC_PATH}/gpcontrib/orca/gporca
     concourse/build_and_test.py --build_type=RelWithDebInfo --output_dir=${OUTPUT_DIR}
     concourse/build_and_test.py --build_type=Debug --output_dir=${OUTPUT_DIR}
     popd

concourse/tasks/check_format.yml

@@ -9,7 +9,7 @@ inputs:
   - name: gpdb_src
     path: .
 run:
-  path: src/tools/fmt
+  path: gpcontrib/orca/tools/fmt
   args: [ chk ]
 params:
   CLANG_FORMAT: clang-format-11

concourse/tasks/clang_tidy.yml

@@ -21,7 +21,7 @@ run:
         --enable-orca \
         --without-python --without-readline --without-zlib --without-zstd --without-libbz2 --disable-gpfdist --disable-gpcloud --without-libcurl \
         CC="ccache clang-11" CXX="ccache clang++-11" CFLAGS=-O0 CXXFLAGS=-O0
-      bear --append make -s -C src/backend/gpopt
+      bear --append -- make -s -C gpcontrib/orca
     )
     (
       mkdir vpath.release
@@ -31,17 +31,17 @@ run:
         --enable-orca \
         --without-python --without-readline --without-zlib --without-zstd --without-libbz2 --disable-gpfdist --disable-gpcloud --without-libcurl \
         CC="ccache clang-11" CXX="ccache clang++-11" CFLAGS=-O0 CXXFLAGS=-O0
-      bear --append make -s -C src/backend/gpopt
+      bear --append -- make -s -C gpcontrib/orca
     )
     (
       cd gpdb_src
-      src/tools/tidy chk-gpopt ../vpath.debug
-      src/tools/tidy chk-gpopt ../vpath.release
+      gpcontrib/orca/tools/tidy chk-gpopt ../vpath.debug
+      gpcontrib/orca/tools/tidy chk-gpopt ../vpath.release
 
-      CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=Debug -Hsrc/backend/gporca -Bbuild.debug
-      src/tools/tidy chk-orca build.debug
-      CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -Hsrc/backend/gporca -Bbuild.relwithdebinfo
-      src/tools/tidy chk-orca build.relwithdebinfo
+      CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=Debug -Hgpcontrib/orca/gporca -Bbuild.debug
+      gpcontrib/orca/tools/tidy chk-orca build.debug
+      CXX=clang++-11 cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -Hgpcontrib/orca/gporca -Bbuild.relwithdebinfo
+      gpcontrib/orca/tools/tidy chk-orca build.relwithdebinfo
     )
 
 params:

configure

@@ -7905,9 +7905,7 @@ if test "${enable_orca+set}" = set; then :
   enableval=$enable_orca;
   case $enableval in
     yes)
-
-$as_echo "#define USE_ORCA 1" >>confdefs.h
-
+      :
       ;;
     no)
       :
@@ -7920,8 +7918,6 @@ $as_echo "#define USE_ORCA 1" >>confdefs.h
 else
   enable_orca=yes
 
-$as_echo "#define USE_ORCA 1" >>confdefs.h
-
 fi
 
 

configure.in

@@ -787,9 +787,7 @@ PGAC_ARG_BOOL(enable, cassert, no, [enable assertion checks (for debugging)],
 #
 # Enable GPORCA optimizer
 #
-PGAC_ARG_BOOL(enable, orca, yes, [disable ORCA optimizer],
-              [AC_DEFINE([USE_ORCA], 1,
-                         [Define to 1 to build with Greenplum ORCA optimizer. (--enable-orca)])])
+PGAC_ARG_BOOL(enable, orca, yes, [disable ORCA optimizer])
 AC_MSG_RESULT([checking whether to build with ORCA... $enable_orca])
 AC_SUBST(enable_orca)
 

contrib/auto_explain/expected/auto_explain.out

@@ -101,7 +101,7 @@ Finalize Aggregate  (cost=35148.64..35148.65 rows=1 width=8) (actual rows=1 loop
                           ->  Broadcast Motion 3:3  (slice2; segments: 3)  (cost=0.00..53.05 rows=1001 width=0) (actual rows=1001 loops=1)
                                 ->  Seq Scan on auto_explain_test.t2  (cost=0.00..13.01 rows=334 width=0) (actual rows=340 loops=1)
 Optimizer: Postgres-based planner
-Settings: enable_nestloop = 'on', optimizer = 'off'
+Settings: enable_nestloop = 'on'
   (slice0)    Executor memory: 131K bytes.
   (slice1)    Executor memory: 152K bytes avg x 3 workers, 152K bytes max (seg0).
   (slice2)    Executor memory: 42K bytes avg x 3 workers, 42K bytes max (seg0).

contrib/auto_explain/expected/auto_explain_optimizer.out

@@ -101,7 +101,7 @@ Finalize Aggregate  (cost=0.00..1326086.34 rows=1 width=8) (actual rows=1 loops=
                           ->  Seq Scan on auto_explain_test.t1  (cost=0.00..431.01 rows=334 width=1) (actual rows=340 loops=1)
                     ->  Seq Scan on auto_explain_test.t2  (cost=0.00..431.01 rows=334 width=1) (actual rows=340 loops=1002)
 Optimizer: GPORCA
-Settings: enable_nestloop = 'on'
+Settings: enable_nestloop = 'on', optimizer = 'on'
   (slice0)    Executor memory: 67K bytes.
   (slice1)    Executor memory: 119K bytes avg x 3 workers, 119K bytes max (seg0).
   (slice2)    Executor memory: 42K bytes avg x 3 workers, 42K bytes max (seg0).

contrib/pg_stat_statements/sql/disable_pg_stat_statements.sql

@@ -1,4 +1,4 @@
 -- start_ignore
-\! gpconfig -r shared_preload_libraries;
+\! gpconfig -c shared_preload_libraries -v "$(psql -At -c "SELECT array_to_string(array_remove(string_to_array(current_setting('shared_preload_libraries'), ','), 'pg_stat_statements'), ',')" postgres)"
 \! gpstop -raq -M fast;
 -- end_ignore

contrib/pg_stat_statements/sql/enable_pg_stat_statements.sql

@@ -1,4 +1,4 @@
 -- start_ignore
-\! gpconfig -c shared_preload_libraries -v 'pg_stat_statements';
+\! gpconfig -c shared_preload_libraries -v "$(psql -At -c "SELECT array_to_string(array_append(string_to_array(current_setting('shared_preload_libraries'), ','), 'pg_stat_statements'), ',')" postgres)"
 \! gpstop -raq -M fast;
 -- end_ignore

gpMgmt/bin/minirepro

@@ -297,7 +297,7 @@ def dump_extstats(cur, oid_str, f_out):
 
 def get_non_default_optimization_gucs(cursor):
     # Ignore 'optimizer' guc as it is turned off by minirepro while creating connection object
-    query = "select name, setting from pg_settings where category like 'Query Tuning%' and setting!=boot_val and name!='optimizer';"
+    query = "select name, setting from pg_settings where (category like 'Query Tuning%' or category='Customized Options') and setting!=boot_val and name!='optimizer';"
     try:
         cursor.execute(query)
     except psycopg2.DatabaseError as e:

gpcontrib/Makefile

@@ -36,6 +36,9 @@ endif
 ifeq "$(with_zstd)" "yes"
 	recurse_targets += zstd
 endif
+ifeq "$(enable_orca)" "yes"
+	recurse_targets += orca
+endif
 $(call recurse,all install clean distclean, $(recurse_targets))
 
 all: gpcloud orafce