diff --git a/SIT/features/09_constant_column_replace/breakData.cql b/SIT/features/07_constant_column_replace/breakData.cql similarity index 100% rename from SIT/features/09_constant_column_replace/breakData.cql rename to SIT/features/07_constant_column_replace/breakData.cql diff --git a/SIT/features/09_constant_column_replace/cdm.fixData.assert b/SIT/features/07_constant_column_replace/cdm.fixData.assert similarity index 100% rename from SIT/features/09_constant_column_replace/cdm.fixData.assert rename to SIT/features/07_constant_column_replace/cdm.fixData.assert diff --git a/SIT/features/09_constant_column_replace/cdm.migrateData.assert b/SIT/features/07_constant_column_replace/cdm.migrateData.assert similarity index 100% rename from SIT/features/09_constant_column_replace/cdm.migrateData.assert rename to SIT/features/07_constant_column_replace/cdm.migrateData.assert diff --git a/SIT/features/09_constant_column_replace/cdm.txt b/SIT/features/07_constant_column_replace/cdm.txt similarity index 100% rename from SIT/features/09_constant_column_replace/cdm.txt rename to SIT/features/07_constant_column_replace/cdm.txt diff --git a/SIT/features/09_constant_column_replace/cdm.validateData.assert b/SIT/features/07_constant_column_replace/cdm.validateData.assert similarity index 100% rename from SIT/features/09_constant_column_replace/cdm.validateData.assert rename to SIT/features/07_constant_column_replace/cdm.validateData.assert diff --git a/SIT/features/09_constant_column_replace/execute.sh b/SIT/features/07_constant_column_replace/execute.sh similarity index 100% rename from SIT/features/09_constant_column_replace/execute.sh rename to SIT/features/07_constant_column_replace/execute.sh diff --git a/SIT/features/09_constant_column_replace/expected.cql b/SIT/features/07_constant_column_replace/expected.cql similarity index 100% rename from SIT/features/09_constant_column_replace/expected.cql rename to SIT/features/07_constant_column_replace/expected.cql diff --git a/SIT/features/09_constant_column_replace/expected.out b/SIT/features/07_constant_column_replace/expected.out similarity index 100% rename from SIT/features/09_constant_column_replace/expected.out rename to SIT/features/07_constant_column_replace/expected.out diff --git a/SIT/features/09_constant_column_replace/fix.properties b/SIT/features/07_constant_column_replace/fix.properties similarity index 100% rename from SIT/features/09_constant_column_replace/fix.properties rename to SIT/features/07_constant_column_replace/fix.properties diff --git a/SIT/features/09_constant_column_replace/migrate.properties b/SIT/features/07_constant_column_replace/migrate.properties similarity index 100% rename from SIT/features/09_constant_column_replace/migrate.properties rename to SIT/features/07_constant_column_replace/migrate.properties diff --git a/SIT/features/09_constant_column_replace/setup.cql b/SIT/features/07_constant_column_replace/setup.cql similarity index 100% rename from SIT/features/09_constant_column_replace/setup.cql rename to SIT/features/07_constant_column_replace/setup.cql diff --git a/SIT/features/07_migrate_rows/cdm.txt b/SIT/features/07_migrate_rows/cdm.txt deleted file mode 100644 index 26804ef8..00000000 --- a/SIT/features/07_migrate_rows/cdm.txt +++ /dev/null @@ -1,2 +0,0 @@ -migrateDataDefault com.datastax.cdm.job.MigrateRowsFromFile migrate.properties -migrateData com.datastax.cdm.job.MigrateRowsFromFile migrate_with_pkrowsfile.properties diff --git a/SIT/features/07_migrate_rows/execute.sh b/SIT/features/07_migrate_rows/execute.sh deleted file mode 100644 index 8610ccda..00000000 --- a/SIT/features/07_migrate_rows/execute.sh +++ /dev/null @@ -1,19 +0,0 @@ -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#!/bin/bash -e - -workingDir="$1" -cd "$workingDir" - -/local/cdm.sh -f cdm.txt -s migrateDataDefault -d "$workingDir" -/local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir" - - diff --git a/SIT/features/07_migrate_rows/expected.cql b/SIT/features/07_migrate_rows/expected.cql deleted file mode 100644 index 9a6d8957..00000000 --- a/SIT/features/07_migrate_rows/expected.cql +++ /dev/null @@ -1,13 +0,0 @@ -/* - Licensed under the Apache License, Version 2.0 (the "License"); you - may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -SELECT * FROM target.feature_migrate_rows; diff --git a/SIT/features/07_migrate_rows/expected.out b/SIT/features/07_migrate_rows/expected.out deleted file mode 100644 index 85074fea..00000000 --- a/SIT/features/07_migrate_rows/expected.out +++ /dev/null @@ -1,7 +0,0 @@ - - key | key2 | value -------+------+-------- - key1 | A | valueA - key3 | C | valueC - -(2 rows) diff --git a/SIT/features/07_migrate_rows/migrate.properties b/SIT/features/07_migrate_rows/migrate.properties deleted file mode 100644 index 382b70d1..00000000 --- a/SIT/features/07_migrate_rows/migrate.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -spark.cdm.connect.origin.host cdm-sit-cass -spark.cdm.connect.target.host cdm-sit-cass - -spark.cdm.schema.origin.keyspaceTable origin.feature_migrate_rows -spark.cdm.schema.target.keyspaceTable target.feature_migrate_rows -spark.cdm.perfops.numParts 1 diff --git a/SIT/features/07_migrate_rows/migrate_with_pkrowsfile.properties b/SIT/features/07_migrate_rows/migrate_with_pkrowsfile.properties deleted file mode 100644 index e081745d..00000000 --- a/SIT/features/07_migrate_rows/migrate_with_pkrowsfile.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -spark.cdm.connect.origin.host cdm-sit-cass -spark.cdm.connect.target.host cdm-sit-cass - -spark.cdm.schema.origin.keyspaceTable origin.feature_migrate_rows -spark.cdm.schema.target.keyspaceTable target.feature_migrate_rows -spark.cdm.perfops.numParts 1 - -spark.cdm.tokenrange.partitionFile ./primary_key_rows.csv diff --git a/SIT/features/07_migrate_rows/origin.feature_migrate_rows_partitions.csv b/SIT/features/07_migrate_rows/origin.feature_migrate_rows_partitions.csv deleted file mode 100644 index b089cce4..00000000 --- a/SIT/features/07_migrate_rows/origin.feature_migrate_rows_partitions.csv +++ /dev/null @@ -1,2 +0,0 @@ -key1 %% A -key3 %% C diff --git a/SIT/features/07_migrate_rows/primary_key_rows.csv b/SIT/features/07_migrate_rows/primary_key_rows.csv deleted file mode 100644 index b089cce4..00000000 --- a/SIT/features/07_migrate_rows/primary_key_rows.csv +++ /dev/null @@ -1,2 +0,0 @@ -key1 %% A -key3 %% C diff --git a/SIT/features/07_migrate_rows/setup.cql b/SIT/features/07_migrate_rows/setup.cql deleted file mode 100644 index f0e00625..00000000 --- a/SIT/features/07_migrate_rows/setup.cql +++ /dev/null @@ -1,20 +0,0 @@ -/* - Licensed under the Apache License, Version 2.0 (the "License"); you - may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -DROP TABLE IF EXISTS origin.feature_migrate_rows; -CREATE TABLE origin.feature_migrate_rows(key text, key2 text, value text, PRIMARY KEY (key,key2)); -INSERT INTO origin.feature_migrate_rows(key,key2,value) VALUES ('key1','A','valueA'); -INSERT INTO origin.feature_migrate_rows(key,key2,value) VALUES ('key2','B','valueB'); -INSERT INTO origin.feature_migrate_rows(key,key2,value) VALUES ('key3','C','valueC'); - -DROP TABLE IF EXISTS target.feature_migrate_rows; -CREATE TABLE target.feature_migrate_rows(key text, key2 text,value text, PRIMARY KEY (key,key2)); diff --git a/src/main/java/com/datastax/cdm/job/CopyPKJobSession.java b/src/main/java/com/datastax/cdm/job/CopyPKJobSession.java deleted file mode 100644 index ccf9b4cc..00000000 --- a/src/main/java/com/datastax/cdm/job/CopyPKJobSession.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright DataStax, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datastax.cdm.job; - -import com.datastax.cdm.cql.statement.OriginSelectByPKStatement; -import com.datastax.cdm.data.EnhancedPK; -import com.datastax.cdm.data.PKFactory; -import com.datastax.cdm.data.Record; -import com.datastax.cdm.feature.Guardrail; -import com.datastax.oss.driver.api.core.CqlSession; -import com.datastax.oss.driver.api.core.cql.Row; -import org.apache.spark.SparkConf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.beans.PropertyEditor; -import java.beans.PropertyEditorManager; -import java.util.ArrayList; -import java.util.List; - -public class CopyPKJobSession extends AbstractJobSession { - - private final PKFactory pkFactory; - private final List originPKClasses; - public Logger logger = LoggerFactory.getLogger(this.getClass().getName()); - private OriginSelectByPKStatement originSelectByPKStatement; - - protected CopyPKJobSession(CqlSession originSession, CqlSession targetSession, SparkConf sc) { - super(originSession, targetSession, sc, true); - this.jobCounter.setRegisteredTypes(JobCounter.CounterType.READ, JobCounter.CounterType.WRITE, JobCounter.CounterType.SKIPPED, JobCounter.CounterType.MISSING); - pkFactory = this.originSession.getPKFactory(); - originPKClasses = this.originSession.getCqlTable().getPKClasses(); - - logger.info("CQL -- origin select: {}", this.originSession.getOriginSelectByPKStatement().getCQL()); - } - - @Override - public void processSlice(SplitPartitions.PKRows slice) { - this.getRowAndInsert(slice); - } - - public void getRowAndInsert(SplitPartitions.PKRows rowsList) { - originSelectByPKStatement = originSession.getOriginSelectByPKStatement(); - jobCounter.threadReset(); - for (String row : rowsList.getPkRows()) { - jobCounter.threadIncrement(JobCounter.CounterType.READ); - EnhancedPK pk = toEnhancedPK(row); - if (null == pk || pk.isError()) { - jobCounter.threadIncrement(JobCounter.CounterType.MISSING); - logger.error("Could not build PK object with value <{}>; error is: {}", row, (null == pk ? "null" : pk.getMessages())); - return; - } - - rateLimiterOrigin.acquire(1); - Record recordFromOrigin = originSelectByPKStatement.getRecord(pk); - if (null == recordFromOrigin) { - jobCounter.threadIncrement(JobCounter.CounterType.MISSING); - logger.error("Could not find origin row with primary-key: {}", row); - return; - } - Row originRow = recordFromOrigin.getOriginRow(); - - Record record = new Record(pkFactory.getTargetPK(originRow), originRow, null); - if (originSelectByPKStatement.shouldFilterRecord(record)) { - jobCounter.threadIncrement(JobCounter.CounterType.SKIPPED); - return; - } - - if (guardrailEnabled) { - String guardrailCheck = guardrailFeature.guardrailChecks(record); - if (guardrailCheck != null && guardrailCheck != Guardrail.CLEAN_CHECK) { - logger.error("Guardrails failed for PrimaryKey {}; {}", record.getPk(), guardrailCheck); - jobCounter.threadIncrement(JobCounter.CounterType.SKIPPED); - return; - } - } - - rateLimiterTarget.acquire(1); - targetSession.getTargetUpsertStatement().putRecord(record); - jobCounter.threadIncrement(JobCounter.CounterType.WRITE); - - jobCounter.globalIncrement(); - printCounts(false); - } - - printCounts(true); - } - - private EnhancedPK toEnhancedPK(String rowString) { - String[] pkFields = rowString.split(" %% "); - List values = new ArrayList<>(originPKClasses.size()); - if (logger.isDebugEnabled()) logger.debug("rowString={}, pkFields={}", rowString, pkFields); - for (int i = 0; i < pkFields.length; i++) { - PropertyEditor editor = PropertyEditorManager.findEditor(originPKClasses.get(i)); - editor.setAsText(pkFields[i]); - values.add(editor.getValue()); - } - return pkFactory.toEnhancedPK(values, pkFactory.getPKClasses(PKFactory.Side.ORIGIN)); - } - -} \ No newline at end of file diff --git a/src/main/java/com/datastax/cdm/job/CopyPKJobSessionFactory.java b/src/main/java/com/datastax/cdm/job/CopyPKJobSessionFactory.java deleted file mode 100644 index 2ed017bf..00000000 --- a/src/main/java/com/datastax/cdm/job/CopyPKJobSessionFactory.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright DataStax, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datastax.cdm.job; - -import com.datastax.oss.driver.api.core.CqlSession; -import org.apache.spark.SparkConf; - -public class CopyPKJobSessionFactory implements IJobSessionFactory { - private static CopyPKJobSession jobSession = null; - - public AbstractJobSession getInstance(CqlSession originSession, CqlSession targetSession, SparkConf sc) { - if (jobSession == null) { - synchronized (CopyPKJobSession.class) { - if (jobSession == null) { - jobSession = new CopyPKJobSession(originSession, targetSession, sc); - } - } - } - return jobSession; - } -} diff --git a/src/main/java/com/datastax/cdm/job/SplitPartitions.java b/src/main/java/com/datastax/cdm/job/SplitPartitions.java index 9e716795..f2a31ce6 100644 --- a/src/main/java/com/datastax/cdm/job/SplitPartitions.java +++ b/src/main/java/com/datastax/cdm/job/SplitPartitions.java @@ -27,7 +27,6 @@ import java.util.Collections; import java.util.List; import java.util.regex.Pattern; -import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -94,25 +93,6 @@ public PartitionMinMax(String line) { } } - public static List getRowPartsFromFile(int numSplits, String inputFilename) throws IOException { - logger.info("ThreadID: {} Splitting rows in file: {} using a split-size of {}" - , Thread.currentThread().getId(), inputFilename, numSplits); - List pkRows = new ArrayList(); - BufferedReader reader = getfileReader(inputFilename); - String pkRow = null; - while ((pkRow = reader.readLine()) != null) { - if (pkRow.startsWith("#")) { - continue; - } - pkRows.add(pkRow); - } - int partSize = pkRows.size() / numSplits; - if (partSize == 0) { - partSize = pkRows.size(); - } - return batches(pkRows, partSize).map(l -> (new PKRows(l))).collect(Collectors.toList()); - } - public static Stream> batches(List source, int length) { if (length <= 0) throw new IllegalArgumentException("length = " + length); @@ -188,19 +168,6 @@ public static String getPartitionFileOutput(PropertyHelper propertyHelper) { return "./" + propertyHelper.getString(KnownProperties.ORIGIN_KEYSPACE_TABLE) + "_partitions.csv"; } - public static class PKRows implements Serializable { - private static final long serialVersionUID = 1L; - private List pkRows; - - public List getPkRows() { - return pkRows; - } - - public PKRows(List rows) { - pkRows = new ArrayList<>(rows); - } - } - public static class Partition implements Serializable { private static final long serialVersionUID = 1L; diff --git a/src/main/scala/com/datastax/cdm/job/BasePKJob.scala b/src/main/scala/com/datastax/cdm/job/BasePKJob.scala deleted file mode 100644 index 1c7d1c66..00000000 --- a/src/main/scala/com/datastax/cdm/job/BasePKJob.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright DataStax, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datastax.cdm.job - -import java.util - -abstract class BasePKJob extends BaseJob[SplitPartitions.PKRows] { - override def getParts(pieces: Int): util.Collection[SplitPartitions.PKRows] = { - // This takes a file with N rows and divides it into pieces of size N/pieces - // Each PKRows object contains a list of Strings that contain the PK to be parsed - SplitPartitions.getRowPartsFromFile(pieces, this.partitionFileNameInput) - } - -} \ No newline at end of file diff --git a/src/main/scala/com/datastax/cdm/job/MigrateRowsFromFile.scala b/src/main/scala/com/datastax/cdm/job/MigrateRowsFromFile.scala deleted file mode 100644 index 36069abb..00000000 --- a/src/main/scala/com/datastax/cdm/job/MigrateRowsFromFile.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright DataStax, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datastax.cdm.job - -object MigrateRowsFromFile extends BasePKJob { - setup("Migrate Rows from File Job", new CopyPKJobSessionFactory()) - execute() - finish() - - protected def execute(): Unit = { - if (!parts.isEmpty()) { - slices.foreach(slice => { - originConnection.withSessionDo(originSession => - targetConnection.withSessionDo(targetSession => - jobFactory.getInstance(originSession, targetSession, sc) - .processSlice(slice))) - }) - } - } -} diff --git a/src/resources/migrate_data.sh b/src/resources/migrate_data.sh deleted file mode 100644 index e285cbf6..00000000 --- a/src/resources/migrate_data.sh +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright DataStax, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#! /bin/bash - -########################################################################################################################### -# -# This script can be used to Migrate data between two Cassandra Clusters (including Astra) in chunks. It migrates data -# sequentially in progressive token-range slices. It also helps to restart migration from a point where the previous -# run might have stopped/failed for whatever reasons. -# -# Before running the script, update the below params -# SPARK_SUBMIT - Path to the spark-submit command -# PROPS_FILE - Path to the spark configuration for the table -# S_IDX - Change this value only if you want to set a custom starting point (e.g. after a previous incomplete run) -# -# *** IMP Note: Run this script using nohup in background using a logfile and tail the logfile to monitor progress *** -# e.g. nohup ./migrate_data.sh > logs/spark/migrate_data.out & -# -# To monitor migration progress, you could use the below command -# grep "Running Migrate for Partition Range" logs/spark/migrate_data.out -# -########################################################################################################################### - -# Path to spark-submit -SPARK_SUBMIT=/home/ubuntu/spark-3.5.1-bin-hadoop3-scala2.13/bin/spark-submit - -# Path to spark configuration for the table -PROPS_FILE=/home/ubuntu/sparkConf.properties - -# Starting partition token (Default is Min possible value of a Cassandra token - min long value in Java). -# Change this value only if you want to start from a custom partition token (e.g. when a migrate job failed midway) -S_IDX=-9223372036854775808 - -# ** DO NOT CHANGE ANYTHING BELOW THIS ** -SLICE=999999999999999999 - -echo "Starting Migration using $PROPS_FILE !!" - -# Migrate initial partition tokens from min-long to -9000000000000000000 -if [ $S_IDX -lt -9000000000000000000 ] -then - E_IDX=-9000000000000000001 - echo "Running Migrate for Partition Range $S_IDX to $E_IDX .." - $SPARK_SUBMIT --properties-file $PROPS_FILE --master "local[*]" --conf spark.origin.minPartition=$S_IDX --conf spark.origin.maxPartition=$E_IDX --class datastax.astra.migrate.Migrate cassandra-data-migrator-*.jar - S_IDX=-9000000000000000000 -fi - -# Migrate partition tokens from -9000000000000000000 to 8999999999999999999 in slices of 1000000000000000000 -while [ $S_IDX -lt 9000000000000000000 ] -do - if [ $S_IDX -gt 8223372036854775807 ] - then - E_IDX=8999999999999999999 - else - E_IDX=$(( $S_IDX + $SLICE )) - fi - echo "Running Migrate for Partition Range $S_IDX to $E_IDX .." - $SPARK_SUBMIT --properties-file $PROPS_FILE --master "local[*]" --conf spark.origin.minPartition=$S_IDX --conf spark.origin.maxPartition=$E_IDX --class datastax.astra.migrate.Migrate cassandra-data-migrator-*.jar - S_IDX=$(( $E_IDX + 1 )) -done - -# Migrate final partition tokens from 9000000000000000000 to max-long -E_IDX=9223372036854775807 -echo "Running Migrate for Partition Range $S_IDX to 9223372036854775807 .." -$SPARK_SUBMIT --properties-file $PROPS_FILE --master "local[*]" --conf spark.origin.minPartition=$S_IDX --conf spark.origin.maxPartition=$E_IDX --class datastax.astra.migrate.Migrate cassandra-data-migrator-*.jar -echo "Completed Migration using $PROPS_FILE !!" diff --git a/src/resources/partitions.csv b/src/resources/partitions.csv deleted file mode 100644 index 95701f5f..00000000 --- a/src/resources/partitions.csv +++ /dev/null @@ -1,7 +0,0 @@ -# This is a sample input file for job: MigratePartitionsFromFile -# list of primary-key fields separated by ' %% ' --6220480106867985210,-6120480106867985210 --4782224463879838018,-4682224463879838018 --507900353496146534,-407285462027022883 -2637884402540451982,4638499294009575633 -798869613692279889,8699484505161403540 diff --git a/src/resources/primary_key_rows.csv b/src/resources/primary_key_rows.csv deleted file mode 100644 index b7e731b6..00000000 --- a/src/resources/primary_key_rows.csv +++ /dev/null @@ -1,13 +0,0 @@ -# This is a sample input file for job: MigrateRowsFromFile -# list of primary-key fields separated by ' %% ' --1000154815969456717 %% 0 %% 10 %% 1021 --1000154815969456717 %% 0 %% 10 %% 1022 --1000154815969456717 %% 0 %% 10 %% 1023 --1000154815969456717 %% 0 %% 10 %% 1024 --1000154815969456717 %% 0 %% 10 %% 1025 --1000154815969456717 %% 0 %% 10 %% 1026 --1000154815969456717 %% 0 %% 10 %% 1027 --1000154815969456717 %% 0 %% 10 %% 1028 --1000154815969456717 %% 0 %% 10 %% 1029 --1000154815969456717 %% 0 %% 10 %% 1030 - diff --git a/src/test/java/com/datastax/cdm/job/SplitPartitionsTest.java b/src/test/java/com/datastax/cdm/job/SplitPartitionsTest.java index 8b9c8720..bbe1fdbd 100644 --- a/src/test/java/com/datastax/cdm/job/SplitPartitionsTest.java +++ b/src/test/java/com/datastax/cdm/job/SplitPartitionsTest.java @@ -64,12 +64,6 @@ void getSubPartitionsFromFileTest() throws IOException { assertEquals(25, partitions.size()); } - @Test - void getRowPartsFromFileTest() throws IOException { - List partitions = SplitPartitions.getRowPartsFromFile(5, "./src/resources/primary_key_rows.csv"); - assertEquals(6, partitions.size()); - } - @Test void getSubPartitionsFromHighNumPartTest() throws IOException { List partitions = SplitPartitions.getSubPartitionsFromFile(1000, "./src/resources/partitions.csv");