steps

####### DOCKER #######

# Start the topology as defined in https://debezium.io/documentation/reference/stable/tutorial.html
# Debezium tutorials github repo is here: https://github.com/debezium/debezium-examples/tree/main/tutorial
# Kafka connect env variables description: https://hub.docker.com/r/debezium/connect-base
# Installing a Debezium Connector : https://debezium.io/documentation/reference/stable/install.html
docker-compose -f docker-compose-mysql.yaml up

# Control status of docker containers
docker-compose -f docker-compose-mysql.yaml ps


####### MYSQL #######

# Connect with root
docker-compose -f docker-compose-mysql.yaml exec mysql bash -c 'mysql -u root -pdebezium'

# Show users
SELECT user,host FROM mysql.user;

# Create database
CREATE DATABASE dataops;

# Authorize -debezium- user for -dataops- database
GRANT ALL ON dataops.* TO 'debezium'@'%';

FLUSH PRIVILEGES;

# Exit from root user using ctrl+d

# Log in with -debezium- user (you can see username and password in register-mysql.json)
docker-compose -f docker-compose-mysql.yaml exec mysql bash -c 'mysql -u debezium -pdbz'

# Create table
CREATE TABLE dataops.demo(
    customerId int,
    customerFName varchar(255),
    customerLName varchar(255),
    customerCity varchar(255)
);

# Select database
use dataops;

# Show tables
show tables;

# show variables like 'server_id';

####### CONNECTOR #######

# Start MySQL connector! You can register the MySQL connector by sending a POST request to the Kafka Connect API.
# Kafka Connect spawns a new connector instance from the class name specified in the connector.class field.
# MySQL Debezium Connector jars is required in Kafka Connect classpath
curl -i -X POST -H "Accept:application/json" -H  "Content-Type:application/json" http://localhost:8083/connectors/ -d @register-mysql.json

# To get more information about configuration use this url: https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-example-configuration

# CheatSheet for Kafka Connect REST API : https://cheatsheetfactory.geekyhacker.com/kafka/kafka-connect-rest

# Delete MySQL connector
curl -i -X DELETE localhost:8083/connectors/dataops-connector/

# You can take a look options of curl from here: https://gist.github.com/subfuzion/08c5d85437d5d4f00e58

# If you have problem with connector, take a look docker-compose logs
docker-compose -f docker-compose-mysql.yaml logs -f


####### KAFKA #######

# List kafka topics
docker-compose -f docker-compose-mysql.yaml exec kafka /kafka/bin/kafka-topics.sh \
--bootstrap-server kafka:9092 \
--list

# Consume messages from a Debezium topic
docker-compose -f docker-compose-mysql.yaml exec kafka /kafka/bin/kafka-console-consumer.sh \
    --bootstrap-server kafka:9092 \
    --from-beginning \
    --property print.key=true \
    --topic dbserver1.dataops.demo


####### FINAL #######

# Generate 2-3 batch data to simulate data flow using spark-mysql.py

# Update, Insert and Delete some records using python-mysql.py (or use mysql shell to do this operations)

# Learn about why there are two message when a row is deleted and more: https://debezium.io/documentation/reference/stable/transformations/event-flattening.html

# Examine console outputs via https://jsoncrack.com/editor

# Shut down the cluster
docker-compose -f docker-compose-mysql.yaml down