-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add make_bucket and scale_csv.py scripts.
- Loading branch information
Showing
2 changed files
with
216 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,115 @@ | ||
#!/bin/bash | ||
|
||
# This shell script creates a new GS bucket and configures it | ||
# with our standard read access and CORS policy. | ||
# | ||
# Usage: just run it and follow the prompts! | ||
|
||
# Default values | ||
DEFAULT_PROJECT="zetta-research" | ||
DEFAULT_REGION="us-east1" | ||
|
||
# Function to prompt with default value | ||
prompt_with_default() { | ||
local prompt=$1 | ||
local default=$2 | ||
local response | ||
|
||
read -p "${prompt} [${default}]: " response | ||
echo ${response:-$default} | ||
} | ||
|
||
# Create CORS configuration file | ||
cat > cors.json << 'EOL' | ||
[{ | ||
"maxAgeSeconds": 3600, | ||
"method": ["GET", "HEAD"], | ||
"origin": ["*"], | ||
"responseHeader": ["Content-Type", "Range"] | ||
}] | ||
EOL | ||
|
||
# Function to create a bucket with specified settings | ||
create_bucket() { | ||
local bucket_name=$1 | ||
local project_id=$2 | ||
local region=$3 | ||
|
||
echo "Creating bucket: gs://${bucket_name}" | ||
echo "Project ID: ${project_id}" | ||
echo "Region: ${region}" | ||
|
||
gsutil mb \ | ||
-p ${project_id} \ | ||
-c regional \ | ||
-l ${region} \ | ||
-b on \ | ||
gs://${bucket_name} | ||
|
||
if [ $? -eq 0 ]; then | ||
echo "Successfully created bucket: ${bucket_name}" | ||
|
||
# Set CORS policy | ||
echo "Setting CORS policy for: ${bucket_name}" | ||
gsutil cors set cors.json gs://${bucket_name} | ||
|
||
# Add public read access | ||
echo "Setting public read access..." | ||
gsutil iam ch allUsers:objectViewer gs://${bucket_name} | ||
|
||
else | ||
echo "Failed to create bucket: ${bucket_name}" | ||
return 1 | ||
fi | ||
} | ||
|
||
# Main execution | ||
echo "Starting bucket creation process..." | ||
|
||
# Check if gsutil is installed | ||
if ! command -v gsutil &> /dev/null; then | ||
echo "Error: gsutil is not installed. Please install Google Cloud SDK first." | ||
exit 1 | ||
fi | ||
|
||
# Check if user is authenticated | ||
if ! gsutil ls &> /dev/null; then | ||
echo "Error: Not authenticated. Please run 'gcloud auth login' first." | ||
exit 1 | ||
fi | ||
|
||
# Get project ID and region with defaults | ||
PROJECT_ID=$(prompt_with_default "Enter project ID" "$DEFAULT_PROJECT") | ||
REGION=$(prompt_with_default "Enter region" "$DEFAULT_REGION") | ||
|
||
# Ask user for bucket name (no default) | ||
read -p "Enter the bucket name (without gs:// prefix): " bucket_name | ||
|
||
# Validate bucket name | ||
if [[ -z "$bucket_name" ]]; then | ||
echo "Error: Bucket name cannot be empty" | ||
exit 1 | ||
fi | ||
|
||
# Show summary before proceeding | ||
echo -e "\nSummary of settings:" | ||
echo "Project ID: ${PROJECT_ID}" | ||
echo "Region: ${REGION}" | ||
echo "Bucket name: ${bucket_name}" | ||
read -p "Proceed with bucket creation? (y/N): " confirm | ||
if [[ ! "$confirm" =~ ^[yY]$ ]]; then | ||
echo "Bucket creation cancelled." | ||
exit 0 | ||
fi | ||
|
||
# Create the bucket | ||
create_bucket "$bucket_name" "$PROJECT_ID" "$REGION" | ||
if [ $? -ne 0 ]; then | ||
echo "Error occurred while creating bucket: ${bucket_name}" | ||
exit 1 | ||
fi | ||
|
||
# Cleanup | ||
rm cors.json | ||
|
||
echo "Bucket creation completed!" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,101 @@ | ||
""" | ||
This script scales the X,Y,Z and X2,Y2,Z2 columns (if any) in a CSV file | ||
from one resolution to another. | ||
""" | ||
|
||
from collections import namedtuple | ||
from pathlib import Path | ||
|
||
import pandas as pd | ||
|
||
# Create a named tuple for 3D vectors | ||
Vec3D = namedtuple("Vec3D", ["x", "y", "z"]) | ||
|
||
|
||
def input_or_default(prompt, value): | ||
response = input(f"{prompt} [{value}]: ") | ||
if response == "": | ||
response = value | ||
return response | ||
|
||
|
||
def input_vec3D(prompt="", default=None): | ||
while True: | ||
s = input(prompt + (f" [{default.x}, {default.y}, {default.z}]" if default else "") + ": ") | ||
if s == "" and default: | ||
return default | ||
try: | ||
x, y, z = map(float, s.replace(",", " ").split()) | ||
return Vec3D(x, y, z) | ||
except: | ||
print("Enter x, y, and z values separated by commas or spaces.") | ||
|
||
|
||
def get_scale_factors(current_res, desired_res): | ||
"""Calculate scale factors for each dimension""" | ||
return Vec3D( | ||
current_res.x / desired_res.x, current_res.y / desired_res.y, current_res.z / desired_res.z | ||
) | ||
|
||
|
||
def scale_coordinates(df, scale_factors): | ||
"""Scale the coordinates in the dataframe""" | ||
# Scale X, Y, Z columns | ||
df["X"] *= scale_factors.x | ||
df["Y"] *= scale_factors.y | ||
df["Z"] *= scale_factors.z | ||
|
||
# Scale X2, Y2, Z2 columns if they exist | ||
if all(col in df.columns for col in ["X2", "Y2", "Z2"]): | ||
df["X2"] *= scale_factors.x | ||
df["Y2"] *= scale_factors.y | ||
df["Z2"] *= scale_factors.z | ||
|
||
return df | ||
|
||
|
||
def main(): | ||
# Get input file path | ||
input_path: str | Path = "" | ||
while True: | ||
input_path = input("Enter path to input CSV file: ").strip() | ||
if Path(input_path).is_file(): | ||
break | ||
print("File not found. Please enter a valid path.") | ||
|
||
# Generate default output path | ||
input_path = Path(input_path) | ||
default_output = str(input_path.parent / f"{input_path.stem}_scaled{input_path.suffix}") | ||
|
||
# Get current and desired resolutions | ||
current_res = input_vec3D("Enter current resolution (x, y, z)") | ||
desired_res = input_vec3D("Enter desired resolution (x, y, z)") | ||
|
||
# Get output path | ||
output_path = input_or_default("\nEnter output file path", default_output) | ||
|
||
try: | ||
# Read the CSV file | ||
df = pd.read_csv(input_path) | ||
|
||
# Calculate scale factors | ||
scale_factors = get_scale_factors(current_res, desired_res) | ||
|
||
# Scale the coordinates | ||
df = scale_coordinates(df, scale_factors) | ||
|
||
# Write the scaled data to output file | ||
df.to_csv(output_path, index=False) | ||
|
||
print(f"\nScaled coordinates have been written to: {output_path}") | ||
print(f"Scale factors used (current/desired):") | ||
print(f"X: {scale_factors.x:.4f}") | ||
print(f"Y: {scale_factors.y:.4f}") | ||
print(f"Z: {scale_factors.z:.4f}") | ||
|
||
except Exception as e: | ||
print(f"An error occurred: {str(e)}") | ||
|
||
|
||
if __name__ == "__main__": | ||
main() |