Convert model to gguf with specified quant #115
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Convert model to gguf with specified quant | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| source_model_id: | |
| description: "Source HuggingFace model ID to pull. For ex: meta-llama/Meta-Llama-3.1-8B-Instruct" | |
| required: true | |
| source_model_size: | |
| description: "The model size. For ex: 8b" | |
| required: true | |
| type: string | |
| target_model_id: | |
| description: "Target HuggingFace model ID to push. For ex: llama3.1" | |
| required: true | |
| type: string | |
| quantization_level: | |
| description: "Quantization level (e.g., 'q4-km') or 'all' for all levels" | |
| required: true | |
| type: string | |
| default: 'all' | |
| env: | |
| USER_NAME: cortexso | |
| SOURCE_MODEL_ID: ${{ inputs.source_model_id }} | |
| SOURCE_MODEL_SIZE: ${{ inputs.source_model_size }} | |
| TARGET_MODEL_ID: ${{ inputs.target_model_id }} | |
| QUANT_LEVEL: ${{ inputs.quantization_level }} | |
| jobs: | |
| converter: | |
| runs-on: ubuntu-20-04-gguf | |
| timeout-minutes: 7200 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| repository: janhq/cortex.llamacpp | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 # v5.1.1 | |
| with: | |
| python-version: '3.12' | |
| # architecture: 'x64' | |
| - name: Cache Python packages | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.local/share/pip | |
| .venv | |
| key: ${{ runner.os }}-pip-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: | | |
| pip3 install -r llama.cpp/requirements.txt | |
| pip3 install hf-transfer | |
| pip3 install openai | |
| git lfs install | |
| - name: Extract MODEL_NAME | |
| run: | | |
| SOURCE_MODEL_ID="${{ env.SOURCE_MODEL_ID }}" | |
| MODEL_NAME="$(echo $SOURCE_MODEL_ID | rev | cut -d/ -f1 | rev)" | |
| echo $MODEL_NAME | |
| MODEL_NAME="$(echo $MODEL_NAME | tr '[:upper:]' '[:lower:]')" | |
| echo $MODEL_NAME | |
| echo "MODEL_NAME=$MODEL_NAME" >> $GITHUB_ENV | |
| - name: Print environment variables | |
| run: | | |
| echo "SOURCE_MODEL_ID: ${{ env.SOURCE_MODEL_ID }}" | |
| echo "MODEL_NAME: ${{ env.MODEL_NAME }}" | |
| # - name: Check file existence | |
| # id: check_files | |
| # uses: andstor/file-existence-action@v1 | |
| # with: | |
| # files: "/mnt/models/${{ env.MODEL_NAME }}/hf" | |
| - name: Prepare folders | |
| # if: steps.check_files.outputs.files_exists == 'false' | |
| run: | | |
| mkdir -p /mnt/models/${{ env.MODEL_NAME }}/hf | |
| mkdir -p /mnt/models/.cache | |
| - name: Download Hugging Face model | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 10 | |
| max_attempts: 5 | |
| command: HF_HUB_ETAG_TIMEOUT=500 huggingface-cli download --repo-type model --local-dir /mnt/models/${{ env.MODEL_NAME }}/hf --cache-dir /mnt/models/.cache --token ${{ secrets.HUGGINGFACE_TOKEN_READ }} ${{ env.SOURCE_MODEL_ID }} | |
| - name: Create README.md | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| run: | | |
| python3 - << EOF | |
| import os | |
| from openai import OpenAI | |
| # Initialize the OpenAI client | |
| client = OpenAI(api_key=os.environ['OPENAI_API_KEY']) | |
| # Inputs | |
| source_model_id = os.environ['SOURCE_MODEL_ID'] | |
| model_name = os.environ['MODEL_NAME'] | |
| model_variant = os.environ['SOURCE_MODEL_SIZE'] | |
| target_model_id = os.environ['TARGET_MODEL_ID'] | |
| user_name = os.environ['USER_NAME'] | |
| # Extract author from source model ID | |
| author = source_model_id.split('/')[0] | |
| # Call OpenAI API to generate overview | |
| completion = client.chat.completions.create( | |
| model="gpt-4o-mini", | |
| messages=[ | |
| {"role": "system", "content": "You are a helpful assistant."}, | |
| { | |
| "role": "user", | |
| "content": f"Write a concise overview for a machine learning model named '{target_model_id}' derived from '{source_model_id}', highlighting its purpose, use cases, and performance. You DO NOT generate title (# heading 1) and summary sections. For the overview section, make it concise and in a paragraph of 5 sentences" | |
| } | |
| ] | |
| ) | |
| overview = completion.choices[0].message.content.strip() | |
| # README.md template | |
| readme_template = f"""\ | |
| --- | |
| license: mit | |
| --- | |
| ## Overview | |
| {overview} | |
| ## Variants | |
| | No | Variant | Cortex CLI command | | |
| | --- | --- | --- | | |
| | 1 | [gguf](https://huggingface.co/{user_name}/{model_name}/tree/main) | cortex run {model_name} | | |
| ## Use it with Jan (UI) | |
| 1. Install **Jan** using [Quickstart](https://jan.ai/docs/quickstart) | |
| 2. Use in Jan model Hub: | |
| ```text | |
| {user_name}/{model_name} | |
| ``` | |
| ## Use it with Cortex (CLI) | |
| 1. Install **Cortex** using [Quickstart](https://cortex.jan.ai/docs/quickstart) | |
| 2. Run the model with command: | |
| ```bash | |
| cortex run {model_name} | |
| ``` | |
| ## Credits | |
| - **Author:** {author} | |
| - **Converter:** [Homebrew](https://www.homebrew.ltd/) | |
| - **Original License:** [License](https://huggingface.co/{user_name}/{model_name}#license) | |
| """ | |
| # Write the README.md file | |
| with open('README.md', 'w') as f: | |
| f.write(readme_template) | |
| EOF | |
| - name: Upload README.md to HuggingFace Repository | |
| env: | |
| HF_TOKEN: ${{ secrets.HUGGINGFACE_TOKEN_WRITE }} | |
| run: | | |
| python3 - << EOF | |
| from huggingface_hub import HfApi | |
| import os | |
| # Initialize the Hugging Face API | |
| api = HfApi(token=os.environ['HF_TOKEN']) | |
| # Repository details | |
| repo_id = f"${{ env.USER_NAME }}/${{ env.MODEL_NAME }}" | |
| # Upload README.md | |
| api.upload_file( | |
| path_or_fileobj="README.md", | |
| path_in_repo="README.md", | |
| repo_id=repo_id, | |
| token=os.environ['HF_TOKEN'] | |
| ) | |
| print("README.md uploaded successfully") | |
| EOF | |
| - name: Build lib for quantize | |
| run: | | |
| cd llama.cpp | |
| cmake -B build | |
| cmake --build build --config Release | |
| cd ../../ | |
| - name: Convert to GGUF | |
| run: | | |
| mkdir -p /mnt/models/${{ env.MODEL_NAME }}/gguf | |
| huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_READ }} --add-to-git-credential | |
| python3 llama.cpp/convert_hf_to_gguf.py "/mnt/models/${{ env.MODEL_NAME }}/hf" --outfile "/mnt/models/${{ env.MODEL_NAME }}/gguf/model-origin.gguf" | |
| huggingface-cli logout | |
| - name: Quantize the model | |
| run: | | |
| declare -A quant_map=( | |
| ["q2-k"]="Q2_K" | |
| ["q3-ks"]="Q3_K_S" | |
| ["q3-km"]="Q3_K_M" | |
| ["q3-kl"]="Q3_K_L" | |
| ["q4-ks"]="Q4_K_S" | |
| ["q4-km"]="Q4_K_M" | |
| ["q5-ks"]="Q5_K_S" | |
| ["q5-km"]="Q5_K_M" | |
| ["q6-k"]="Q6_K" | |
| ["q8-0"]="Q8_0" | |
| ) | |
| if [ "${{ env.QUANT_LEVEL }}" = "all" ]; then | |
| quant_levels=("q2-k" "q3-ks" "q3-km" "q3-kl" "q4-ks" "q4-km" "q5-ks" "q5-km" "q6-k" "q8-0") | |
| else | |
| quant_levels=("${{ env.QUANT_LEVEL }}") | |
| fi | |
| for quant in "${quant_levels[@]}"; do | |
| mkdir -p /mnt/models/${{ env.MODEL_NAME }}/gguf/${quant}/ | |
| [ ! -f /mnt/models/${{ env.MODEL_NAME }}/gguf/${quant}/model.gguf ] && ./llama.cpp/build/bin/llama-quantize /mnt/models/${{ env.MODEL_NAME }}/gguf/model-origin.gguf /mnt/models/${{ env.MODEL_NAME }}/gguf/${quant}/model.gguf ${quant_map[${quant}]} | |
| done | |
| rm -rf /mnt/models/${{ env.MODEL_NAME }}/gguf/model-origin.gguf | |
| - name: Upload to Hugging Face | |
| run: | | |
| huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_WRITE }} --add-to-git-credential | |
| declare -A quant_map=( | |
| ["q2-k"]="q2_k" | |
| ["q3-ks"]="q3_k_s" | |
| ["q3-km"]="q3_k_m" | |
| ["q3-kl"]="q3_k_l" | |
| ["q4-ks"]="q4_k_s" | |
| ["q4-km"]="q4_k_m" | |
| ["q5-ks"]="q5_k_s" | |
| ["q5-km"]="q5_k_m" | |
| ["q6-k"]="q6_k" | |
| ["q8-0"]="q8_0" | |
| ) | |
| if [ "${{ env.QUANT_LEVEL }}" = "all" ]; then | |
| quant_levels=("q2-k" "q3-ks" "q3-km" "q3-kl" "q4-ks" "q4-km" "q5-ks" "q5-km" "q6-k" "q8-0") | |
| else | |
| quant_levels=("${{ env.QUANT_LEVEL }}") | |
| fi | |
| for quant in "${quant_levels[@]}"; do | |
| new_name="${{ env.MODEL_NAME }}-${quant_map[${quant}]}.gguf" | |
| mv "/mnt/models/${{ env.MODEL_NAME }}/gguf/${quant}/model.gguf" "/mnt/models/${{ env.MODEL_NAME }}/gguf/${quant}/${new_name}" | |
| huggingface-cli upload "${{ env.USER_NAME }}/${{ env.TARGET_MODEL_ID }}" "/mnt/models/${{ env.MODEL_NAME }}/gguf/${quant}/" . | |
| done | |
| rm -rf /mnt/models/${{ env.MODEL_NAME }}/gguf/* | |
| huggingface-cli logout | |
| rm -rf llama.cpp/build/ |