Installation
# Create conda environment
conda create -n bitnet-cpp python=3.9
conda activate bitnet-cpp
# Install dependencies
pip install -r requirements.txt
# Download model
huggingface-cli download microsoft/BitNet-b1.58-2B-4T-gguf \
--local-dir models/BitNet-b1.58-2B-4T
# Setup environment
python setup_env.py -md models/BitNet-b1.58-2B-4T -q i2_s
Run Inference
python run_inference.py \
-m models/BitNet-b1.58-2B-4T/ggml-model-i2_s.gguf \
-p "You are a helpful assistant" \
-cnv