From d942658721578d5a19dc0a3a1fc1e4e122ab6a74 Mon Sep 17 00:00:00 2001
From: slSeanWU <summer7sean@gmail.com>
Date: Thu, 30 Oct 2025 15:27:24 -0400
Subject: [PATCH] add readme for inference

---
 README.md        | 61 +++++++++++++++++++++++++++++++++++++++++++++---
 generate_vllm.py |  8 +++++--
 2 files changed, 64 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 4cd0d92..b113ba4 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,13 @@
 # MIDI-LLM
-[NeurIPS AI4Music '25] MIDI-LLM: Adapting LLMs for text-to-MIDI music generation.
 
-Built with Llama 3.2 (1B) LLM.
+Built on **Llama 3.2 (1B)** with an extended vocabulary for MIDI tokens.
 
-# Setup
+## Research Paper
+- Shih-Lun Wu, Yoon Kim, and Cheng-Zhi Anna Huang.  
+  "**MIDI-LLM: Adapting large language models for text-to-MIDI music generation**."  
+  NeurIPS AI4Music Workshop, 2025.
+
+## Setup
 
 - A GPU with 16GB+ VRAM and CUDA 12.x is recommended
 
@@ -48,4 +52,55 @@ pip install -r requirements.txt
 - Verify all installation
 ```bash
 python -c "import torch; from vllm import LLM; from anticipation.convert import events_to_midi; print('Setup successful')"
+```
+
+## Run Inference with vLLM
+### Example 1: Single prompt
+```bash
+python generate_vllm.py \
+    --model slseanwu/MIDI-LLM_Llama-3.2-1B # will pull from huggingface hub \
+    --prompt "A cheerful piano melody"
+```
+This will output 4 MIDIs (and the synthesized MP3s) conditioned on the same input prompt
+
+### Example 2: Batch generation from file
+
+```bash
+python generate_vllm.py \
+    --model slseanwu/MIDI-LLM_Llama-3.2-1B \
+    --prompts_file some_example_prompts.txt \
+    --fp8 \
+    --no-synthesize
+```
+- `some_example_prompts.txt` should contain one prompt per line.
+- `--fp8` performs dynamic weight quantization for faster inference.
+- `--no-synthesize` skips audio synthesis (i.e., outputs MIDI only).
+
+### Example 3: Interactive mode
+
+```bash
+python generate_vllm.py \
+    --model slseanwu/MIDI-LLM_Llama-3.2-1B \
+    --output_root generations_interactive/ \
+    --interactive
+```
+- Outputs will be saved under `generations_interactive/`
+This loads the model once, then lets you enter prompts interactively. Press Enter with empty prompt to exit.
+
+### More options
+See full options with:
+```bash
+python generate_vllm.py --help
+```
+
+### Inference Output Structure
+```
+[output_root]/
+└── 2025-10-30_143022/           # Session timestamp
+    ├── 20251030_143022_prompt_1/
+    │   ├── prompt.txt
+    │   ├── gen_1.mid
+    │   ├── gen_1.mp3
+    │   └── ...
+    └── generation_stats.json
 ```
\ No newline at end of file
diff --git a/generate_vllm.py b/generate_vllm.py
index 5958d47..cc728f0 100644
--- a/generate_vllm.py
+++ b/generate_vllm.py
@@ -83,6 +83,7 @@ def prepare_vllm_model(
     
     model = LLM(
         model=model_path,
+        tokenizer=model_path,  # Explicitly use tokenizer from model checkpoint
         quantization="fp8" if do_fp8_quantization else None,
         gpu_memory_utilization=gpu_memory_utilization,
         trust_remote_code=True,
@@ -364,10 +365,10 @@ Examples:
     
     print(f"Output directory: {output_dir.absolute()}\n")
     
-    # Load tokenizer
+    # Load tokenizer from the model checkpoint (not base Llama!)
     print("Loading tokenizer...")
     tokenizer = AutoTokenizer.from_pretrained(
-        LLAMA_MODEL_NAME,
+        args.model,  # Use the model checkpoint path, not base Llama
         cache_dir=args.cache_dir,
         pad_token="<|eot_id|>",
     )
@@ -472,6 +473,9 @@ Examples:
                     soundfont_path=args.soundfont if args.synthesize else None,
                     synthesize=args.synthesize
                 )
+
+                # Print input prompt
+                print(f"Input prompt: {user_prompt}")
                 
                 # Print mini summary
                 print(f"\n✓ Generated {interactive_stats['successful_generations']}/{args.n_outputs} outputs")