Compare model versions

Diff

Before

base_model:
- Qwen/Qwen3.8-27B
tags:
- vllm
- llm-compressor
- qwen3_5
- compressed-tensors
- int4
- conversational
license: apache-2.0
pipeline_tag: image-text-to-text
library_name: transformers

After

base_model:
- Qwen/Qwen3.8-27B
tags:
- vllm
- llm-compressor
- qwen3_5
- compressed-tensors
- int4
- conversational
- fp8
license: apache-2.0
pipeline_tag: image-text-to-text
library_name: transformers

Diff

Before

| Eval | Qwen/Qwen3.8-27B (BF16) | RedHatAI/Qwen3.8-27B-INT4 | Recovery (INT4/BF16) |
| --- | --- | --- | --- |
| gsm8k_platinum_cot_llama | 0.9589 | 0.9677 | 100.92% |
| ifeval | 0.9187 | 0.9211 | 100.27% |
| aime25 | 0.9375 | 0.9542 | 101.78% |
| math_500 | 0.8360 | 0.8380 | 100.24% |
| gpqa_diamond | 0.8923 | 0.8636 | 96.78% |

After

| Eval | Qwen/Qwen3.8-27B (BF16) | RedHatAI/Qwen3.8-27B-INT4 | Recovery (INT4/BF16) |
| --- | --- | --- | --- |
| gsm8k_platinum (strict-match) | 95.75% | 96.77% | 101.07% |
| ifeval (prompt-level strict) | 92.24% | 91.93% | 99.67% |
| mmlu_pro_chat (exact-match) | 84.46% | 83.45% | 98.81% |
| math_500 (pass@1) | 83.73% | 83.33% | 99.52% |
| gpqa_diamond (pass@1) | 89.23% | 87.88% | 98.49% |
| aime25 (avg@1, 8 seeds) | 95.42% | 94.17% | 98.69% |

Diff

Before

This model was obtained by quantizing the weights of [Qwen/Qwen3.8-27B](https://huggingface.co/Qwen/Qwen3.8-27B) to int4, ready for inference with vLLM.
Weights are quantized to int4 with a group size of 128. Only the weights of the linear operators within transformer blocks are quantized using [LLM Compressor](https://github.com/vllm-project/llm-compressor). Vision tower and outputhead layers are kept in their original precision.

After

This model was obtained by quantizing the weights of [Qwen/Qwen3.8-27B](https://huggingface.co/Qwen/Qwen3.8-27B) to int4, ready for inference with vLLM.
Weights are quantized to int4 with a group size of 128 and the KV Cache has been quantized to FP8. Only the weights of the linear operators within transformer blocks are quantized using [LLM Compressor](https://github.com/vllm-project/llm-compressor). Vision tower and outputhead layers are kept in their original precision.

Diff

Before

```python
import torch
from compressed_tensors.quantization.quant_scheme import W4A16, QuantizationScheme
from datasets import load_dataset
from transformers import AutoProcessor, Qwen3_5ForConditionalGeneration

from llmcompressor import oneshot
from llmcompressor.modifiers.gptq import GPTQModifier
from llmcompressor.modifiers.transform.awq import AWQModifier
from llmcompressor.utils import load_context

MODEL_ID = "Qwen/Qwen3.8-27B"

# Load model.
with load_context(Qwen3_5ForConditionalGeneration):
    model = Qwen3_5ForConditionalGeneration.from_pretrained(MODEL_ID)
processor = AutoProcessor.from_pretrained(MODEL_ID)


recipe = [
    AWQModifier(duo_scaling="both"),
    GPTQModifier(
        targets="Linear", 
        scheme="W4A16",
        ignore=[
            "re:visual.*",
            "re:model.visual.*",
            r"re:.*lm_head",
            "re:.*embed_tokens$",
            r"re:.*linear_attn\.in_proj_a$",
            r"re:.*linear_attn\.in_proj_b$",
        ],
    )
]

NUM_CALIBRATION_SAMPLES = 512
MAX_SEQUENCE_LENGTH = 4096

ds = load_dataset(
    "mlabonne/open-perfectblend",
    split=f"train[:{NUM_CALIBRATION_SAMPLES}]",
)
ds = ds.shuffle(seed=42)

ROLE_MAP = {"human": "user", "gpt": "assistant"}


def preprocess_function(example):
    messages = [
        {
            "role": ROLE_MAP.get(msg["from"], msg["from"]),
            "content": [{"type": "text", "text": msg["value"]}],
        }
        for msg in example["conversations"]
    ]
    return processor.apply_chat_template(
        messages,
        tokenize=True,
        return_dict=True,
        add_generation_prompt=False,
        processor_kwargs={
            "return_tensors": "pt",
            "padding": False,
            "truncation": True,
            "max_length": MAX_SEQUENCE_LENGTH,
            "add_special_tokens": False,
        },
    )


ds = ds.map(preprocess_function, batched=False, remove_columns=ds.column_names)


def data_collator(batch):
    assert len(batch) == 1
    return {key: torch.tensor(value) for key, value in batch[0].items()}


# Apply quantization.
oneshot(

14 more lines not shown.

After

```python
import torch
from datasets import load_dataset
from transformers import AutoProcessor, Qwen3_5ForConditionalGeneration

from llmcompressor import oneshot
from llmcompressor.modifiers.gptq import GPTQModifier
from llmcompressor.modifiers.transform.awq import AWQModifier
from llmcompressor.utils import load_context

MODEL_ID = "Qwen/Qwen3.8-27B"

# Load model.
with load_context(Qwen3_5ForConditionalGeneration):
    model = Qwen3_5ForConditionalGeneration.from_pretrained(MODEL_ID)
processor = AutoProcessor.from_pretrained(MODEL_ID)


recipe = [
    AWQModifier(duo_scaling="both"),
    GPTQModifier(
        targets="Linear",
        scheme="W4A16",
        ignore=[
            "re:visual.*",
            "re:model.visual.*",
            r"re:.*lm_head",
            "re:.*embed_tokens$",
            r"re:.*linear_attn\.in_proj_a$",
            r"re:.*linear_attn\.in_proj_b$",
        ],
        kv_cache_scheme={
            "num_bits": 8,
            "type": "float",
            "symmetric": True,
            "strategy": "tensor",
            "dynamic": False,
            "observer": "static_minmax",
        },
    ),
]

NUM_CALIBRATION_SAMPLES = 512
MAX_SEQUENCE_LENGTH = 4096

ds = load_dataset(
    "mlabonne/open-perfectblend",
    split=f"train[:{NUM_CALIBRATION_SAMPLES}]",
)
ds = ds.shuffle(seed=42)

ROLE_MAP = {"human": "user", "gpt": "assistant"}


def preprocess_function(example):
    messages = [
        {
            "role": ROLE_MAP.get(msg["from"], msg["from"]),
            "content": [{"type": "text", "text": msg["value"]}],
        }
        for msg in example["conversations"]
    ]
    return processor.apply_chat_template(
        messages,
        tokenize=True,
        return_dict=True,
        add_generation_prompt=False,
        processor_kwargs={
            "return_tensors": "pt",
            "padding": False,
            "truncation": True,
            "max_length": MAX_SEQUENCE_LENGTH,
            "add_special_tokens": False,
        },
    )


ds = ds.map(preprocess_function, batched=False, remove_columns=ds.column_names)

22 more lines not shown.