Instructions to use unsloth/Qwen3.6-27B-NVFP4 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Local Apps Settings
- Unsloth Studio
How to use unsloth/Qwen3.6-27B-NVFP4 with Unsloth Studio:
Install Unsloth Studio (macOS, Linux, WSL)
curl -fsSL https://unsloth.ai/install.sh | sh # Run unsloth studio unsloth studio -H 0.0.0.0 -p 8888 # Then open http://localhost:8888 in your browser # Search for unsloth/Qwen3.6-27B-NVFP4 to start chatting
Install Unsloth Studio (Windows)
irm https://unsloth.ai/install.ps1 | iex # Run unsloth studio unsloth studio -H 0.0.0.0 -p 8888 # Then open http://localhost:8888 in your browser # Search for unsloth/Qwen3.6-27B-NVFP4 to start chatting
Using HuggingFace Spaces for Unsloth
# No setup required # Open https://huggingface.co/spaces/unsloth/studio in your browser # Search for unsloth/Qwen3.6-27B-NVFP4 to start chatting
Load model with FastModel
pip install unsloth from unsloth import FastModel model, tokenizer = FastModel.from_pretrained( model_name="unsloth/Qwen3.6-27B-NVFP4", max_seq_length=2048, )
Tool allucinations in RTX PRO 6000
Hi, experiencing a few tools hallucinations with unsloth/Qwen3.6-27B-NVFP4 versus FP8 with "vllm" in a single RTX PRO 6000 for Hermes Agent. Also sometimes getting tool template output.
I tried running it with the following command:
SAFETENSORS_FAST_GPU=1
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
vllm serve unsloth/Qwen3.6-27B-NVFP4
--trust-remote-code \
--tensor-parallel-size 1 \
--max-num-batched-tokens 8192 \
--gpu-memory-utilization 0.95 \
--max-num-seqs 16 \
--enable-auto-tool-choice \
--tool-call-parser qwen3_coder \
--reasoning-parser qwen3 \
--dtype bfloat16 \
--max-model-len 262144 \
--enable-prefix-caching \
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
Use this chat tempate -
{%- set image_count = namespace(value=0) %}
{%- set video_count = namespace(value=0) %}
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
{%- if content is string %}
{{- content }}
{%- elif content is iterable and content is not mapping %}
{%- for item in content %}
{%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
{%- if is_system_content %}
{{- raise_exception('System message cannot contain images.') }}
{%- endif %}
{%- if do_vision_count %}
{%- set image_count.value = image_count.value + 1 %}
{%- endif %}
{%- if add_vision_id is defined and add_vision_id %}
{{- 'Picture ' ~ image_count.value ~ ': ' }}
{%- endif %}
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
{%- elif 'video' in item or item.type == 'video' %}
{%- if is_system_content %}
{{- raise_exception('System message cannot contain videos.') }}
{%- endif %}
{%- if do_vision_count %}
{%- set video_count.value = video_count.value + 1 %}
{%- endif %}
{%- if add_vision_id is defined and add_vision_id %}
{{- 'Video ' ~ video_count.value ~ ': ' }}
{%- endif %}
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
{%- elif 'text' in item %}
{{- item.text }}
{%- else %}
{{- raise_exception('Unexpected item type in content.') }}
{%- endif %}
{%- endfor %}
{%- elif content is none or content is undefined %}
{{- '' }}
{%- else %}
{{- raise_exception('Unexpected content type.') }}
{%- endif %}
{%- endmacro %}
{%- set ns_flags = namespace(enable_thinking=true, has_tools=false) %}
{%- if enable_thinking is defined %}
{%- set ns_flags.enable_thinking = enable_thinking %}
{%- endif %}
{%- if not messages %}
{{- raise_exception('No messages provided.') }}
{%- endif %}
{%- set system_content = '' %}
{%- set has_system = false %}
{%- if messages[0].role == 'system' or messages[0].role == 'developer' %}
{%- set has_system = true %}
{%- set system_content = render_content(messages[0].content, false, true)|trim %}
{%- if '<|think_off|>' in system_content %}
{%- set ns_flags.enable_thinking = false %}
{%- set system_content = system_content | replace('<|think_off|>', '') %}
{%- endif %}
{%- if '<|think_on|>' in system_content %}
{%- set ns_flags.enable_thinking = true %}
{%- set system_content = system_content | replace('<|think_on|>', '') %}
{%- endif %}
{%- set system_content = system_content | trim %}
{%- endif %}
{%- if tools and tools is iterable and tools is not mapping %}
{%- set ns_flags.has_tools = true %}
{{- '<|im_start|>system\n' }}
{{- "# Tools\n\nYou have access to the following functions:\n\n" }}
{%- for tool in tools %}
{%- set fn = tool.function if tool.function is defined else tool %}
{%- set props = {} %}
{%- set req = [] %}
{%- if fn.parameters is defined and fn.parameters is mapping %}
{%- if fn.parameters.properties is defined %}
{%- set props = fn.parameters.properties %}
{%- endif %}
{%- if fn.parameters.required is defined %}
{%- set req = fn.parameters.required %}
{%- endif %}
{%- endif %}
{%- set ns_p = namespace(sig='') %}
{%- for pname in props %}
{%- set pdef = props[pname] %}
{%- set ptype = 'any' %}
{%- if pdef.type is defined %}
{%- if pdef.type == 'array' or pdef.type == 'object' %}
{%- set ptype = 'array|object' %}
{%- elif pdef.enum is defined and pdef.enum is iterable and pdef.enum is not string and pdef.enum is not mapping %}
{%- set ptype = pdef.enum | join('|') %}
{%- else %}
{%- set ptype = pdef.type %}
{%- endif %}
{%- endif %}
{%- set part = pname ~ ('' if pname in req else '?') ~ ': ' ~ ptype %}
{%- set ns_p.sig = ns_p.sig ~ ', ' ~ part if ns_p.sig else part %}
{%- endfor %}
{{- '\n- ' ~ fn.name ~ '(' ~ ns_p.sig ~ ')' }}
{%- if fn.description is defined %}
{{- ' — ' ~ fn.description }}
{%- endif %}
{%- if fn.parameters is defined and fn.parameters is mapping %}
{%- for pname in props %}
{%- set pdef = props[pname] %}
{%- if pdef.type is defined and (pdef.type == 'array' or pdef.type == 'object') %}
{{- '\n - ' ~ pname ~ ' schema: ' ~ pdef | tojson }}
{%- elif pdef.description is defined and pdef.description is string %}
{{- '\n - ' ~ pname ~ ' description: ' ~ pdef.description }}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- endfor %}
{{- "\n" }}
{%- set ns_scan = namespace(final_enable=ns_flags.enable_thinking) %}
{%- for message in messages %}
{%- set content_str = message.content | string %}
{%- if '<|think_off|>' in content_str %}
{%- set ns_scan.final_enable = false %}
{%- elif '<|think_on|>' in content_str %}
{%- set ns_scan.final_enable = true %}
{%- endif %}
{%- endfor %}
{%- if ns_scan.final_enable %}
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\nBrief explanation of tool call\n\n\n\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...> block must be nested within XML tags\n- Required parameters MUST be specified\n- You MUST provide reasoning for your function call within a block BEFORE the \n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }}
{%- else %}
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...> block must be nested within XML tags\n- Required parameters MUST be specified\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }}
{%- endif %}
{%- if has_system and system_content %}
{{- '\n\n' + system_content }}
{%- endif %}
{{- '<|im_end|>\n' }}
{%- elif has_system and system_content %}
{{- '<|im_start|>system\n' + system_content + '<|im_end|>\n' }}
{%- endif %}
{%- for message in messages %}
{%- set is_system = (message.role == "system" or message.role == "developer") %}
{%- set content = render_content(message.content, true, is_system)|trim %}
{%- if '<|think_off|>' in content %}
{%- set ns_flags.enable_thinking = false %}
{%- set content = content | replace('<|think_off|>', '') | trim %}
{%- elif '<|think_on|>' in content %}
{%- set ns_flags.enable_thinking = true %}
{%- set content = content | replace('<|think_on|>', '') | trim %}
{%- endif %}
{%- if is_system %}
{%- if not loop.first and content %}
{{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
{%- endif %}
{%- elif message.role == "user" %}
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
{%- elif message.role == "assistant" %}
{%- set content = content | replace('</ think>', '') | replace('< /think>', '') | replace('', '') | replace('', '') | replace('< think>', '') %}
{%- set reasoning_content = '' %}
{%- if message.reasoning_content is string %}
{%- set reasoning_content = message.reasoning_content %}
{%- else %}
{%- if '' in content or '' in content %}
{%- set think_end_token = '' if '' in content else '' %}
{%- set reasoning_parts = content.split(think_end_token) %}
{%- set reasoning_content = reasoning_parts[0] %}
{%- set content = reasoning_parts[1] %}
{%- if '' in reasoning_content %}
{%- set reasoning_content = reasoning_content.split('')[1] %}
{%- endif %}
{%- elif '' in content %}
{%- if '' in content %}
{%- set content = content | replace('', '\n', 1) %}
{%- set reasoning_parts = content.split('') %}
{%- set reasoning_content = reasoning_parts[0] %}
{%- set content = reasoning_parts[1] %}
{%- if '' in reasoning_content %}
{%- set reasoning_content = reasoning_content.split('')[1] %}
{%- endif %}
{%- else %}
{%- set reasoning_content = content.split('')[1] %}
{%- set content = '' %}
{%- endif %}
{%- endif %}
{%- endif %}
{%- set reasoning_content = reasoning_content | trim %}
{%- set content = content | trim %}
{%- set show_think = false %}
{%- if reasoning_content %}
{%- if preserve_thinking is defined and preserve_thinking %}
{%- set show_think = true %}
{%- elif loop.last %}
{%- set show_think = true %}
{%- endif %}
{%- endif %}
{%- if show_think %}
{{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
{%- for tool_call in message.tool_calls %}
{%- if tool_call.function is defined %}
{%- set tool_call = tool_call.function %}
{%- endif %}
{%- if loop.first and content %}
{{- '\n\n\n<function=' + tool_call.name + '>\n' }}
{%- else %}
{{- '\n\n<function=' + tool_call.name + '>\n' }}
{%- endif %}
{%- if tool_call.arguments is defined and tool_call.arguments is mapping %}
{%- if tool_call.arguments|length > 0 %}
{%- for args_name in tool_call.arguments %}
{%- set args_value = tool_call.arguments[args_name] %}
{{- '<parameter=' + args_name + '>\n' }}
{%- set args_value = args_value | tojson if args_value is mapping or (args_value is iterable and args_value is not string) else args_value | string %}
{{- args_value }}
{{- '\n\n' }}
{%- endfor %}
{%- endif %}
{%- elif tool_call.arguments is defined and tool_call.arguments is string %}
{%- if tool_call.arguments|trim|length > 0 %}
{{- tool_call.arguments }}
{{- '\n' }}
{%- endif %}
{%- endif %}
{{- '\n' }}
{%- endfor %}
{%- endif %}
{{- '<|im_end|>\n' }}
{%- elif message.role == "tool" %}
{%- if loop.previtem and loop.previtem.role != "tool" %}
{{- '<|im_start|>user' }}
{%- endif %}
{{- '\n\n' }}
{{- content }}
{{- '\n' }}
{%- if not loop.last and loop.nextitem.role != "tool" %}
{{- '<|im_end|>\n' }}
{%- elif loop.last %}
{{- '<|im_end|>\n' }}
{%- endif %}
{%- else %}
{{- raise_exception('Unexpected message role.') }}
{%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}
{{- '<|im_start|>assistant\n' }}
{%- if ns_flags.enable_thinking is false %}
{{- '\n\n\n\n' }}
{%- else %}
{{- '\n' }}
{%- endif %}
{%- endif %}