[fix](cli): detect bound ports before launch (#2071)

* [fix](cli): detect bound ports before launch

* [fix](cli): align port reuse check by platform

.github/workflows/release-fake-tag.yml

@@ -30,7 +30,12 @@ jobs:
 
       - name: Create and push tag
         run: |
+          TAG=${{ steps.get_version.outputs.TAG }}
           git config user.name "ktransformers-bot"
           git config user.email "ktransformers-bot@users.noreply.github.com"
-          git tag ${{ steps.get_version.outputs.TAG }}
-          git push origin ${{ steps.get_version.outputs.TAG }}
+          if git ls-remote --tags --exit-code origin "refs/tags/${TAG}" > /dev/null 2>&1; then
+            echo "Tag ${TAG} already exists on origin, skipping."
+            exit 0
+          fi
+          git tag "${TAG}"
+          git push origin "${TAG}"

.github/workflows/release-pypi.yml

@@ -99,9 +99,12 @@ jobs:
 
       - name: Install dependencies
         run: |
-          apt-get update && apt-get install -y cmake libhwloc-dev pkg-config libnuma-dev
+          # System packages (cmake/libhwloc-dev/pkg-config/libnuma-dev) are expected to be
+          # preinstalled on self-hosted runners. Skip apt-get to avoid sudo dependency.
+          for pkg in cmake pkg-config; do command -v $pkg >/dev/null || { echo "ERROR: $pkg missing on runner"; exit 1; }; done
           python -m pip install --upgrade pip
-          pip install build wheel setuptools torch --index-url https://download.pytorch.org/whl/cu118
+          pip install build wheel setuptools
+          pip install torch --index-url https://download.pytorch.org/whl/cu128
 
       - name: Build kt-kernel wheel
         working-directory: kt-kernel
@@ -109,15 +112,15 @@ jobs:
           CPUINFER_BUILD_ALL_VARIANTS: '1'
           CPUINFER_ENABLE_CPPTRACE: '0'
           CPUINFER_USE_CUDA: '1'
-          CPUINFER_CUDA_ARCHS: '80;86;89;90'
+          CPUINFER_CUDA_ARCHS: '80;86;89;90;120'
           CPUINFER_CUDA_STATIC_RUNTIME: '1'
           CPUINFER_BUILD_TYPE: 'Release'
           CPUINFER_PARALLEL: '4'
           CPUINFER_FORCE_REBUILD: '1'
-          CUDA_HOME: '/usr/local/cuda-11.8'
+          CUDA_HOME: '/usr/local/cuda-12.8'
         run: |
           echo "Building kt-kernel with:"
-          echo "  - CUDA support (SM 80, 86, 89, 90)"
+          echo "  - CUDA support (SM 80, 86, 89, 90, 120)"
           echo "  - CPU multi-variant (AMX, AVX512, AVX2)"
           python -m build --wheel -v
 
@@ -148,10 +151,13 @@ jobs:
           python -m zipfile -l dist/*.whl | grep "_kt_kernel_ext_avx512" && echo "✓ AVX512 variants found" || echo "Note: AVX512 variants missing"
           python -m zipfile -l dist/*.whl | grep "_kt_kernel_ext_avx2.cpython" && echo "✓ AVX2 variant found" || echo "Note: AVX2 variant missing"
 
-          # Verify static linking (should NOT depend on libcudart.so)
-          rm -rf /tmp/check
-          unzip -q dist/*.whl -d /tmp/check
-          if ldd /tmp/check/kt_kernel/*.so 2>/dev/null | grep -q "libcudart.so"; then
+          # Verify static linking (should NOT depend on libcudart.so).
+          # Use $RUNNER_TEMP (honors TMPDIR redirect to /mnt) — /tmp is the
+          # system disk on self-hosted runners and can be tight.
+          CHECK_DIR="${RUNNER_TEMP:-/tmp}/check"
+          rm -rf "$CHECK_DIR"
+          unzip -q dist/*.whl -d "$CHECK_DIR"
+          if ldd "$CHECK_DIR"/kt_kernel/*.so 2>/dev/null | grep -q "libcudart.so"; then
             echo "ERROR: Dynamic cudart found, should be statically linked"
             exit 1
           else
@@ -164,8 +170,7 @@ jobs:
           pip install auditwheel patchelf
           mkdir -p wheelhouse
           for wheel in dist/*.whl; do
-            auditwheel repair "$wheel" --plat manylinux_2_17_x86_64 --exclude libcuda.so.1 -w wheelhouse/ || \
-              cp "$wheel" wheelhouse/$(basename "$wheel" | sed 's/linux_x86_64/manylinux_2_17_x86_64/')
+            auditwheel repair "$wheel" --plat manylinux_2_35_x86_64 --exclude libcuda.so.1 -w wheelhouse/
           done
           rm -f dist/*.whl && cp wheelhouse/*.whl dist/
 
@@ -192,6 +197,11 @@ jobs:
         with:
           path: artifacts/
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
       - name: Organize wheels into dist/
         run: |
           mkdir -p dist/

.github/workflows/release-sglang-kt.yml

@@ -6,7 +6,6 @@ on:
       - main
     paths:
       - "third_party/sglang"
-      - "version.py"
   workflow_dispatch:
     inputs:
       test_pypi:

MANIFEST.in

@@ -0,0 +1 @@
+include version.py

README.md

@@ -16,6 +16,7 @@
 KTransformers is a research project focused on efficient inference and fine-tuning of large language models through CPU-GPU heterogeneous computing. The project now exposes two user-facing capabilities from the kt-kernel source tree: [Inference](./kt-kernel/README.md) and [SFT](./doc/en/SFT/KTransformers-Fine-Tuning_Quick-Start.md).
 
 ## 🔥 Updates
+* **June 21, 2026**: MiniMax-M3 Day0 Support! ([Tutorial](./doc/en/kt-kernel/MiniMax-M3-Tutorial.md))
 * **June 17, 2026**: GLM-5.2 Day0 Support! ([Tutorial](./doc/en/kt-kernel/GLM-5.2-Tutorial.md))
 * **May 6, 2026**: KTransformers at [GOSIM Paris 2026](https://paris2026.gosim.org/zh/schedule/) — "Agentic AI on Edge" track. We'll present KT's inference performance on consumer hardware.
 * **May 02, 2026**: DeepSeek-V4-Flash Support! ([Tutorial](./doc/en/DeepSeek-V4-Flash.md))

doc/en/DeepSeek-V4-Flash.md

@@ -25,7 +25,7 @@ This tutorial demonstrates how to run **DeepSeek-V4-Flash** model inference usin
 - **RAM**: ≥256GB system memory
 - **Storage**: ~340GB for model weights
 
-**Supported GPU architectures** (auto-detected at startup; non-validated configurations should work but have not been benchmarked end-to-end):
+**architectures** (auto-detected at startup; non-validated configurations should work but have not been benchmarked end-to-end):
 
 | Arch | Compute Cap | MXFP4 MoE | NSA sparse MLA | Validated |
 |------|------------|-----------|----------------|-----------|
@@ -33,7 +33,7 @@ This tutorial demonstrates how to run **DeepSeek-V4-Flash** model inference usin
 | Datacenter Blackwell (B100 / B200) | SM_100 | trtllm-fp4 | Triton fallback | — |
 | Consumer Blackwell (RTX 5090) | SM_120 | triton_kernels | Triton fallback | ✓ |
 | Ada Lovelace (RTX 4090 / L20 / L40) | SM_89 | triton_kernels | Triton fallback | ✓ |
-| Ampere (A100 / A6000) | SM_80 / SM_86 | triton_kernels | Triton fallback | ✗ (not supported) |
+| Ampere (A100 / A6000) | SM_80 / SM_86 | triton_kernels | Triton fallback | Now supported |
 
 
 ## Prerequisites

doc/en/FAQ.md

@@ -1,2 +1,17 @@
-<!-- omit in toc -->
-# see the issue [FAQ page](https://github.com/kvcache-ai/ktransformers/issues/1608)
+# Frequently Asked Questions
+
+## 1. SGLang "Using default MoE kernel config" warning at startup
+
+When using kt-kernel with SGLang, you may see a warning like:
+
+```
+[2026-05-15 20:31:38] Using default MoE kernel config. Performance might be sub-optimal!
+Config file not found at .../fused_moe_triton/configs/...
+```
+
+This warning is **expected and can be safely ignored**. kt-kernel replaces SGLang's built-in MoE implementation with its own CPU/GPU hybrid dispatch, so SGLang's fused-MoE Triton kernel configuration is never used. The warning is emitted by SGLang before kt-kernel takes over MoE execution and has no impact on performance or correctness.
+
+## 2. Where can I find more help?
+
+Check the [existing issues](https://github.com/kvcache-ai/ktransformers/issues) or open a [new one](https://github.com/kvcache-ai/ktransformers/issues/new).
+

doc/en/ROCm.md

@@ -42,6 +42,44 @@ pip3 install packaging ninja cpufeature numpy
 
 > **Tip:** For other ROCm versions, visit [PyTorch Previous Versions](https://pytorch.org/get-started/previous-versions/)
 
+### Hygon DCU / DTK Notes
+
+Hygon DCU uses a ROCm-compatible DTK stack. For DCU systems, use the Hygon DCU
+PyTorch environment that matches your DTK release instead of installing the
+generic PyPI `torch` package or the official AMD ROCm wheel.
+
+For example, a reported working `gfx936` setup uses a Hygon DCU PyTorch image
+from the SourceFind/Hygon developer image portal:
+
+https://sourcefind.cn/#/image/dcu/pytorch
+
+The reported environment provides:
+
+- DTK 26.04, typically under `/opt/dtk`
+- PyTorch `2.5.1+das.opt1.dtk2604`
+- Python 3.10
+
+Before building, verify that the DCU PyTorch package is active:
+
+```bash
+python -c "import torch; print(torch.__version__); print(torch.version.hip); print(torch.__file__)"
+```
+
+Then build `kt-kernel` without letting pip replace the vendor PyTorch package:
+
+```bash
+export CPUINFER_USE_ROCM=1
+export PYTORCH_ROCM_ARCH=gfx936
+export ROCM_PATH=/opt/dtk  # change this if DTK is installed elsewhere
+
+cd kt-kernel
+pip install . --no-build-isolation --no-deps
+```
+
+> **Tip:** Keep `--no-deps` when building in a vendor PyTorch environment. A
+> plain `pip install .` may resolve `kt-kernel`'s normal `torch` dependency and
+> shadow or replace the installed DCU PyTorch package with a generic torch wheel.
+
 ### 4. Build ktransformers
 
 ```bash

doc/en/SFT/Qwen3.5-SGLang-LoRA-Serving.md

@@ -1,6 +1,6 @@
-# KT-FT Fine-Tuning and Inference Loop
+# Qwen3.5 MoE KT LoRA Serving with SGLang-KT
 
-Last updated: 2026-06-01
+Last updated: 2026-06-23
 
 This guide documents the current KT-FT loop for Qwen3.5 MoE: train with KT SFT, convert the output once, and serve the fine-tuned result through SGLang with a single merged adapter path.
 
@@ -20,9 +20,11 @@ Training-side KT SFT docs remain separate. This page focuses on the bridge from
 Current supported and validated workflow:
 
 - Base model: Qwen3.5 MoE, for example `Qwen3.5-35B-A3B`
+- KTransformers version: v0.6.3 or newer
 - KT expert weights: AMX/BF16 SFT-compatible KT CPU expert path
 - User-facing serving input: one converted merged adapter directory
 - Runtime split: expert LoRA goes to the KT CPU expert path; non-expert LoRA goes to SGLang's LoRA manager. This split happens automatically at server startup.
+- This workflow is for KT MoE expert LoRA artifacts. Standard dense-model PEFT LoRA adapters usually do not need this converter.
 
 ## 2. Artifacts At Each Stage
 
@@ -65,14 +67,16 @@ Example:
 
 ```bash
 python kt-kernel/scripts/convert_kt_to_sglang_adapter.py \
-  saves/KT_FT_qwen35B_Moe_nekoqa_eod_240 \
-  saves/KT_FT_qwen35B_Moe_nekoqa_eod_240_sglang \
+  saves/KT_FT_qwen35B_Moe_custom \
+  saves/KT_FT_qwen35B_Moe_custom_sglang \
   --base-model-name-or-path /mnt/data3/models/Qwen3.5-35B-A3B \
   --overwrite
 ```
 
 The converter reads `fused_expert_lora.safetensors` and the existing non-expert `adapter_model.safetensors`, then writes one merged adapter directory.
 
+If the raw KT SFT output does not contain an `adapter_config.json` with `lora_alpha`, pass `--lora-alpha <value>` explicitly. The converter does not fold LoRA scaling into the tensors; runtime scaling remains `lora_alpha / r`.
+
 Optional split outputs for debugging:
 
 ```bash
@@ -120,7 +124,7 @@ python -m sglang.launch_server \
   --disable-custom-all-reduce \
   --enable-lora \
   --lora-backend triton \
-  --lora-paths qwen35b_neko=/path/to/KT_FT_qwen35B_Moe_nekoqa_eod_240_sglang \
+  --lora-paths qwen35b_lora=/path/to/KT_FT_qwen35B_Moe_custom_sglang \
   --log-level info
 ```
 
@@ -137,6 +141,7 @@ Current constraints:
 - `--kt-num-gpu-experts 0`
 - do not enable `--kt-enable-dynamic-expert-update`
 - do not use `--kt-gpu-prefill-token-threshold`
+- `--max-running-requests` must be at least 2
 - use an AMX/BF16 SFT-compatible KT method such as `AMXINT4`, `AMXINT8`, `AMXBF16`, or `BF16`
 
 ## 5. Request Semantics
@@ -145,7 +150,7 @@ The OpenAI-compatible request `model` field uses names, not paths.
 
 ```text
 --served-model-name qwen3.5-kt-ft
---lora-paths qwen35b_neko=/path/to/merged_adapter
+--lora-paths qwen35b_lora=/path/to/merged_adapter
 ```
 
 Request behavior in the current single-adapter implementation:
@@ -154,20 +159,22 @@ Request behavior in the current single-adapter implementation:
 model=qwen3.5-kt-ft
 => base + KT expert LoRA
 
-model=qwen3.5-kt-ft:qwen35b_neko
+model=qwen3.5-kt-ft:qwen35b_lora
 => base + KT expert LoRA + SGLang non-expert LoRA
 ```
 
 The suffix after `:` must match the left-side name in `--lora-paths`.
 
+If you need a true base-only comparison, launch a separate server without `--lora-paths`.
+
 ## 6. Smoke Test
 
 ```bash
 curl -sS http://127.0.0.1:30006/v1/chat/completions \
   -H 'Content-Type: application/json' \
   -d '{
-    "model": "qwen3.5-kt-ft:qwen35b_neko",
-    "messages": [{"role": "user", "content": "我回来了，你在干嘛？"}],
+    "model": "qwen3.5-kt-ft:qwen35b_lora",
+    "messages": [{"role": "user", "content": "Explain what LoRA is in one sentence."}],
     "temperature": 0.7,
     "max_tokens": 160,
     "chat_template_kwargs": {"enable_thinking": false}
@@ -198,11 +205,11 @@ This is not the recommended user-facing path. Normal users should pass one merge
 
 ### `Got LoRA adapter that has never been loaded: lora0`
 
-The adapter name in the request must match the left side of `--lora-paths`. If you launched with `qwen35b_neko=...`, request `model=qwen3.5-kt-ft:qwen35b_neko`, not `:lora0`.
+The adapter name in the request must match the left side of `--lora-paths`. If you launched with `qwen35b_lora=...`, request `model=qwen3.5-kt-ft:qwen35b_lora`, not `:lora0`.
 
 ### No visible adapter effect
 
-Make sure you are serving the intended merged adapter directory. For example, use the Neko adapter at `..._nekoqa_eod_240_sglang`, not a generic sanity adapter such as `..._Moe_sglang`.
+Make sure you are serving the converted merged adapter directory produced by the converter, not the raw KT SFT output directory or a different test adapter.
 
 ### `connection refused`
 

doc/zh/Qwen3.5-SGLang-LoRA-Serving_zh.md

@@ -1,6 +1,6 @@
-# KT-FT 微调推理闭环
+# Qwen3.5 MoE KT LoRA 的 SGLang-KT Serving
 
-最后更新：2026-06-01
+最后更新：2026-06-23
 
 本文档描述当前 Qwen3.5 MoE 的 KT-FT 闭环：用 KT SFT 完成微调，转换一次输出，再通过 SGLang 用单个 merged adapter path 把微调结果服务化。
 
@@ -20,9 +20,11 @@ KT SFT 原始输出
 当前已验证路径：
 
 - 基座模型：Qwen3.5 MoE，例如 `Qwen3.5-35B-A3B`
+- KTransformers 版本：v0.6.3 或更新版本
 - KT expert 权重：AMX/BF16 SFT 兼容的 KT CPU expert 路径
 - 用户侧 serving 输入：一个 converted merged adapter 目录
 - Runtime 内部仍会 split：expert LoRA 走 KT CPU expert path，non-expert LoRA 走 SGLang LoRA manager，但这一步对用户不可见
+- 该工作流面向 KT MoE expert LoRA 产物；普通 dense 模型的标准 PEFT LoRA 通常不需要使用此转换器（converter）。
 
 ## 2. 各阶段产物
 
@@ -65,14 +67,16 @@ python kt-kernel/scripts/convert_kt_to_sglang_adapter.py \
 
 ```bash
 python kt-kernel/scripts/convert_kt_to_sglang_adapter.py \
-  saves/KT_FT_qwen35B_Moe_nekoqa_eod_240 \
-  saves/KT_FT_qwen35B_Moe_nekoqa_eod_240_sglang \
+  saves/KT_FT_qwen35B_Moe_custom \
+  saves/KT_FT_qwen35B_Moe_custom_sglang \
   --base-model-name-or-path /mnt/data3/models/Qwen3.5-35B-A3B \
   --overwrite
 ```
 
 converter 会读取 `fused_expert_lora.safetensors` 和已有的 non-expert `adapter_model.safetensors`，写出一个 merged adapter 目录。
 
+如果原始 KT SFT 输出目录没有包含带 `lora_alpha` 的 `adapter_config.json`，需要显式传入 `--lora-alpha <value>`。converter 不会把 LoRA scaling 折进 tensor；运行时 scaling 仍然是 `lora_alpha / r`。
+
 如需调试，也可以额外输出 split 目录：
 
 ```bash
@@ -120,7 +124,7 @@ python -m sglang.launch_server \
   --disable-custom-all-reduce \
   --enable-lora \
   --lora-backend triton \
-  --lora-paths qwen35b_neko=/path/to/KT_FT_qwen35B_Moe_nekoqa_eod_240_sglang \
+  --lora-paths qwen35b_lora=/path/to/KT_FT_qwen35B_Moe_custom_sglang \
   --log-level info
 ```
 
@@ -137,6 +141,7 @@ python -m sglang.launch_server \
 - `--kt-num-gpu-experts 0`
 - 不启用 `--kt-enable-dynamic-expert-update`
 - 不使用 `--kt-gpu-prefill-token-threshold`
+- `--max-running-requests` 必须至少为 2
 - 使用 AMX/BF16 SFT 兼容 KT method，例如 `AMXINT4`、`AMXINT8`、`AMXBF16`、`BF16`
 
 ## 5. 请求语义
@@ -145,7 +150,7 @@ OpenAI-compatible 请求里的 `model` 字段用 name，不用 path。
 
 ```text
 --served-model-name qwen3.5-kt-ft
---lora-paths qwen35b_neko=/path/to/merged_adapter
+--lora-paths qwen35b_lora=/path/to/merged_adapter
 ```
 
 当前 single-adapter 实现的请求语义：
@@ -154,20 +159,22 @@ OpenAI-compatible 请求里的 `model` 字段用 name，不用 path。
 model=qwen3.5-kt-ft
 => base + KT expert LoRA
 
-model=qwen3.5-kt-ft:qwen35b_neko
+model=qwen3.5-kt-ft:qwen35b_lora
 => base + KT expert LoRA + SGLang non-expert LoRA
 ```
 
 冒号后的 adapter 名必须和 `--lora-paths` 左侧注册名一致。
 
+如果需要 true base-only 对照，请单独启动一个不带 `--lora-paths` 的服务。
+
 ## 6. Smoke Test
 
 ```bash
 curl -sS http://127.0.0.1:30006/v1/chat/completions \
   -H 'Content-Type: application/json' \
   -d '{
-    "model": "qwen3.5-kt-ft:qwen35b_neko",
-    "messages": [{"role": "user", "content": "我回来了，你在干嘛？"}],
+    "model": "qwen3.5-kt-ft:qwen35b_lora",
+    "messages": [{"role": "user", "content": "用一句话解释什么是 LoRA。"}],
     "temperature": 0.7,
     "max_tokens": 160,
     "chat_template_kwargs": {"enable_thinking": false}
@@ -198,11 +205,11 @@ Using triton as backend of LoRA kernels.
 
 ### `Got LoRA adapter that has never been loaded: lora0`
 
-请求里的 adapter 名必须和 `--lora-paths` 左侧一致。如果启动时写的是 `qwen35b_neko=...`，请求应使用 `model=qwen3.5-kt-ft:qwen35b_neko`，而不是 `:lora0`。
+请求里的 adapter 名必须和 `--lora-paths` 左侧一致。如果启动时写的是 `qwen35b_lora=...`，请求应使用 `model=qwen3.5-kt-ft:qwen35b_lora`，而不是 `:lora0`。
 
 ### 看不出 adapter 效果
 
-确认 serving 用的是目标 merged adapter。例如 Neko 风格应使用 `..._nekoqa_eod_240_sglang`，而不是通用 sanity adapter `..._Moe_sglang`。
+确认 serving 使用的是 converter 生成的 merged adapter 目录，而不是原始 KT SFT 输出目录或其他测试 adapter。
 
 ### `connection refused`
 

kt-kernel/CMakeLists.txt

@@ -273,7 +273,7 @@ elseif(CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR CMAKE_GENERATOR_PLATFORM_LWR
             list(APPEND ARCH_FLAGS -mavx2 -mfma -msse3 -mf16c)
         endif()
         if(LLAMA_AVX512)
-            list(APPEND ARCH_FLAGS -mavx512f -mavx512bw -mavx512dq -mfma -mf16c -msse3)
+            list(APPEND ARCH_FLAGS -mavx512f -mavx512bw -mavx512dq -mavx512vl -mfma -mf16c -msse3)
         endif()
         if(LLAMA_AVX512_VBMI)
             list(APPEND ARCH_FLAGS -mavx512vbmi)

kt-kernel/python/cli/utils/port_checker.py

@@ -3,6 +3,7 @@ Port availability checking utilities.
 """
 
 import socket
+import sys
 from typing import Tuple
 
 
@@ -17,22 +18,14 @@ def is_port_available(host: str, port: int) -> bool:
         True if port is available, False if occupied
     """
     try:
-        # Try to bind to the port
-        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        sock.settimeout(1)
+        bind_host = "" if host == "0.0.0.0" else host
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+            if sys.platform != "win32":
+                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            sock.bind((bind_host, port))
+        return True
 
-        # Use SO_REUSEADDR to allow binding to recently closed ports
-        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-
-        # Try to bind
-        result = sock.connect_ex((host if host != "0.0.0.0" else "127.0.0.1", port))
-        sock.close()
-
-        # If connect_ex returns 0, port is occupied
-        # If it returns error (non-zero), port is available
-        return result != 0
-
-    except Exception:
+    except OSError:
         # If any error occurs, assume port is not available
         return False
 

kt-kernel/python/utils/loader.py

@@ -679,6 +679,49 @@ class BF16SafeTensorLoader(SafeTensorLoader):
 class CompressedSafeTensorLoader(SafeTensorLoader):
     """Loader for compressed SafeTensor layouts (RAWINT4 weights)."""
 
+    @staticmethod
+    def _normalize_rawint4_weight(weight_tensor, scale_tensor, shape_tensor=None, key: str = "weight_packed"):
+        """Return byte-packed uint8 RAWINT4 weights expected by kt_kernel_ext."""
+        if weight_tensor.dtype == torch.int32:
+            # compressed-tensors pack-quantized stores 8 int4 values per int32.
+            # The RAWINT4 kernels consume the same bytes as uint8, two int4 values per byte.
+            rows, int32_cols = weight_tensor.shape
+            weight_tensor = weight_tensor.contiguous().view(torch.uint8).view(rows, int32_cols * 4).contiguous()
+        elif weight_tensor.dtype == torch.uint8:
+            weight_tensor = weight_tensor.contiguous()
+        else:
+            raise TypeError(f"{key} must be torch.uint8 or torch.int32, got {weight_tensor.dtype}")
+
+        if shape_tensor is None:
+            return weight_tensor
+
+        shape_values = shape_tensor.detach().cpu().tolist()
+        if len(shape_values) != 2:
+            raise ValueError(f"{key}.weight_shape must contain [out_features, in_features], got {shape_values}")
+
+        out_features, in_features = (int(shape_values[0]), int(shape_values[1]))
+        if out_features <= 0 or in_features <= 0:
+            return weight_tensor
+
+        if in_features % 2 != 0:
+            return weight_tensor
+
+        expected_weight_shape = (out_features, in_features // 2)
+        if tuple(weight_tensor.shape) != expected_weight_shape:
+            return weight_tensor
+
+        if scale_tensor.dim() != 2 or scale_tensor.shape[0] != out_features or scale_tensor.shape[1] <= 0:
+            raise ValueError(
+                f"{key} scale shape {tuple(scale_tensor.shape)} is incompatible with weight_shape={shape_values}"
+            )
+
+        if in_features % int(scale_tensor.shape[1]) != 0:
+            raise ValueError(
+                f"{key} in_features={in_features} is not divisible by scale columns={scale_tensor.shape[1]}"
+            )
+
+        return weight_tensor
+
     def load_experts(self, base_key: str, device: str = "cpu"):
         """Load raw expert weights stored in compressed safetensor format."""
 
@@ -703,6 +746,7 @@ class CompressedSafeTensorLoader(SafeTensorLoader):
             for exp_id in range(expert_idx):
                 weight_key = f"{experts_prefix}.{exp_id}.{proj_name}_proj.weight_packed"
                 scale_key = f"{experts_prefix}.{exp_id}.{proj_name}_proj.weight_scale"
+                shape_key = f"{experts_prefix}.{exp_id}.{proj_name}_proj.weight_shape"
 
                 if not self.has_tensor(weight_key):
                     raise KeyError(f"Missing tensor: {weight_key}")
@@ -711,6 +755,8 @@ class CompressedSafeTensorLoader(SafeTensorLoader):
 
                 weight_tensor = self.load_tensor(weight_key, device).contiguous()
                 scale_tensor = self.load_tensor(scale_key, device).contiguous()
+                shape_tensor = self.load_tensor(shape_key, "cpu") if self.has_tensor(shape_key) else None
+                weight_tensor = self._normalize_rawint4_weight(weight_tensor, scale_tensor, shape_tensor, weight_key)
 
                 weight_entries.append(weight_tensor)
                 scale_entries.append(scale_tensor)

kt-kernel/test/per_commit/test_moe_rawint4_accuracy.py

@@ -110,6 +110,13 @@ def rawint4_dequantize(qweight, scales, out_features, in_features):
     return result
 
 
+def pack_rawint4_uint8_as_int32(qweight):
+    """Pack byte RAWINT4 layout into compressed-tensors int32 storage."""
+    assert qweight.dtype == torch.uint8
+    assert qweight.shape[1] % 4 == 0
+    return qweight.contiguous().view(torch.int32).contiguous()
+
+
 def act_fn(x):
     return x / (1.0 + torch.exp(-x))
 
@@ -279,6 +286,77 @@ def test_rawint4_accuracy():
         run_backend_accuracy_test(backend_name, backend_cls, threshold, qlen=16)
 
 
+def test_compressed_loader_normalizes_int32_pack_quantized_weights():
+    load_amx_utils()
+    loader_mod = sys.modules["kt_kernel.utils.loader"]
+
+    weight_bf16 = (torch.randn((intermediate_size, hidden_size), dtype=torch.float32) / 10.0).to(torch.bfloat16)
+    qweight, scales = rawint4_quantize(weight_bf16)
+    packed_int32 = pack_rawint4_uint8_as_int32(qweight)
+    weight_shape = torch.tensor([intermediate_size, hidden_size], dtype=torch.int32)
+
+    normalized = loader_mod.CompressedSafeTensorLoader._normalize_rawint4_weight(
+        packed_int32, scales, weight_shape, "test.weight_packed"
+    )
+
+    assert normalized.dtype == torch.uint8
+    assert normalized.shape == qweight.shape
+    assert torch.equal(normalized, qweight)
+
+
+def test_compressed_loader_accepts_uint8_rawint4_weights():
+    load_amx_utils()
+    loader_mod = sys.modules["kt_kernel.utils.loader"]
+
+    weight_bf16 = (torch.randn((intermediate_size, hidden_size), dtype=torch.float32) / 10.0).to(torch.bfloat16)
+    qweight, scales = rawint4_quantize(weight_bf16)
+    weight_shape = torch.tensor([intermediate_size, hidden_size], dtype=torch.int32)
+
+    normalized = loader_mod.CompressedSafeTensorLoader._normalize_rawint4_weight(
+        qweight, scales, weight_shape, "test.weight_packed"
+    )
+
+    assert normalized.dtype == torch.uint8
+    assert normalized.shape == qweight.shape
+    assert torch.equal(normalized, qweight)
+
+
+def test_compressed_loader_ignores_invalid_weight_shape_metadata():
+    load_amx_utils()
+    loader_mod = sys.modules["kt_kernel.utils.loader"]
+
+    weight_bf16 = (torch.randn((intermediate_size, hidden_size), dtype=torch.float32) / 10.0).to(torch.bfloat16)
+    qweight, scales = rawint4_quantize(weight_bf16)
+    packed_int32 = pack_rawint4_uint8_as_int32(qweight)
+    invalid_shape = torch.tensor([-1752796263, -1707567530], dtype=torch.int32)
+
+    normalized = loader_mod.CompressedSafeTensorLoader._normalize_rawint4_weight(
+        packed_int32, scales, invalid_shape, "test.weight_packed"
+    )
+
+    assert normalized.dtype == torch.uint8
+    assert normalized.shape == qweight.shape
+    assert torch.equal(normalized, qweight)
+
+
+def test_compressed_loader_ignores_odd_weight_shape_metadata():
+    load_amx_utils()
+    loader_mod = sys.modules["kt_kernel.utils.loader"]
+
+    weight_bf16 = (torch.randn((intermediate_size, hidden_size), dtype=torch.float32) / 10.0).to(torch.bfloat16)
+    qweight, scales = rawint4_quantize(weight_bf16)
+    packed_int32 = pack_rawint4_uint8_as_int32(qweight)
+    invalid_shape = torch.tensor([241597647, 1216029047], dtype=torch.int32)
+
+    normalized = loader_mod.CompressedSafeTensorLoader._normalize_rawint4_weight(
+        packed_int32, scales, invalid_shape, "test.weight_packed"
+    )
+
+    assert normalized.dtype == torch.uint8
+    assert normalized.shape == qweight.shape
+    assert torch.equal(normalized, qweight)
+
+
 def test_rawint4_backend_selection_falls_back_to_avx2_for_large_group_size(monkeypatch):
     amx_utils = load_amx_utils()
     fake_amx_backend = object()

kt-kernel/test/per_commit/test_port_checker.py

@@ -0,0 +1,45 @@
+import importlib.util
+import socket
+from pathlib import Path
+import unittest
+from unittest.mock import MagicMock, patch
+
+from ci.ci_register import register_cpu_ci
+
+
+register_cpu_ci(est_time=0.1, suite="default")
+
+
+PORT_CHECKER_PATH = Path(__file__).resolve().parents[2] / "python" / "cli" / "utils" / "port_checker.py"
+SPEC = importlib.util.spec_from_file_location("port_checker", PORT_CHECKER_PATH)
+assert SPEC is not None and SPEC.loader is not None
+port_checker = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(port_checker)
+
+
+class TestPortChecker(unittest.TestCase):
+    def test_bound_port_is_not_available_before_listen(self):
+        holder = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            holder.bind(("127.0.0.1", 0))
+            port = holder.getsockname()[1]
+
+            self.assertFalse(port_checker.is_port_available("127.0.0.1", port))
+            self.assertEqual(port_checker.find_available_port("127.0.0.1", port, max_attempts=1), (False, port))
+        finally:
+            holder.close()
+
+    def test_non_windows_bind_check_uses_reuseaddr(self):
+        sock = MagicMock()
+        sock.__enter__.return_value = sock
+
+        with patch.object(port_checker.sys, "platform", "linux"):
+            with patch.object(port_checker.socket, "socket", return_value=sock):
+                self.assertTrue(port_checker.is_port_available("127.0.0.1", 12345))
+
+        sock.setsockopt.assert_called_once_with(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        sock.bind.assert_called_once_with(("127.0.0.1", 12345))
+
+
+if __name__ == "__main__":
+    unittest.main()

setup.py

@@ -23,7 +23,7 @@ setup(
             "accelerate-kt==1.14.0.post1",
         ],
         "sglang": [
-            "sglang-kt==0.6.2.post3",
+            f"sglang-kt=={_v}",
         ],
     },
 )

third_party/sglang

@@ -1 +1 @@
-Subproject commit 8b636f9008dbad58c0a8e481b03e794739e6c146
+Subproject commit 5d6bef9f61637aaeaf047bf8209def2af3eaa83f

version.py

@@ -3,4 +3,4 @@ KTransformers version information.
 Shared across the top-level package and kt-kernel.
 """
 
-__version__ = "0.6.2.post3"
+__version__ = "0.6.3.post1"