Optimize & export: all-MiniLM-L6-v2

python -m pip install "transformers>=4.40" "optimum[onnxruntime]>=1.19" "onnx>=1.16" "onnxruntime>=1.17" "safetensors>=0.4"

python -m pip install "transformers>=4.40" "accelerate>=0.30" "bitsandbytes>=0.43" "safetensors>=0.4"

from optimum.onnxruntime import ORTModelForFeatureExtraction
from transformers import AutoTokenizer

repo = "sentence-transformers/all-MiniLM-L6-v2"
revision = "ea78891063587eb050ed4166b20062eaf978037c"

tokenizer = AutoTokenizer.from_pretrained(
    repo, revision=revision, trust_remote_code=False
)
model = ORTModelForFeatureExtraction.from_pretrained(
    repo,
    revision=revision,
    export=True,
    trust_remote_code=False,
)
model.save_pretrained("./financegpt-onnx")
tokenizer.save_pretrained("./financegpt-onnx")

# Record the FinanceGPT source artifact SHA-256 alongside this derived export.

from pathlib import Path
from optimum.onnxruntime import ORTQuantizer
from optimum.onnxruntime.configuration import AutoQuantizationConfig

model_dir = Path("./financegpt-onnx")
quantizer = ORTQuantizer.from_pretrained(model_dir)
qconfig = AutoQuantizationConfig.avx2(
    is_static=False,
    per_channel=False,
)
quantizer.quantize(
    save_dir="./financegpt-onnx-int8",
    quantization_config=qconfig,
)

# Benchmark and evaluate the quantized artifact before any publication request.

from transformers import AutoModel, AutoTokenizer, BitsAndBytesConfig

repo = "sentence-transformers/all-MiniLM-L6-v2"
revision = "ea78891063587eb050ed4166b20062eaf978037c"
quantization = BitsAndBytesConfig(load_in_8bit=True)

tokenizer = AutoTokenizer.from_pretrained(
    repo, revision=revision, trust_remote_code=False
)
model = AutoModel.from_pretrained(
    repo,
    revision=revision,
    quantization_config=quantization,
    device_map="auto",
    trust_remote_code=False,
)

# Load-time quantization is hardware dependent; benchmark on your target runtime.

from transformers import AutoModel, BitsAndBytesConfig
import torch

repo = "sentence-transformers/all-MiniLM-L6-v2"
revision = "ea78891063587eb050ed4166b20062eaf978037c"
quantization = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
)
model = AutoModel.from_pretrained(
    repo,
    revision=revision,
    quantization_config=quantization,
    device_map="auto",
    trust_remote_code=False,
)

# NF4 is an optional developer-side runtime recipe, not a universal compatibility claim.

{
    "source_repository": "sentence-transformers/all-MiniLM-L6-v2",
    "source_revision": "ea78891063587eb050ed4166b20062eaf978037c",
    "source_artifact_sha256": "997404dcf8e7ea89cdf598ebab9594ec77c94a08f80861e4beece2c0d06d0b6d",
    "derived_format": "onnx|onnx-dynamic-int8|runtime-quantized",
    "derived_artifact_sha256": "<compute after export>",
    "evaluation_reference": "<required before publication>",
    "publication_is_promotion": false
}

1. Keep the exact source repository and immutable revision.
2. Retain the source artifact SHA-256 when FinanceGPT has one.
3. Compute SHA-256 for every derived export or quantized artifact.
4. Benchmark on the actual target hardware; quantization is hardware sensitive.
5. Run model evaluation after conversion or quantization and before publication.
6. Publish only through an explicit governed publication action.
7. Never treat publication as production promotion or QLM rebinding.