AI多模态大模型：文本不够用了之后

from transformers import CLIPProcessor, CLIPModel, AutoModelForCausalLM, AutoTokenizer

# 加载视觉模型
clip_model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
clip_processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")

# 加载语言模型
llm = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf")
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")

def simple_multimodal_inference(image, text_prompt):
    # 提取图像特征
    image_inputs = clip_processor(images=image, return_tensors="pt")
    image_features = clip_model.get_image_features(**image_inputs)

    # 编码文本
    text_inputs = tokenizer(text_prompt, return_tensors="pt")

    # 简单拼接
    combined_features = torch.cat([
        image_features.squeeze(0),
        text_inputs['input_ids'].squeeze(0).float()
    ], dim=0)

    # 这里就是问题所在：特征维度根本不匹配，强行拼接会导致问题
    # 实际上这种简单拼接的方法效果很差
    pass

from transformers import LlavaProcessor, LlavaForConditionalGeneration
from PIL import Image

# 加载模型和处理器
model = LlavaForConditionalGeneration.from_pretrained(
    "llava-hf/llava-1.5-7b-hf"
)
processor = LlavaProcessor.from_pretrained(
    "llava-hf/llava-1.5-7b-hf"
)

def inference_with_llava(image_path, prompt):
    image = Image.open(image_path)

    # 构建输入
    inputs = processor(
        text=prompt,
        images=image,
        return_tensors="pt"
    )

    # 生成回答
    outputs = model.generate(
        **inputs,
        max_new_tokens=500,
        do_sample=False
    )

    # 解码结果
    answer = processor.decode(
        outputs[0],
        skip_special_tokens=True
    )

    return answer

from transformers import BitsAndBytesConfig

# 4-bit 量化配置
quantization_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.float16,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4"
)

model = LlavaForConditionalGeneration.from_pretrained(
    "llava-hf/llava-1.5-7b-hf",
    quantization_config=quantization_config,
    device_map="auto"
)

# 使用 vLLM 加速
from vllm import LLM, SamplingParams

llm = LLM(
    model="llava-hf/llava-1.5-7b-hf",
    quantization="awq",
    tensor_parallel_size=1
)

sampling_params = SamplingParams(
    temperature=0.0,
    top_p=1.0,
    max_tokens=500
)

# 推理
outputs = llm.generate([prompt], sampling_params)

def adaptive_image_resize(image, max_pixels=512*512):
    """根据图像大小和内容复杂度动态调整分辨率"""
    width, height = image.size
    current_pixels = width * height

    # 如果当前像素数超过阈值，等比例缩放
    if current_pixels > max_pixels:
        scale_factor = (max_pixels / current_pixels) ** 0.5
        new_width = int(width * scale_factor)
        new_height = int(height * scale_factor)
        image = image.resize((new_width, new_height), Image.Resampling.LANCZOS)

    return image

def robust_inference(image, question, confidence_threshold=0.6):
    """带置信度的推理"""

    # 先做图像分类，获得置信度
    classification = image_classifier(image)
    confidence = classification['confidence']

    # 如果置信度太低，拒绝回答
    if confidence < confidence_threshold:
        return "抱歉，这张图片不够清晰，我无法准确识别。请提供更清晰的照片。"

    # 正常推理
    answer = llm.generate(image, question)

    # 后处理，添加置信度说明
    if "不确定" in answer or "可能" in answer:
        answer += "\n\n（注：基于当前图片的分析，建议您提供更多细节以获得更准确的判断。）"

    return answer

class SmartRouter:
    def __init__(self):
        self.small_model = load_small_model()  # 1B 参数
        self.large_model = load_large_model()  # 7B 参数
        self.cache = LRUCache(maxsize=10000)

    def route(self, image, question):
        # 生成缓存 key
        cache_key = generate_cache_key(image, question)

        # 检查缓存
        if cache_key in self.cache:
            return self.cache[cache_key]

        # 根据问题复杂度选择模型
        complexity = estimate_complexity(question)

        if complexity == "low":
            model = self.small_model
        else:
            model = self.large_model

        # 推理
        answer = model.generate(image, question)

        # 缓存结果
        self.cache[cache_key] = answer

        return answer