帮我对比两个函数的差异，A: def forward( self, hidden_states: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.LongTensor] = None, past_key_value: Optional[Cache] = None, output_attentions: bool = False, use_cache: bool = False, cache_position: Optional[torch.LongTensor] = None, **kwargs, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: bsz, q_len, _ = hidden_states.size() if self.config.pretraining_tp > 1: key_value_slicing = (self.num_key_value_heads * self.head_dim) // self.config.pretraining_tp query_slices = self.q_proj.weight.split( (self.num_heads * self.head_dim) // self.config.pretraining_tp, dim=0 ) key_slices = self.k_proj.weight.split(key_value_slicing, dim=0) value_slices = self.v_proj.weight.split(key_value_slicing, dim=0) query_states = [F.linear(hidden_states, query_slices[i]) for i in range(self.config.pretraining_tp)] query_states = torch.cat(query_states, dim=-1) key_states = [F.linear(hidden_states, key_slices[i]) for i in range(self.config.pretraining_tp)] key_states = torch.cat(key_states, dim=-1) value_states = [F.linear(hidden_states, value_slices[i]) for i in range(self.config.pretraining_tp)] value_states = torch.cat(value_states, dim=-1) else: query_states = self.q_proj(hidden_states) key_states = self.k_proj(hidden_states) value_states = self.v_proj(hidden_states) query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) cos, sin = self.rotary_emb(value_states, position_ids) query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_value is not None: # sin and cos are specific to RoPE models; cache_position needed for the static cache cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) key_states = repeat_kv(key_states, self.num_key_value_groups) value_states = repeat_kv(value_states, self.num_key_value_groups) attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) if attention_mask is not None: # no matter the length, we just slice it causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] attn_weights = attn_weights + causal_mask # upcast attention to fp32 attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) attn_output = torch.matmul(attn_weights, value_states) if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): raise ValueError( f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" f" {attn_output.size()}" ) attn_output = attn_output.transpose(1, 2).contiguous() attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) if self.config.pretraining_tp > 1: attn_output = attn_output.split(self.hidden_size // self.config.pretraining_tp, dim=2) o_proj_slices = self.o_proj.weight.split(self.hidden_size // self.config.pretraining_tp, dim=1) attn_output = sum([F.linear(attn_output[i], o_proj_slices[i]) for i in range(self.config.pretraining_tp)]) else: attn_output = self.o_proj(attn_output) if not output_attentions: attn_weights = None return attn_output, attn_weights, past_key_value

def gcp_forward( self, hidden_states: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.LongTensor] = None, past_key_value: Optional[Cache] = None, output_attentions: bool = False, use_cache: bool = False, cache_position: Optional[torch.LongTensor] = None, position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, **kwargs, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: bsz, q_len, _ = hidden_states.size()

attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.LongTensor] = None, cache_position: Optional[torch.LongTensor] = None, **kwargs ) -> Tuple[torch.Tensor, ...]: # 1. ...

import json import torch from typing import Dict, List, Optional, Tuple from torch.utils.data import Dataset from collections import defaultdict import transformers from peft import LoraConfig, TaskType, get_peft_model from torch.utils.data import DataLoader from transformers import Trainer, TrainingArguments from lora_plus import LoraPlusTrainer from swanlab.integration.transformers import SwanLabCallback import swanlab import numpy as np import pandas as pd import re from tqdm import tqdm from transformers import PreTrainedTokenizer, AutoTokenizer import torch.nn as nn from transformers import PreTrainedModel from torch.nn import CrossEntropyLoss, MSELoss # 分子公式解析函数 def parse_chem_formula(formula): pattern = r'([A-Z][a-z]?)(\d)' matches = re.findall(pattern, formula) element_counts = defaultdict(int) for (element, count) in matches: count = int(count) if count else 1 element_counts[element] += count return element_counts def generate_element_list(formula): element_counts = parse_chem_formula(formula) elements = [] for element, count in element_counts.items(): if element != "H": elements.extend([element] count) return ''.join(elements) # 初始化SwanLab swanlab.init("Finetune-Llama3.2-with-Encoder") swanlab_callback = SwanLabCallback( project="Finetune-Llama3.2-with-Encoder", experiment_name="Finetune-Llama3.2-with-Encoder" ) # 常量定义 CHEM_FORMULA_SIZE = r"([A-Z][a-z])([0-9])" VALID_ELEMENTS = ["C", "N", "P", "O", "S", "Si", "I", "H", "Cl", "F", "Br", "B", "Se", "Fe", "Co", "As", "K", "Na"] element_to_idx = {elem: idx for idx, elem in enumerate(VALID_ELEMENTS)} # 化学式转密集向量 def formula_to_dense(chem_formula: str) -> torch.Tensor: dense_vec = torch.zeros(len(VALID_ELEMENTS), dtype=torch.float32) matches = re.findall(CHEM_FORMULA_SIZE, chem_formula) for chem_symbol, num_str in matches: num = 1 if num_str == "" else int(num_str) if chem_symbol in element_to_idx: idx = element_to_idx[chem_symbol] dense_vec[idx] += num return dense_vec # 位置编码生成 def positional_encoding(max_position: int, d_model: int, min_freq: float = 1e-4) -> torch.Tensor: position = torch.arange(max_position).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2) * (-torch.log(torch.tensor(min_freq)) / d_model)) pos_enc = torch.zeros(max_position, d_model) pos_enc[:, 0::2] = torch.sin(position * div_term) pos_enc[:, 1::2] = torch.cos(position * div_term) return pos_enc # 初始化位置编码矩阵 P = positional_encoding(2000000, 254) dimn = 254 # 与位置编码维度一致 # 质谱数据编码 def encode_spectra(rag_tensor: list, P: torch.Tensor, dimn: int) -> torch.Tensor: encoded_list = [] for sample in rag_tensor: mz_list, intensity_list = sample base_features = torch.tensor([mz_list, intensity_list], dtype=torch.float32).T pos_enc = torch.stack([P[min(int(mz), P.size(0)-1)] for mz in mz_list]) features = torch.cat([base_features, pos_enc], dim=1) if features.size(0) < 501: padding = torch.zeros(501 - features.size(0), features.size(1)) features = torch.cat([features, padding], dim=0) else: features = features[:501] encoded_list.append(features) return torch.stack(encoded_list) # 质谱数据预处理 def preprocess_spectra(df: pd.DataFrame) -> list: spectra_list = [] for idx, row in tqdm(df.iterrows(), total=len(df)): spectrum_str = row['Spectrum'] total_mass = row['Total Exact Mass'] pairs = spectrum_str.split() mz_list, intensity_list = [], [] for pair in pairs: mz, intensity = pair.split(':') mz_list.append(float(mz)) intensity_list.append(float(intensity)) mz_list.append(total_mass) intensity_list.append(0.0) mz_list = [round(mz, 2) for mz in mz_list] intensity_list = [round(intensity, 2) for intensity in intensity_list] spectra_list.append([mz_list, intensity_list]) return spectra_list class MolecularDataset(Dataset): def init(self, csv_path: str, tokenizer: AutoTokenizer, max_seq_len: int = 512): self.df = pd.read_csv(csv_path) self.tokenizer = tokenizer self.max_seq_len = max_seq_len self.pad_token_id = tokenizer.pad_token_id self.mask_token_id = tokenizer.mask_token_id if tokenizer.mask_token_id is not None else tokenizer.convert_tokens_to_ids("<mask>") spectra_data = preprocess_spectra(self.df) self.spec_encoded = encode_spectra(spectra_data, P, dimn) self.element_lists = [generate_element_list(formula) for formula in self.df['Molecular Formula']] self.element_lengths = [] for elem_list in self.element_lists: elem_tokens = self.tokenizer(elem_list, add_special_tokens=False)['input_ids'] self.element_lengths.append(len(elem_tokens)) def len(self): return len(self.df) def getitem(self, idx) -> dict: formula = self.df.iloc[idx]['Molecular Formula'] formula_vec = formula_to_dense(formula).squeeze(0) # 压缩为1D向量 spec_matrix = self.spec_encoded[idx] element_list = self.element_lists[idx] element_text = f"<|Spectrum|>{element_list}" selfies_str = self.df.iloc[idx]['SELFIES'] selfies_text = f"{selfies_str}" input_text = f"{element_text}{selfies_text}" encoding = self.tokenizer( input_text, add_special_tokens=False, padding='max_length', truncation=True, max_length=self.max_seq_len, return_tensors='pt' ) input_ids = encoding['input_ids'].squeeze(0) attention_mask = encoding['attention_mask'].squeeze(0) labels = input_ids.clone() labels[labels == self.pad_token_id] = -100 element_len = self.element_lengths[idx] element_end = 3 + element_len # , <|Spectrum|>, 元素列表 if element_end < len(labels): labels[:element_end] = -100 return { 'encoder1_inputs': formula_vec, # 注意：现在是1D向量 'encoder2_inputs': spec_matrix, 'input_ids': input_ids, 'attention_mask': attention_mask, 'labels': labels, 'formula_labels': formula_vec, # 添加元素计数标签 } # 加载tokenizer tokenizer = AutoTokenizer.from_pretrained('/root/workspace/d21lv5s7v38s73b4ddlg/checkpoint-2500') if tokenizer.mask_token is None: tokenizer.add_special_tokens({"mask_token": "<mask>"}) # 创建数据集 dataset = MolecularDataset('/root/workspace/d21lv5s7v38s73b4ddlg/SELFIES-SFT.csv', tokenizer) def custom_collator(features: List[Dict]) -> Dict: batch = { 'encoder1_inputs': torch.stack([f['encoder1_inputs'] for f in features]), # 形状: (batch_size, 18) 'encoder2_inputs': torch.stack([f['encoder2_inputs'] for f in features]), 'input_ids': torch.stack([f['input_ids'] for f in features]), 'attention_mask': torch.stack([f['attention_mask'] for f in features]), 'labels': torch.stack([f['labels'] for f in features]), 'formula_labels': torch.stack([f['formula_labels'] for f in features]), # 形状: (batch_size, 18) } return batch class ElementPredictionHead(nn.Module): """化学元素计数预测头部""" def init(self, hidden_size, output_size=18): super().init() self.dense = nn.Linear(hidden_size, hidden_size) self.activation = nn.ReLU() self.layer_norm = nn.LayerNorm(hidden_size) self.out_proj = nn.Linear(hidden_size, output_size) def forward(self, hidden_states): x = self.dense(hidden_states) x = self.activation(x) x = self.layer_norm(x) x = self.out_proj(x) return x class LlamaWithEncoder(PreTrainedModel): def init(self, base_model, encoder1_dim=18, encoder2_dim=256, hidden_dim=512): self.config = base_model.config super().init(self.config) self.model = base_model # 分子式编码器 encoder1_layer = nn.TransformerEncoderLayer( d_model=encoder1_dim, nhead=3, dim_feedforward=hidden_dim, batch_first=True ) self.encoder1 = nn.TransformerEncoder(encoder1_layer, num_layers=2) # 质谱编码器 encoder2_layer = nn.TransformerEncoderLayer( d_model=encoder2_dim, nhead=8, dim_feedforward=hidden_dim, batch_first=True ) self.encoder2 = nn.TransformerEncoder(encoder2_layer, num_layers=2) # 投影层 self.proj1 = nn.Linear(encoder1_dim, base_model.config.hidden_size) self.proj2 = nn.Linear(encoder2_dim, base_model.config.hidden_size) # 嵌入层 self.embed_tokens = nn.Embedding( num_embeddings=base_model.config.vocab_size, embedding_dim=base_model.config.hidden_size, padding_idx=base_model.config.pad_token_id ) self.embed_tokens.weight.data = base_model.get_input_embeddings().weight.data.clone() # 添加元素计数预测头 self.element_head = ElementPredictionHead(base_model.config.hidden_size) # PEFT所需方法 def get_input_embeddings(self): return self.embed_tokens def set_input_embeddings(self, value): self.embed_tokens = value def get_output_embeddings(self): return self.model.get_output_embeddings() def set_output_embeddings(self, new_embeddings): self.model.set_output_embeddings(new_embeddings) def get_base_model(self): return self.model def forward( self, input_ids: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.FloatTensor] = None, encoder1_inputs: Optional[torch.FloatTensor] = None, encoder2_inputs: Optional[torch.FloatTensor] = None, labels: Optional[torch.LongTensor] = None, formula_labels: Optional[torch.FloatTensor] = None, # 新增：元素计数标签 past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, kwargs ): return_dict = return_dict if return_dict is not None else self.config.use_return_dict # 1. 编码器处理 enc1_out = self.encoder1(encoder1_inputs.unsqueeze(1)) # 添加序列维度 enc1_out = enc1_out.mean(dim=1) # (batch_size, encoder1_dim) enc1_proj = self.proj1(enc1_out) # (batch_size, hidden_size) enc2_out = self.encoder2(encoder2_inputs) # (batch_size, 501, encoder2_dim) enc2_out = enc2_out.mean(dim=1) # (batch_size, encoder2_dim) enc2_proj = self.proj2(enc2_out) # (batch_size, hidden_size) # 合并编码器输出 mask_replacement = (enc1_proj + enc2_proj) / 2 # (batch_size, hidden_size) # 2. 获取原始嵌入 embeddings = self.embed_tokens(input_ids) # (batch_size, seq_len, hidden_size) batch_size, seq_len, hidden_size = embeddings.size() # 3. 替换<mask> token if seq_len > 2: mask_embed = mask_replacement.unsqueeze(1) # (batch_size, 1, hidden_size) part1 = embeddings[:, :2, :] # (batch_size, 2, hidden_size) part2 = mask_embed # (batch_size, 1, hidden_size) part3 = embeddings[:, 3:, :] # (batch_size, seq_len-3, hidden_size) new_embeddings = torch.cat([part1, part2, part3], dim=1) # (batch_size, seq_len, hidden_size) else: new_embeddings = embeddings # 4. 调用基础模型 model_output = self.model( inputs_embeds=new_embeddings, attention_mask=attention_mask, labels=labels, past_key_values=past_key_values, output_attentions=output_attentions, output_hidden_states=True, # 必须返回隐藏状态用于元素预测 return_dict=return_dict, kwargs ) # 5. 元素计数预测 element_pred = None element_loss = None if formula_labels is not None: # 获取最后一个非填充token的隐藏状态 seq_lengths = attention_mask.sum(dim=1) - 1 # 最后一个有效token的索引 batch_indices = torch.arange(batch_size, device=model_output.hidden_states[-1].device) last_token_hidden = model_output.hidden_states[-1][batch_indices, seq_lengths] # (batch_size, hidden_size) # 预测元素计数 element_pred = self.element_head(last_token_hidden) # (batch_size, 18) # 计算元素计数损失（MSE损失） element_loss = MSELoss()(element_pred, formula_labels) # 组合总损失：语言模型损失 + 元素计数损失 total_loss = model_output.loss + 0.5 * element_loss else: total_loss = model_output.loss # 返回结果 if not return_dict: output = (model_output.logits,) if element_pred is not None: output += (element_pred,) return (total_loss,) + output if total_loss is not None else output return { 'loss': total_loss, 'logits': model_output.logits, 'element_pred': element_pred, 'element_loss': element_loss, 'hidden_states': model_output.hidden_states, 'past_key_values': model_output.past_key_values, 'attentions': model_output.attentions } # 加载预训练模型 base_model = transformers.AutoModelForCausalLM.from_pretrained( "/root/workspace/d21lv5s7v38s73b4ddlg/checkpoint-2500", trust_remote_code=True, torch_dtype=torch.bfloat16, ) model = LlamaWithEncoder(base_model) # 配置LoRA lora_config = LoraConfig( r=8, lora_alpha=16, target_modules="all-linear", lora_dropout=0.0, bias="none", task_type="CAUSAL_LM" ) model = get_peft_model(model, lora_config) model.print_trainable_parameters() # 训练参数 training_args = TrainingArguments( output_dir="./llama3.2-SELFIES-SFT", per_device_train_batch_size=24, gradient_accumulation_steps=24, num_train_epochs=12, learning_rate=5.0e-05, optim="adamw_torch", logging_steps=10, bf16=True, save_strategy="steps", lr_scheduler_type='cosine', max_grad_norm=1.0, save_steps=2000, warmup_steps=0 ) class CustomTrainer(LoraPlusTrainer): def get_train_dataloader(self) -> DataLoader: return DataLoader( self.train_dataset, batch_size=self.args.train_batch_size, shuffle=True, collate_fn=self.data_collator, drop_last=False, ) # 训练模型 lp_trainer = CustomTrainer( model, training_args, train_dataset=dataset, tokenizer=tokenizer, data_collator=custom_collator, callbacks=[swanlab_callback], ) lp_trainer.train() lp_trainer.save_model(output_dir='./llama3.2-SELFIES-SFT') # 合并LoRA权重并移除元素预测头 model = model.merge_and_unload() model.element_head = None # 移除元素预测头 # 保存模型（不包括元素预测头） save_directory = './llama3.2-SELFIES' model.save_pretrained(save_directory, safe_serialization=True) tokenizer.save_pretrained(save_directory)不对，要对应修改为 element_text = f"<｜User｜><mask>{element_list}" # SELFIES目标序列并添加标记 selfies_str = self.df.iloc[idx]['SELFIES'] selfies_text = f"<｜Assistant｜>{selfies_str}"，同时化学元素计数预测模型的输入token取<｜Assistant｜>token之后的，写出完整的修改代码

但是，注意：在元素部分，我们添加了mask_token，这个mask_token在模型的前向传播中会被替换为两个编码器的融合表示。因此，在输入文本中，mask_token的位置需要被替换。这个替换操作在模型的前向传播中已经实现...

这个是我用来对大模型微调以实现对商品的属性进行预测的代码，注意这个数据集的分布情况比较特别，两类重量数值90%的样本均分在0-100这个区间，然后体积参数70%的样本分布在5000-500000这个区间，在进行训练时，一开始损失上万，但是马上损失降到1以内，结合代码进行分析，重点分析损失函数部分import os import json import json5 import math import logging import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from torch.optim import AdamW from torch.utils.data import Dataset, DataLoader from typing import List, Dict, Any, Optional, Tuple from pathlib import Path from PIL import Image from tqdm import tqdm from transformers import ( AutoProcessor, Qwen2_5_VLForConditionalGeneration, Trainer, TrainingArguments, get_scheduler, TrainerCallback ) import swanlab from swift import Swift logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[logging.StreamHandler()] ) logger = logging.getLogger(__name__) class MultiModalDataProcessor: def __init__( self, processor_path: str, max_pixels: int = 301056 // 2, max_length: int = 4096, ): self.max_pixels = max_pixels self.max_length = max_length self.processor = AutoProcessor.from_pretrained(processor_path) # 添加归一化参数（根据实际数据分布调整） self.weight_scale = 1000.0 # 重量缩放因子 self.size_scale = 10000.0 # 尺寸缩放因子 def load_data(self, file_paths: List[str]) -> List[Dict[str, Any]]: data = [] for path in file_paths: try: with open(path, "r", encoding="utf-8") as fp: for line in fp: try: item = json.loads(line.strip()) if self._validate_sample(item): data.append(item) except (json.JSONDecodeError, KeyError) as e: logger.warning(f"Error processing line in {path}: {e}") except FileNotFoundError: logger.warning(f"File not found: {path}") logger.info(f"Loaded {len(data)} samples") return data def _validate_sample(self, sample: Dict[str, Any]) -> bool: required = ["messages", "images"] if not all(key in sample for key in required): return False try: if len(sample["messages"]) < 2: return False if not isinstance(sample["messages"][0]["content"], str): return False json5.loads(sample["messages"][1]["content"]) return True except (KeyError, json.JSONDecodeError): return False @staticmethod def rescale_image(img: Image.Image, max_pixels: int) -> Image.Image: if img is None: return Image.new('RGB', (224, 224), (0, 0, 0)) if max_pixels > 0 and img.width * img.height > max_pixels: ratio = img.width / img.height height_scaled = math.sqrt(max_pixels / ratio) width_scaled = height_scaled * ratio img = img.resize((int(width_scaled), int(height_scaled)), Image.BILINEAR) # Pad to fixed size if needed if img.size != (224, 224): # or your desired fixed size img = img.resize((224, 224), Image.BILINEAR) return img def _load_image(self, image_path: str) -> Optional[Image.Image]: try: if not os.path.exists(image_path): return None return Image.open(image_path).convert("RGB") except (IOError, OSError): return None def preprocess_sample( self, sample: Dict[str, Any], require_image: bool = False ) -> Optional[Dict[str, torch.Tensor]]: try: text_content = sample["messages"][0]['content'] image_path = sample.get("images", "") image = self._load_image(image_path) if image_path else None content = [{"type": "text", "text": text_content}] content.insert(0, {"type": "image"}) messages = [{"role": "user", "content": content}] text = self.processor.apply_chat_template( messages, tokenize=False, add_generation_prompt=True ) inputs = self.processor( text=text, images=self.rescale_image(image, self.max_pixels), return_tensors="pt", padding="max_length", truncation=True, max_length=self.max_length, add_special_tokens=True, ) target_data = json5.loads(sample["messages"][1]["content"]) res = { "input_ids": inputs["input_ids"].squeeze(0), "attention_mask": inputs["attention_mask"].squeeze(0), # 归一化目标值 "product_weight": torch.tensor(target_data["product_weight"] / self.weight_scale, dtype=torch.float32), "package_length": torch.tensor(target_data["package_length"] / self.size_scale, dtype=torch.float32), "package_width": torch.tensor(target_data["package_width"] / self.size_scale, dtype=torch.float32), "package_height": torch.tensor(target_data["package_height"] / self.size_scale, dtype=torch.float32), "package_weight": torch.tensor(target_data["package_weight"] / self.weight_scale, dtype=torch.float32), } res["pixel_values"] = inputs["pixel_values"].squeeze(0) res["image_grid_thw"] = inputs["image_grid_thw"].squeeze(0) return res except Exception as e: logger.warning(f"Error processing sample: {e}") return None class MultiModalDataset(Dataset): def __init__( self, data: List[Dict[str, Any]], processor: MultiModalDataProcessor, require_image: bool = False ): self.data = data self.processor = processor self.require_image = require_image def __len__(self) -> int: return len(self.data) def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: sample = self.data[idx] processed = self.processor.preprocess_sample(sample, self.require_image) if processed is None: return self.__getitem__(torch.randint(0, len(self), (1,)).item()) return processed class MultiOutputPredictionHead(nn.Module): def __init__(self, hidden_size: int, intermediate_size: int = 1024, dropout: float = 0.2): # 增加中间层大小 super().__init__() self.dtype = torch.bfloat16 # 分离重量和尺寸的特征提取 self.weight_feature = nn.Sequential( nn.Linear(hidden_size, intermediate_size, dtype=self.dtype), nn.GELU(), nn.Dropout(dropout), nn.LayerNorm(intermediate_size, dtype=self.dtype) ) self.size_feature = nn.Sequential( nn.Linear(hidden_size, intermediate_size, dtype=self.dtype), nn.GELU(), nn.Dropout(dropout), nn.LayerNorm(intermediate_size, dtype=self.dtype) ) # 任务特定头 self.weight_heads = nn.ModuleDict({ name: nn.Sequential( nn.Linear(intermediate_size, intermediate_size // 2, dtype=self.dtype), nn.GELU(), nn.Dropout(dropout), nn.Linear(intermediate_size // 2, 1, dtype=self.dtype), nn.Softplus() ) for name in ['product_weight', 'package_weight'] }) self.size_heads = nn.ModuleDict({ name: nn.Sequential( nn.Linear(intermediate_size, intermediate_size // 2, dtype=self.dtype), nn.GELU(), nn.Dropout(dropout), nn.Linear(intermediate_size // 2, 1, dtype=self.dtype), nn.Softplus() ) for name in ['package_length', 'package_width', 'package_height'] }) # 初始化权重 for module in self.modules(): if isinstance(module, nn.Linear): nn.init.kaiming_normal_(module.weight) if module.bias is not None: nn.init.zeros_(module.bias) def forward(self, hidden_states: torch.Tensor) -> Dict[str, torch.Tensor]: weight_features = self.weight_feature(hidden_states) size_features = self.size_feature(hidden_states) outputs = {} for name in ['product_weight', 'package_weight']: outputs[name] = self.weight_heads[name](weight_features).squeeze(-1) for name in ['package_length', 'package_width', 'package_height']: outputs[name] = self.size_heads[name](size_features).squeeze(-1) return outputs class WeightAdaptiveLoss(nn.Module): def __init__(self, weight_threshold=100.0, huber_delta=1.0, logscale_factor=0.5): super().__init__() self.weight_threshold = weight_threshold self.huber_delta = huber_delta self.logscale_factor = logscale_factor self.huber_loss = nn.HuberLoss(reduction='none', delta=self.huber_delta) def forward(self, predictions, targets): losses = {} for name in ['product_weight', 'package_weight']: pred = predictions[name] target = targets[name] low_weight_mask = target < self.weight_threshold high_weight_mask = ~low_weight_mask if torch.any(low_weight_mask): losses[f"{name}_low"] = self.huber_loss(pred[low_weight_mask], target[low_weight_mask]).mean() if torch.any(high_weight_mask): log_pred = torch.log1p(pred[high_weight_mask]) log_target = torch.log1p(target[high_weight_mask]) losses[f"{name}_high"] = F.mse_loss(log_pred, log_target) * self.logscale_factor return losses class DynamicHuberLoss(nn.Module): """动态调整delta的Huber损失，适应不同尺度""" def __init__(self, base_delta=1.0, scale_factor=0.01): super().__init__() self.base_delta = base_delta self.scale_factor = scale_factor def forward(self, input, target): # 根据目标值动态调整delta delta = self.base_delta + self.scale_factor * target.abs().mean() loss = 0.5 * (input - target)**2 * (torch.abs(input - target) <= delta).float() + delta * (torch.abs(input - target) - 0.5 * delta) * (torch.abs(input - target) > delta).float() return loss.mean() class PhysicsAwareLoss(nn.Module): def __init__(self, weight=1.0, size=1.0, constraint=0.1): super().__init__() self.weight_factor = weight self.size_factor = size self.constraint_factor = constraint def forward(self, predictions, targets): total_loss = 0 # 1. 重量损失 - 分层处理 for name in ['product_weight', 'package_weight']: # 小值样本 (0-50) small_mask = targets[name] < 50 if torch.any(small_mask): small_loss = F.mse_loss( predictions[name][small_mask], targets[name][small_mask] ) total_loss += self.weight_factor * small_loss # 中值样本 (50-100) medium_mask = (targets[name] >= 50) & (targets[name] < 100) if torch.any(medium_mask): medium_loss = F.huber_loss( predictions[name][medium_mask], targets[name][medium_mask], delta=5.0 ) total_loss += self.weight_factor * medium_loss # 大值样本 (100+) large_mask = targets[name] >= 100 if torch.any(large_mask): log_pred = torch.log1p(predictions[name][large_mask]) log_target = torch.log1p(targets[name][large_mask]) large_loss = F.mse_loss(log_pred, log_target) total_loss += self.weight_factor * large_loss * 2.0 # 增加权重 # 2. 尺寸损失 - 分层处理 for name in ['package_length', 'package_width', 'package_height']: # 小尺寸样本 (0-5000) small_mask = targets[name] < 5000 if torch.any(small_mask): small_loss = F.huber_loss( predictions[name][small_mask], targets[name][small_mask], delta=50.0 ) total_loss += self.size_factor * small_loss # 大尺寸样本 (5000+) large_mask = targets[name] >= 5000 if torch.any(large_mask): log_pred = torch.log1p(predictions[name][large_mask]) log_target = torch.log1p(targets[name][large_mask]) large_loss = F.mse_loss(log_pred, log_target) total_loss += self.size_factor * large_loss * 1.5 # 增加权重 # 3. 物理约束（降低权重） weight_constraint = F.relu(predictions['product_weight'] - predictions['package_weight']) total_loss += self.constraint_factor * torch.mean(weight_constraint) # 4. 尺寸合理性约束 size_constraint = F.relu(-predictions['package_length']) + \ F.relu(-predictions['package_width']) + \ F.relu(-predictions['package_height']) total_loss += self.constraint_factor * torch.mean(size_constraint) # 5. 体积一致性约束（显著降低权重） volume_pred = (predictions['package_length'] * predictions['package_width'] * predictions['package_height']) volume_target = (targets['package_length'] * targets['package_width'] * targets['package_height']) volume_loss = F.huber_loss(volume_pred, volume_target, delta=5000) relative_volume_loss = torch.mean( torch.abs(volume_pred - volume_target) / (volume_target + 1e-6) ) total_loss += 0.02 * (volume_loss + relative_volume_loss) return total_loss class MultiModalModel(nn.Module): def __init__(self, base_model_path: str): super().__init__() self.base_model = Qwen2_5_VLForConditionalGeneration.from_pretrained( base_model_path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True ) for param in self.base_model.parameters(): param.requires_grad = False hidden_size = self.base_model.config.hidden_size self.prediction_head = MultiOutputPredictionHead(hidden_size) self.layer_selector = nn.Sequential( nn.Linear(hidden_size, 128, dtype=torch.bfloat16), nn.GELU(), nn.Linear(128, 4, dtype=torch.bfloat16), nn.Softmax(dim=-1) ) self.base_model = Swift.prepare_model(self.base_model, config={}) # 添加梯度检查点支持 if hasattr(self.base_model, "gradient_checkpointing_enable"): self.gradient_checkpointing_enable = self.base_model.gradient_checkpointing_enable if hasattr(self.base_model, "gradient_checkpointing_disable"): self.gradient_checkpointing_disable = self.base_model.gradient_checkpointing_disable def forward(self, input_ids, attention_mask, pixel_values=None, image_grid_thw=None, **kwargs): outputs = self.base_model( input_ids=input_ids, attention_mask=attention_mask, pixel_values=pixel_values, image_grid_thw=image_grid_thw, output_hidden_states=True ) last_4_layers = outputs.hidden_states[-4:] layer_weights = self.layer_selector(outputs.hidden_states[-1][:, 0]) weighted_features = torch.zeros_like(last_4_layers[0]) for i in range(4): # 正确广播权重以匹配特征维度 weight_expanded = layer_weights[:, i].view(-1, 1, 1).expand_as(last_4_layers[i]) weighted_features += weight_expanded * last_4_layers[i] return self.prediction_head(weighted_features[:, -1, :]) class CustomTrainer(Trainer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.loss_fn = PhysicsAwareLoss(weight=1.0, size=0.8, constraint=0.5) def compute_loss(self, model, inputs, return_outputs=False, **kwargs): targets = { 'product_weight': inputs["product_weight"], 'package_length': inputs["package_length"], 'package_width': inputs["package_width"], 'package_height': inputs["package_height"], 'package_weight': inputs["package_weight"] } model_inputs = {k: v for k, v in inputs.items() if k not in targets} predictions = model(**model_inputs) total_loss = self.loss_fn(predictions, targets) if return_outputs: return (total_loss, predictions) return total_loss def compute_metrics(eval_pred): predictions, labels = eval_pred metrics = {} output_names = ['product_weight', 'package_length', 'package_width', 'package_height', 'package_weight'] for i, name in enumerate(output_names): pred = predictions[i].squeeze() label = labels[i].squeeze() abs_error = np.abs(pred - label) squared_error = (pred - label) ** 2 metrics.update({ f"{name}_mse": float(np.mean(squared_error)), f"{name}_mae": float(np.mean(abs_error)), f"{name}_r2": float(1 - np.sum(squared_error)/np.sum((label - np.mean(label)) ** 2)), f"{name}_max_error": float(np.max(abs_error)), f"{name}_median_error": float(np.median(abs_error)), f"{name}_std_error": float(np.std(abs_error)), f"{name}_correlation": float(np.corrcoef(pred, label)[0, 1]) }) for metric in ['mse', 'mae', 'r2', 'max_error', 'median_error', 'std_error', 'correlation']: values = [metrics[f"{name}_{metric}"] for name in output_names] metrics[f"avg_{metric}"] = float(np.mean(values)) return metrics def train(): TRAIN_FILES = ["./data/train_26.4k_20250703.jsonl"] VAL_FILES = ['./data/test_1.0k_20250703.jsonl'] CONFIG = { "model_path": '/nas_data/xiao/models/Qwen2.5-VL-7B-Instruct', "processor_path": '/nas_data/xiao/models/Qwen2.5-VL-7B-Instruct', "train_files": TRAIN_FILES, "val_files": VAL_FILES, "max_pixels": 301056, "max_length": 4096, "output_dir": "./output", "learning_rate": 1e-5, "batch_size": 4, "grad_accum_steps": 2, "num_epochs": 1, "warmup_ratio": 0.05, "weight_decay": 0.1, "logging_steps": 5, "eval_steps": 500, "save_steps": 500, "save_total_limit": 5, "fp16": False, "bf16": True, "gradient_checkpointing": True, "deepspeed_config": { "train_micro_batch_size_per_gpu": "auto", "gradient_accumulation_steps": "auto", "optimizer": { "type": "AdamW", "params": { "lr": "auto", "weight_decay": "auto", "betas": "auto", } }, "scheduler": { "type": "WarmupDecayLR", "params": { "warmup_min_lr": 0, "warmup_max_lr": 1e-4, "warmup_num_steps": "auto", "total_num_steps": "auto", } }, "fp16": { "enabled": "auto", }, "bf16": { "enabled": "auto", }, "zero_optimization": { "stage": 2, "offload_optimizer": { "device": "none", "pin_memory": True }, "allgather_partitions": True, "allgather_bucket_size": 2e8, "overlap_comm": True, "reduce_scatter": True, "reduce_bucket_size": 2e8, "contiguous_gradients": True }, "gradient_clipping": 1.0, "steps_per_print": 5, "wall_clock_breakdown": False } } # 初始化数据处理器和数据集 logger.info("Initializing data processor...") data_processor = MultiModalDataProcessor( processor_path=CONFIG["processor_path"], max_pixels=CONFIG["max_pixels"], max_length=CONFIG["max_length"], ) logger.info("Loading training data...") train_data = data_processor.load_data(CONFIG["train_files"]) logger.info("Loading validation data...") val_data = data_processor.load_data(CONFIG["val_files"]) logger.info("Creating datasets...") train_dataset = MultiModalDataset(train_data, data_processor) val_dataset = MultiModalDataset(val_data, data_processor) logger.info("Loading model...") model = MultiModalModel(CONFIG["model_path"]) # 计算总步数和1%的步数 batch_size = CONFIG["batch_size"] grad_accum_steps = CONFIG["grad_accum_steps"] num_epochs = CONFIG["num_epochs"] total_steps = (len(train_dataset) * num_epochs) // (batch_size * grad_accum_steps) logging_steps = max(1, total_steps // 100) # 每1%打印一次 logger.info(f"Total training steps: {total_steps}") logger.info(f"Logging every {logging_steps} steps") # 训练参数 - 添加自动日志步长 training_args = TrainingArguments( output_dir=CONFIG["output_dir"], remove_unused_columns=False, learning_rate=CONFIG["learning_rate"], per_device_train_batch_size=batch_size, gradient_accumulation_steps=grad_accum_steps, num_train_epochs=num_epochs, weight_decay=CONFIG["weight_decay"], warmup_ratio=CONFIG["warmup_ratio"], # 关键修改：设置日志记录策略 logging_strategy="steps", logging_steps=logging_steps, # 使用计算出的步数 save_steps=CONFIG["save_steps"], save_total_limit=CONFIG["save_total_limit"], fp16=CONFIG["fp16"], bf16=CONFIG["bf16"], gradient_checkpointing=CONFIG["gradient_checkpointing"], logging_first_step=True, save_strategy="steps", eval_strategy="steps", dataloader_num_workers=16, data_seed=42, greater_is_better=False, deepspeed=CONFIG["deepspeed_config"], eval_steps=CONFIG["eval_steps"], load_best_model_at_end=True, metric_for_best_model="eval_avg_mae", ) # 自定义回调类 class ProgressCallback(TrainerCallback): def __init__(self, total_steps): self.total_steps = total_steps self.last_reported = 0 def on_log(self, args, state, control, logs=None, **kwargs): if state.global_step == 0 or self.total_steps is None: return # 计算当前训练进度百分比 progress_percent = (state.global_step / self.total_steps) * 100 completed_percent = int(progress_percent) # 每1%打印一次 if completed_percent > self.last_reported: logger.info(f"Training progress: {completed_percent}% complete") logger.info(f"Current metrics: {logs}") self.last_reported = completed_percent # 创建训练器 logger.info("Creating trainer...") trainer = CustomTrainer( model=model, args=training_args, train_dataset=train_dataset, eval_dataset=val_dataset, compute_metrics=compute_metrics, callbacks=[ProgressCallback(total_steps)] # 添加自定义回调 ) # 开始训练 logger.info("Starting training...") try: train_result = trainer.train() trainer.save_model("./saved_models") logger.info(f"Training completed successfully. Metrics: {train_result.metrics}") except Exception as e: logger.error(f"Training failed: {e}") raise if __name__ == "__main__": train()

我们注意到用户描述的数据分布特点：两类重量数值90%的样本在0-100区间，体积参数70%的样本分布在5000-500000区间。训练一开始损失上万，但很快降到1以内。重点分析损失函数部分：用户代码中使用了自定义的...

代码出现问题：(style_tune) C:\Users\28996\Desktop\AI\persona_contrastive_finetuning>python Contrastive_Training_LM.py INFO:accelerate.utils.modeling:We will use 90% of the memory on device 0 for storing the model, and 10% for the buffer to avoid OOM. You can set max_memory in to a higher value to use more memory (at your own risk). trainable params: 1,572,864 || all params: 1,838,401,536 || trainable%: 0.0856 训练集样本示例: {'anchor_input_ids': [56568, 118919, 116122, 11319], 'positive_input_ids': [116122, 20412, 107340, 9370, 100357, 102323, 3837, 109202, 104078, 103975, 100675, 101940, 100912, 105054, 6313], 'negative_input_ids': [100323, 104307, 99245, 9370, 106059, 104060, 3837, 104530, 115604, 99329, 11319]} 验证集样本示例: {'anchor_input_ids': [56568, 118919, 116122, 11319], 'positive_input_ids': [116122, 20412, 107340, 9370, 100357, 102323, 3837, 109202, 104078, 103975, 100675, 101940, 100912, 105054, 6313], 'negative_input_ids': [100323, 104307, 99245, 9370, 106059, 104060, 3837, 104530, 115604, 99329, 11319]} Trainer.tokenizer is now deprecated. You should use Trainer.processing_class = processing_class instead. INFO:__main__:GPU内存使用: 已分配 2.93GB, 保留 4.13GB 可训练参数列表: - base_model.model.model.layers.0.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.0.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.0.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.0.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.1.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.1.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.1.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.1.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.2.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.2.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.2.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.2.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.3.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.3.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.3.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.3.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.4.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.4.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.4.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.4.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.5.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.5.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.5.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.5.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.6.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.6.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.6.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.6.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.7.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.7.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.7.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.7.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.8.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.8.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.8.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.8.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.9.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.9.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.9.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.9.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.10.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.10.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.10.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.10.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.11.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.11.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.11.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.11.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.12.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.12.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.12.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.12.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.13.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.13.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.13.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.13.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.14.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.14.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.14.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.14.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.15.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.15.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.15.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.15.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.16.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.16.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.16.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.16.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.17.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.17.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.17.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.17.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.18.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.18.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.18.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.18.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.19.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.19.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.19.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.19.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.20.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.20.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.20.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.20.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.21.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.21.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.21.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.21.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.22.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.22.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.22.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.22.self_attn.v_proj.lora_B.default.weight - base_model.model.model.layers.23.self_attn.q_proj.lora_A.default.weight - base_model.model.model.layers.23.self_attn.q_proj.lora_B.default.weight - base_model.model.model.layers.23.self_attn.v_proj.lora_A.default.weight - base_model.model.model.layers.23.self_attn.v_proj.lora_B.default.weight 0%| | 0/3 [00:00<?, ?it/s]You're using a Qwen2TokenizerFast tokenizer. Please note that with a fast tokenizer, using the __call__ method is faster than using a method to encode the text followed by a call to the pad method to get a padded encoding. Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead. Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead. INFO:__main__:GPU内存使用: 已分配 4.00GB, 保留 4.21GB Could not estimate the number of tokens of the input, floating-point operations will not be computed Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead. Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead. INFO:__main__:GPU内存使用: 已分配 4.02GB, 保留 4.22GB 33%|████████████████████████████ | 1/3 [00:03<00:06, 3.25s/it]Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead. Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead. INFO:__main__:GPU内存使用: 已分配 4.01GB, 保留 4.25GB Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead. Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead. INFO:__main__:GPU内存使用: 已分配 4.02GB, 保留 4.26GB 67%|████████████████████████████████████████████████████████ | 2/3 [00:06<00:02, 2.98s/it]Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead. Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead. INFO:__main__:GPU内存使用: 已分配 4.01GB, 保留 4.25GB Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead. Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead. INFO:__main__:GPU内存使用: 已分配 4.02GB, 保留 4.26GB {'train_runtime': 9.034, 'train_samples_per_second': 0.664, 'train_steps_per_second': 0.332, 'train_loss': 1.0772175788879395, 'epoch': 3.0} 100%|████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:09<00:00, 3.01s/it] Traceback (most recent call last): File "C:\Users\28996\Desktop\AI\persona_contrastive_finetuning\Contrastive_Training_LM.py", line 356, in <module> eval_results = trainer.evaluate() File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\transformers\trainer.py", line 4076, in evaluate output = eval_loop( File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\transformers\trainer.py", line 4270, in evaluation_loop losses, logits, labels = self.prediction_step(model, inputs, prediction_loss_only, ignore_keys=ignore_keys) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\transformers\trainer.py", line 4496, in prediction_step outputs = model(**inputs) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\torch\nn\modules\module.py", line 1736, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\torch\nn\modules\module.py", line 1747, in _call_impl return forward_call(*args, **kwargs) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\accelerate\utils\operations.py", line 818, in forward return model_forward(*args, **kwargs) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\accelerate\utils\operations.py", line 806, in __call__ return convert_to_fp32(self.model_forward(*args, **kwargs)) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\torch\amp\autocast_mode.py", line 44, in decorate_autocast return func(*args, **kwargs) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\peft\peft_model.py", line 1719, in forward return self.base_model( File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\torch\nn\modules\module.py", line 1736, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\torch\nn\modules\module.py", line 1747, in _call_impl return forward_call(*args, **kwargs) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\peft\tuners\tuners_utils.py", line 197, in forward return self.model.forward(*args, **kwargs) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\transformers\models\qwen2\modeling_qwen2.py", line 816, in forward outputs = self.model( File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\torch\nn\modules\module.py", line 1736, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\torch\nn\modules\module.py", line 1747, in _call_impl return forward_call(*args, **kwargs) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\transformers\models\qwen2\modeling_qwen2.py", line 521, in forward raise ValueError("You must specify exactly one of input_ids or inputs_embeds") ValueError: You must specify exactly one of input_ids or inputs_embeds (style_tune) C:\Users\28996\Desktop\AI\persona_contrastive_finetuning>python Contrastive_Training_LM.py Traceback (most recent call last): File "C:\Users\28996\Desktop\AI\persona_contrastive_finetuning\Contrastive_Training_LM.py", line 57, in <module> class ContrastiveTrainer(Trainer): File "C:\Users\28996\Desktop\AI\persona_contrastive_finetuning\Contrastive_Training_LM.py", line 63, in ContrastiveTrainer eval_dataset: Optional[Dataset] = None, NameError: name 'Dataset' is not defined 原代码如下：import torch import torch.nn as nn import torch.nn.functional as F from transformers import ( AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer, PreTrainedTokenizerBase, BitsAndBytesConfig ) from transformers.tokenization_utils_base import PreTrainedTokenizerBase from transformers.utils import PaddingStrategy from datasets import load_dataset from typing import Any, Dict, List, Optional, Tuple, Union import logging from dataclasses import dataclass import os import gc from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training @dataclass class EvalDataCollator: """评估专用的数据收集器""" tokenizer: PreTrainedTokenizerBase padding: Union[bool, str, PaddingStrategy] = True max_length: Optional[int] = None pad_to_multiple_of: Optional[int] = None return_tensors: str = "pt" def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, torch.Tensor]: # 评估时只使用正样本（用于语言建模评估） positive_features = [{"input_ids": f["positive_input_ids"]} for f in features] # 对正样本进行填充 batch_positive = self.tokenizer.pad( positive_features, padding=self.padding, max_length=self.max_length, pad_to_multiple_of=self.pad_to_multiple_of, return_tensors=self.return_tensors, ) # 创建注意力掩码 attention_mask = (batch_positive["input_ids"] != self.tokenizer.pad_token_id).int() # 创建标签（用于语言建模） labels = batch_positive["input_ids"].clone() labels[labels == self.tokenizer.pad_token_id] = -100 return { "input_ids": batch_positive["input_ids"], "attention_mask": attention_mask, "labels": labels } class ContrastiveTrainer(Trainer): """内存优化的训练器""" # ... [保持其他方法不变] ... def evaluate( self, eval_dataset: Optional[Dataset] = None, ignore_keys: Optional[List[str]] = None, metric_key_prefix: str = "eval", ) -> Dict[str, float]: """重写评估方法以使用专用的数据收集器""" # 创建评估专用的数据收集器 eval_data_collator = EvalDataCollator( tokenizer=self.tokenizer, max_length=256, padding="max_length" ) # 临时保存原始数据收集器 original_collator = self.data_collator try: # 使用评估专用的数据收集器 self.data_collator = eval_data_collator # 调用父类的评估方法 return super().evaluate( eval_dataset=eval_dataset, ignore_keys=ignore_keys, metric_key_prefix=metric_key_prefix ) finally: # 恢复原始数据收集器 self.data_collator = original_collator # 设置日志 logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) # 内存优化工具函数 def clear_memory(): """清除Python和CUDA缓存""" gc.collect() if torch.cuda.is_available(): torch.cuda.empty_cache() torch.cuda.reset_peak_memory_stats() def print_memory_usage(): """打印当前内存使用情况""" if torch.cuda.is_available(): allocated = torch.cuda.memory_allocated() / (1024 ** 3) reserved = torch.cuda.memory_reserved() / (1024 ** 3) logger.info(f"GPU内存使用: 已分配 {allocated:.2f}GB, 保留 {reserved:.2f}GB") else: logger.info("未检测到GPU") def tokenize_function(examples, tokenizer, max_length=256): """将文本转换为token IDs""" tokenized = {} # 对每个字段进行分词 for key in ['anchor', 'positive', 'negative']: if key in examples: # 使用分词器处理文本 result = tokenizer( examples[key], max_length=max_length, truncation=True, padding=False, return_tensors=None ) tokenized[f"{key}_input_ids"] = result["input_ids"] return tokenized @dataclass class ContrastiveDataCollator: """内存优化的数据收集器""" tokenizer: PreTrainedTokenizerBase padding: Union[bool, str, PaddingStrategy] = True max_length: Optional[int] = None pad_to_multiple_of: Optional[int] = None return_tensors: str = "pt" def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, torch.Tensor]: # 分离出三元组的各个部分 anchor_features = [{"input_ids": f["anchor_input_ids"]} for f in features] positive_features = [{"input_ids": f["positive_input_ids"]} for f in features] negative_features = [{"input_ids": f["negative_input_ids"]} for f in features] # 对每个部分分别进行填充 batch_anchor = self.tokenizer.pad( anchor_features, padding=self.padding, max_length=self.max_length, pad_to_multiple_of=self.pad_to_multiple_of, return_tensors=self.return_tensors, ) batch_positive = self.tokenizer.pad( positive_features, padding=self.padding, max_length=self.max_length, pad_to_multiple_of=self.pad_to_multiple_of, return_tensors=self.return_tensors, ) batch_negative = self.tokenizer.pad( negative_features, padding=self.padding, max_length=self.max_length, pad_to_multiple_of=self.pad_to_multiple_of, return_tensors=self.return_tensors, ) # 创建注意力掩码 def create_attention_mask(input_ids): return (input_ids != self.tokenizer.pad_token_id).int() # 释放中间变量内存 del anchor_features, positive_features, negative_features clear_memory() return { "anchor_input_ids": batch_anchor["input_ids"], "anchor_attention_mask": create_attention_mask(batch_anchor["input_ids"]), "positive_input_ids": batch_positive["input_ids"], "positive_attention_mask": create_attention_mask(batch_positive["input_ids"]), "negative_input_ids": batch_negative["input_ids"], "negative_attention_mask": create_attention_mask(batch_negative["input_ids"]), } class ContrastiveTrainer(Trainer): """内存优化的训练器""" def __init__(self, tokenizer=None, *args, contrastive_config=None, **kwargs): # 首先调用父类初始化 super().__init__(*args, **kwargs) # 关键修复：设置tokenizer self.tokenizer = tokenizer if contrastive_config is None: contrastive_config = {} # 设置默认值 self.temperature = contrastive_config.get("temperature", 0.07) self.margin = contrastive_config.get("margin", 0.3) self.contrastive_weight = contrastive_config.get("weight", 0.8) self.repr_layer = contrastive_config.get("repr_layer", -1) # 验证必要参数 if not hasattr(self.model.config, "output_hidden_states") or not self.model.config.output_hidden_states: raise ValueError("模型必须设置output_hidden_states=True") self.cross_entropy = nn.CrossEntropyLoss() def compute_contrastive_loss(self, anchor_emb, pos_emb, neg_emb): """计算对比损失""" # 计算余弦相似度 pos_sim = F.cosine_similarity(anchor_emb, pos_emb) neg_sim = F.cosine_similarity(anchor_emb, neg_emb) # 计算InfoNCE损失 numerator = torch.exp(pos_sim / self.temperature) denominator = numerator + torch.exp(neg_sim / self.temperature) info_nce_loss = -torch.log(numerator / (denominator + 1e-8)).mean() # 计算三元组损失 triplet_loss = F.relu(neg_sim - pos_sim + self.margin).mean() return info_nce_loss + triplet_loss def get_sequence_representation(self, outputs, attention_mask): """获取序列表示（内存优化版）""" # 只获取需要的隐藏状态层 hidden_states = outputs.hidden_states[self.repr_layer] # 获取每个序列的最后一个非填充token seq_lengths = attention_mask.sum(dim=1) - 1 batch_indices = torch.arange(hidden_states.size(0)) # 返回对应位置的隐藏状态 return hidden_states[batch_indices, seq_lengths] def compute_loss(self, model, inputs, return_outputs=False): """内存优化的损失计算""" # 确保模型处于训练模式 model.train() # 提取输入 anchor_ids = inputs["anchor_input_ids"] anchor_mask = inputs["anchor_attention_mask"] positive_ids = inputs["positive_input_ids"] positive_mask = inputs["positive_attention_mask"] negative_ids = inputs["negative_input_ids"] negative_mask = inputs["negative_attention_mask"] # 前向传播获取隐藏状态 def get_embeddings(input_ids, attention_mask): outputs = model( input_ids=input_ids, attention_mask=attention_mask, output_hidden_states=True, return_dict=True ) return self.get_sequence_representation(outputs, attention_mask) # 获取三元组的嵌入表示 anchor_emb = get_embeddings(anchor_ids, anchor_mask) pos_emb = get_embeddings(positive_ids, positive_mask) neg_emb = get_embeddings(negative_ids, negative_mask) # 计算对比损失 cl_loss = self.compute_contrastive_loss(anchor_emb, pos_emb, neg_emb) cl_loss = cl_loss * self.contrastive_weight # 关键修复：确保tokenizer已设置 if self.tokenizer is None: raise ValueError("Tokenizer未设置！") # 计算语言建模损失 lm_labels = positive_ids.clone() # 关键修复：使用tokenizer的pad_token_id pad_token_id = self.tokenizer.pad_token_id lm_labels[lm_labels == pad_token_id] = -100 # 计算语言建模损失 lm_outputs = model( input_ids=positive_ids, attention_mask=positive_mask, labels=lm_labels ) lm_loss = lm_outputs.loss # 总损失 = LM损失 + 对比损失 total_loss = lm_loss + cl_loss # 记录内存使用 print_memory_usage() return (total_loss, lm_outputs) if return_outputs else total_loss # ================ 主程序 ================ # if __name__ == "__main__": # 配置量化以减少内存使用 bnb_config = BitsAndBytesConfig( load_in_4bit=True, # 使用4位量化 bnb_4bit_quant_type="nf4", # 使用NF4量化类型 bnb_4bit_use_double_quant=True, # 双重量化 bnb_4bit_compute_dtype=torch.float16 # 计算使用FP16 ) # 加载模型和分词器（使用量化） model = AutoModelForCausalLM.from_pretrained( "model/Qwen/Qwen1.5-1.8B", quantization_config=bnb_config, # 应用量化配置 device_map="auto", # 自动选择设备 output_hidden_states=True, # 必须设置以获取隐藏状态 return_dict_in_generate=True, use_cache=False # 禁用缓存以节省内存 ) tokenizer = AutoTokenizer.from_pretrained("model/Qwen/Qwen1.5-1.8B") tokenizer.pad_token = tokenizer.eos_token # 设置填充token # 为量化模型添加LoRA适配器 lora_config = LoraConfig( r=8, lora_alpha=32, target_modules=["q_proj", "v_proj"], # 针对Qwen1.5-1.8B模型 lora_dropout=0.05, bias="none", task_type="CAUSAL_LM" ) # 关键修复：准备模型用于k位训练 model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=True) # 添加LoRA适配器 model = get_peft_model(model, lora_config) # 关键修复：显式启用LoRA参数的梯度 for param in model.parameters(): if param.requires_grad: param.requires_grad = True model.print_trainable_parameters() # 打印可训练参数数量 # 加载数据集 def load_and_tokenize_dataset(file_path, tokenizer): """加载数据集并进行分词处理""" # 加载原始数据集 dataset_dict = load_dataset('json', data_files=file_path) raw_dataset = dataset_dict['train'] # 应用分词函数 tokenized_dataset = raw_dataset.map( lambda ex: tokenize_function(ex, tokenizer, max_length=256), batched=True, batch_size=8, # 减小批处理大小 remove_columns=['anchor', 'positive', 'negative'] ) return tokenized_dataset train_dataset = load_and_tokenize_dataset('data/processed/train_style_triplets.json', tokenizer) val_dataset = load_and_tokenize_dataset('data/processed/val_style_triplets.json', tokenizer) # 验证数据集格式 print("训练集样本示例:", train_dataset[0]) print("验证集样本示例:", val_dataset[0]) # 训练参数配置（内存优化） training_args = TrainingArguments( output_dir="./model/lora_adapter", per_device_train_batch_size=1, # 减小批量大小 gradient_accumulation_steps=8, # 增加梯度累积步数 num_train_epochs=3, learning_rate=2e-4, logging_steps=10, # 更频繁的日志记录以监控内存 save_steps=500, fp16=True, report_to="none", remove_unused_columns=False, gradient_checkpointing=True, # 启用梯度检查点 optim="adafactor", # 使用内存更少的优化器 ) # 对比学习配置 contrastive_config = { "temperature": 0.07, "margin": 0.3, "weight": 0.8, "repr_layer": -1 } # 初始化数据收集器 data_collator = ContrastiveDataCollator( tokenizer=tokenizer, max_length=256, # 减少最大长度 padding="max_length" ) # 初始化训练器 - 关键修复：传递tokenizer trainer = ContrastiveTrainer( model=model, args=training_args, tokenizer=tokenizer, # 传递tokenizer data_collator=data_collator, train_dataset=train_dataset, eval_dataset=val_dataset, contrastive_config=contrastive_config ) # 开始训练前打印内存状态 print_memory_usage() # 关键修复：验证可训练参数 print("可训练参数列表:") for name, param in model.named_parameters(): if param.requires_grad: print(f"- {name}") # 开始训练 trainer.train() # 保存LoRA适配器 model.save_pretrained("./model/lora_adapter") # 评估模型 try: eval_results = trainer.evaluate() print("评估结果:", eval_results) except Exception as e: print(f"评估过程中发生错误: {e}") import traceback traceback.print_exc()

我们遇到了两个问题： 1. 在评估阶段出现了错误：ValueError: You must specify exactly one of input_ids or inputs_embeds 2. 在重新运行代码时出现了NameError: name 'Dataset' is not defined 第一个问题是...

import json import torch from typing import Dict, List from torch.utils.data import Dataset from collections import defaultdict import transformers from peft import LoraConfig, TaskType, get_peft_model from torch.utils.data import DataLoader, SequentialSampler from transformers import Trainer, TrainingArguments from lora_plus import LoraPlusTrainer from torch.utils.data import RandomSampler from swanlab.integration.transformers import SwanLabCallback import swanlab import numpy as np import pandas as pd import re from typing import Dict, List import torch from tqdm import tqdm from transformers import PreTrainedTokenizer from transformers import AutoTokenizer import torch.nn as nn from lora_plus import LoraPlusTrainer # 确保已安装lora_plus库 from transformers import PreTrainedModel # 新增的分子公式解析函数 def parse_chem_formula(formula): pattern = r'([A-Z][a-z]?)(\d*)' matches = re.findall(pattern, formula) element_counts = defaultdict(int) for (element, count) in matches: count = int(count) if count else 1 element_counts[element] += count return element_counts def generate_element_list(formula): element_counts = parse_chem_formula(formula) elements = [] for element, count in element_counts.items(): # 跳过氢元素 if element != "H": elements.extend([element] * count) return ''.join(elements) # 初始化SwanLab swanlab.init("Finetune-Llama3.2-with-Encoder") swanlab_callback = SwanLabCallback( project="Finetune-Llama3.2-with-Encoder", experiment_name="Finetune-Llama3.2-with-Encoder" ) # 常量定义 CHEM_FORMULA_SIZE = r"([A-Z][a-z]*)([0-9]*)" VALID_ELEMENTS = ["C", "N", "P", "O", "S", "Si", "I", "H", "Cl", "F", "Br", "B", "Se", "Fe", "Co", "As", "K", "Na"] element_to_idx = {elem: idx for idx, elem in enumerate(VALID_ELEMENTS)} # 化学式转密集向量 def formula_to_dense(chem_formula: str) -> torch.Tensor: dense_vec = torch.zeros(len(VALID_ELEMENTS), dtype=torch.float32) matches = re.findall(CHEM_FORMULA_SIZE, chem_formula) for chem_symbol, num_str in matches: num = 1 if num_str == "" else int(num_str) if chem_symbol in element_to_idx: idx = element_to_idx[chem_symbol] dense_vec[idx] += num return dense_vec # 位置编码生成 (PyTorch实现) def positional_encoding(max_position: int, d_model: int, min_freq: float = 1e-4) -> torch.Tensor: position = torch.arange(max_position).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2) * (-torch.log(torch.tensor(min_freq)) / d_model)) pos_enc = torch.zeros(max_position, d_model) pos_enc[:, 0::2] = torch.sin(position * div_term) pos_enc[:, 1::2] = torch.cos(position * div_term) return pos_enc # 初始化位置编码矩阵 P = positional_encoding(2000000, 254) dimn = 254 # 与位置编码维度一致 # 质谱数据编码 - 优化短数据处理：仅截断过长数据，不填充短数据 def encode_spectra(rag_tensor: list, P: torch.Tensor, dimn: int) -> list: # 返回列表而非堆叠张量 encoded_list = [] max_len = 501 # 仅对过长数据截断，不强制填充短数据 for sample in rag_tensor: mz_list, intensity_list = sample # 创建基础特征矩阵 [m/z, intensity] base_features = torch.tensor([mz_list, intensity_list], dtype=torch.float32).T # 添加位置编码特征（保留原始m/z的位置信息） pos_enc = torch.stack([P[min(int(mz), P.size(0)-1)] for mz in mz_list]) # 组合所有特征 [m/z, intensity, pos_enc...] features = torch.cat([base_features, pos_enc], dim=1) # 仅截断过长数据，短数据保持原始长度（不填充） if features.size(0) > max_len: features = features[:max_len] encoded_list.append(features) # 保留原始长度特征 return encoded_list # 质谱数据预处理 - 确保短数据完整保留 def preprocess_spectra(df: pd.DataFrame) -> list: spectra_list = [] for idx, row in tqdm(df.iterrows(), total=len(df)): spectrum_str = row['Spectrum'] total_mass = row['Total Exact Mass'] # 解析质谱字符串 pairs = spectrum_str.split() mz_list, intensity_list = [], [] for pair in pairs: mz, intensity = pair.split(':') mz_list.append(float(mz)) intensity_list.append(float(intensity)) # 对于仅含一组数据的情况，额外保留原始精度（不四舍五入） if len(pairs) == 1: # 保留原始精度，不进行四舍五入 mz_list = [float(mz) for mz, _ in [pair.split(':') for pair in pairs]] intensity_list = [float(intensity) for _, intensity in [pair.split(':') for pair in pairs]] # 添加总精确质量（作为补充特征，不影响原始数据长度） mz_list.append(total_mass) intensity_list.append(0.0) # 仅对长数据进行四舍五入，短数据保留更多精度 if len(mz_list) > 5: # 数据较长时才简化 mz_list = [round(mz, 2) for mz in mz_list] intensity_list = [round(intensity, 2) for intensity in intensity_list] spectra_list.append([mz_list, intensity_list]) return spectra_list class MolecularDataset(Dataset): def __init__(self, csv_path: str, tokenizer: AutoTokenizer, max_seq_len: int = 512): self.df = pd.read_csv(csv_path) self.tokenizer = tokenizer self.max_seq_len = max_seq_len self.pad_token_id = tokenizer.pad_token_id self.mask_token_id = tokenizer.mask_token_id if tokenizer.mask_token_id is not None else tokenizer.convert_tokens_to_ids("<mask>") # 预处理质谱数据（保留短数据原始长度） spectra_data = preprocess_spectra(self.df) self.spec_encoded = encode_spectra(spectra_data, P, dimn) # 现在是列表，每个元素为不同长度的张量 # 预处理分子公式为元素列表 self.element_lists = [generate_element_list(formula) for formula in self.df['Molecular Formula']] # 预计算element_list本身的token长度 self.element_lengths = [] for elem_list in self.element_lists: elem_tokens = self.tokenizer(elem_list, add_special_tokens=False)['input_ids'] self.element_lengths.append(len(elem_tokens)) def __len__(self): return len(self.df) def __getitem__(self, idx) -> dict: # 分子式向量和质谱矩阵（保留原始长度） formula = self.df.iloc[idx]['Molecular Formula'] formula_vec = formula_to_dense(formula).unsqueeze(0) spec_matrix = self.spec_encoded[idx] # 直接使用原始长度的特征矩阵 # 获取处理后的元素列表并添加标记 element_list = self.element_lists[idx] element_text = f"<｜User｜>~~<｜Spectrum｜>{element_list}~~" # SELFIES目标序列并添加标记 selfies_str = self.df.iloc[idx]['SELFIES'] selfies_text = f"<｜Assistant｜>~~{selfies_str}~~" # 组合输入：元素列表 + SELFIES序列 input_text = f"{element_text}{selfies_text}" # 关键修改：添加padding='max_length'，强制所有序列长度为max_seq_len encoding = self.tokenizer( input_text, add_special_tokens=False, max_length=self.max_seq_len, padding='max_length', # 强制填充到max_seq_len truncation=True, # 超过max_seq_len则截断 return_tensors='pt' ) # 输入序列（此时长度均为max_seq_len） input_ids = encoding['input_ids'].squeeze(0) attention_mask = encoding['attention_mask'].squeeze(0) # 标签为完整的目标序列（替换padding为-100） labels = input_ids.clone() labels[labels == self.pad_token_id] = -100 # 计算element部分在labels中的范围 element_len = self.element_lengths[idx] element_end = 3 + element_len if element_end < len(labels): labels[:element_end] = -100 # 仅保留SELFIES部分的标签 return { 'encoder1_inputs': formula_vec, 'encoder2_inputs': spec_matrix, # 原始长度特征 'input_ids': input_ids, 'attention_mask': attention_mask, 'labels': labels, } # 加载tokenizer tokenizer = AutoTokenizer.from_pretrained('/root/workspace/d21lv5s7v38s73b4ddlg/SELFIES/checkpoint-1280') # 确保mask token存在 if tokenizer.mask_token is None: tokenizer.add_special_tokens({"mask_token": "<mask>"}) # 确保pad token存在（如果不存在则添加） if tokenizer.pad_token is None: tokenizer.pad_token = tokenizer.eos_token # 用eos_token作为pad_token # 创建数据集 dataset = MolecularDataset('/root/workspace/d21lv5s7v38s73b4ddlg/SELFIES-SFT.csv', tokenizer) # 自定义collate函数：对批次内质谱数据进行动态填充（仅填充到批次最大长度） def custom_collator(features: List[Dict]) -> Dict: # 处理encoder1_inputs（固定形状，直接堆叠） encoder1_inputs = torch.stack([f['encoder1_inputs'] for f in features]) # 处理encoder2_inputs（可变长度，动态填充到批次最大长度） encoder2_inputs = [f['encoder2_inputs'] for f in features] # 仅在批次内填充到最长样本长度，短数据少填充 encoder2_padded = torch.nn.utils.rnn.pad_sequence( encoder2_inputs, batch_first=True, padding_value=0.0 # 填充值设为0（无信息） ) # 处理文本相关字段（此时长度均为max_seq_len，可直接stack） input_ids = torch.stack([f['input_ids'] for f in features]) attention_mask = torch.stack([f['attention_mask'] for f in features]) labels = torch.stack([f['labels'] for f in features]) return { 'encoder1_inputs': encoder1_inputs, 'encoder2_inputs': encoder2_padded, 'input_ids': input_ids, 'attention_mask': attention_mask, 'labels': labels, } class LlamaWithEncoder(PreTrainedModel): def __init__(self, base_model, encoder1_dim=18, encoder2_dim=256, hidden_dim=256): # 添加config属性 self.config = base_model.config super().__init__(self.config) # 存储基础模型 self.model = base_model # 第一个编码器：CNN + 简化Transformer（处理分子式向量） # 简单CNN层：1x1卷积提取特征 self.encoder1_cnn = nn.Conv1d( in_channels=encoder1_dim, out_channels=hidden_dim, kernel_size=1, stride=1 ) # 简化的Transformer编码器（仅1层） encoder1_layer = nn.TransformerEncoderLayer( d_model=hidden_dim, nhead=4, # 减少注意力头数 dim_feedforward=hidden_dim * 2, # 简化前馈网络 batch_first=True ) self.encoder1_transformer = nn.TransformerEncoder(encoder1_layer, num_layers=1) # 仅1层 # 第二个编码器：CNN + 简化Transformer（处理质谱矩阵） # 简单CNN层：提取局部特征 self.encoder2_cnn = nn.Sequential( nn.Conv1d( in_channels=encoder2_dim, out_channels=hidden_dim, kernel_size=3, stride=1, padding=1 ), nn.ReLU(), nn.MaxPool1d(kernel_size=2, stride=2) # 降采样 ) # 简化的Transformer编码器（仅1层） encoder2_layer = nn.TransformerEncoderLayer( d_model=hidden_dim, nhead=4, # 减少注意力头数 dim_feedforward=hidden_dim * 2, # 简化前馈网络 batch_first=True ) self.encoder2_transformer = nn.TransformerEncoder(encoder2_layer, num_layers=1) # 仅1层 # 投影层：将编码器输出映射到模型隐藏层维度 self.proj1 = nn.Linear(hidden_dim, base_model.config.hidden_size) self.proj2 = nn.Linear(hidden_dim, base_model.config.hidden_size) # 嵌入层（复制基础模型权重但不共享） self.embed_tokens = nn.Embedding( num_embeddings=base_model.config.vocab_size, embedding_dim=base_model.config.hidden_size, padding_idx=base_model.config.pad_token_id ) self.embed_tokens.weight.data = base_model.get_input_embeddings().weight.data.clone() # PEFT所需方法 def get_input_embeddings(self): return self.embed_tokens def set_input_embeddings(self, value): self.embed_tokens = value def get_output_embeddings(self): return self.model.get_output_embeddings() def set_output_embeddings(self, new_embeddings): self.model.set_output_embeddings(new_embeddings) def get_base_model(self): return self.model def forward( self, input_ids=None, attention_mask=None, encoder1_inputs=None, encoder2_inputs=None, labels=None, past_key_values=None, output_attentions=None, output_hidden_states=None, return_dict=None,** kwargs ): # 1. 编码器处理（支持可变长度输入） # 分子式编码器：CNN + Transformer batch_size = encoder1_inputs.size(0) enc1 = encoder1_inputs.permute(0, 2, 1) # (batch_size, encoder1_dim, seq_len) enc1 = self.encoder1_cnn(enc1) # (batch_size, hidden_dim, seq_len) enc1 = enc1.permute(0, 2, 1) # (batch_size, seq_len, hidden_dim) enc1_out = self.encoder1_transformer(enc1) # (batch_size, seq_len, hidden_dim) enc1_out = enc1_out.mean(dim=1) # (batch_size, hidden_dim) enc1_proj = self.proj1(enc1_out) # (batch_size, hidden_size) # 质谱编码器：CNN + Transformer enc2 = encoder2_inputs.permute(0, 2, 1) # (batch_size, encoder2_dim, seq_len) enc2 = self.encoder2_cnn(enc2) # (batch_size, hidden_dim, seq_len/2) enc2 = enc2.permute(0, 2, 1) # (batch_size, seq_len/2, hidden_dim) enc2_out = self.encoder2_transformer(enc2) # (batch_size, seq_len/2, hidden_dim) enc2_out = enc2_out.mean(dim=1) # (batch_size, hidden_dim) enc2_proj = self.proj2(enc2_out) # (batch_size, hidden_size) # 合并编码器输出（用于替换<mask>） mask_replacement = (enc1_proj + enc2_proj) / 2 # (batch_size, hidden_size) # 2. 获取原始嵌入 embeddings = self.embed_tokens(input_ids) # (batch_size, seq_len, hidden_size) batch_size, seq_len, hidden_size = embeddings.size() # 3. 替换<mask> token（第三个token，索引=2） if seq_len > 2: mask_embed = mask_replacement.unsqueeze(1) # (batch_size, 1, hidden_size) # 拆分张量并拼接（避免inplace操作） part1 = embeddings[:, :2, :] # (batch_size, 2, hidden_size) part2 = mask_embed # (batch_size, 1, hidden_size) part3 = embeddings[:, 3:, :] # (batch_size, seq_len-3, hidden_size) new_embeddings = torch.cat([part1, part2, part3], dim=1) # (batch_size, seq_len, hidden_size) else: new_embeddings = embeddings # 序列过短时直接使用原始嵌入 # 5. 调用基础模型 return self.model( inputs_embeds=new_embeddings, attention_mask=attention_mask, labels=labels, past_key_values=past_key_values, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, ) # 加载预训练模型 base_model = transformers.AutoModelForCausalLM.from_pretrained( "/root/workspace/d21lv5s7v38s73b4ddlg/SELFIES/checkpoint-1280", trust_remote_code=True, torch_dtype=torch.bfloat16, ) model = LlamaWithEncoder(base_model) lora_config = LoraConfig( r=8, lora_alpha=16, target_modules="all-linear", # 目标注意力层 lora_dropout=0.0, bias="none", task_type="CAUSAL_LM" ) model = get_peft_model(model, lora_config) model.print_trainable_parameters() # 输出可训练参数比例 training_args = TrainingArguments( output_dir="./llama3.2-SELFIES-SFT", per_device_train_batch_size=24, gradient_accumulation_steps=8, num_train_epochs=6, learning_rate=5.0e-05, optim="adamw_torch", logging_steps=10, bf16=True, save_strategy="steps", lr_scheduler_type='cosine', max_grad_norm=1.0, save_steps=2000, warmup_steps=0 ) class CustomTrainer(LoraPlusTrainer): def get_train_dataloader(self) -> DataLoader: return DataLoader( self.train_dataset, batch_size=self.args.train_batch_size, shuffle=True, collate_fn=self.data_collator, drop_last=False, ) # 使用修改后的 CustomTrainer lp_trainer = CustomTrainer( model, training_args, train_dataset=dataset, tokenizer=tokenizer, data_collator=custom_collator, callbacks=[swanlab_callback], ) lp_trainer.train() lp_trainer.save_model(output_dir='./llama3.2-SELFIES-SFT') # 合并LoRA权重 model = model.merge_and_unload() # 保存整个模型（包括自定义编码器和融合层）为safetensors格式 save_directory = './llama3.2-SELFIES' model.save_pretrained(save_directory, safe_serialization=True) # 同时保存tokenizer tokenizer.save_pretrained(save_directory)修改代码，改为使用这个获取的模型根据csv文件进行批量推理的代码，并将csv文件的SELFIES和对应的生成SELFIES保存为同一行

我们有两个任务： 1. 训练代码已经完成，现在需要编写一个推理脚本，使用训练好的模型对CSV文件中的数据进行批量推理。 2. 推理过程中，我们读取CSV文件中的每一行数据（包括质谱数据、分子式等），使用模型生成...

以上代码出现问题：(style_tune) C:\Users\28996\Desktop\AI\persona_contrastive_finetuning>python Contrastive_Training_LM.py Generating train split: 2 examples [00:00, 2.15 examples/s] Map: 100%|████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 71.39 examples/s] Generating train split: 2 examples [00:00, 252.61 examples/s] Map: 100%|███████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 399.72 examples/s] 训练集样本示例: {'anchor_input_ids': [56568, 118919, 116122, 11319], 'positive_input_ids': [116122, 20412, 107340, 9370, 100357, 102323, 3837, 109202, 104078, 103975, 100675, 101940, 100912, 105054, 6313], 'negative_input_ids': [100323, 104307, 99245, 9370, 106059, 104060, 3837, 104530, 115604, 99329, 11319]} 验证集样本示例: {'anchor_input_ids': [56568, 118919, 116122, 11319], 'positive_input_ids': [116122, 20412, 107340, 9370, 100357, 102323, 3837, 109202, 104078, 103975, 100675, 101940, 100912, 105054, 6313], 'negative_input_ids': [100323, 104307, 99245, 9370, 106059, 104060, 3837, 104530, 115604, 99329, 11319]} 0%| | 0/3 [00:00<?, ?it/s]You're using a Qwen2TokenizerFast tokenizer. Please note that with a fast tokenizer, using the __call__ method is faster than using a method to encode the text followed by a call to the pad method to get a padded encoding. Traceback (most recent call last): File "C:\Users\28996\Desktop\AI\persona_contrastive_finetuning\Contrastive_Training_LM.py", line 290, in <module> trainer.train() File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\transformers\trainer.py", line 2171, in train return inner_training_loop( File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\transformers\trainer.py", line 2531, in _inner_training_loop tr_loss_step = self.training_step(model, inputs, num_items_in_batch) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\transformers\trainer.py", line 3676, in training_step loss = self.compute_loss(model, inputs) File "C:\Users\28996\Desktop\AI\persona_contrastive_finetuning\Contrastive_Training_LM.py", line 173, in compute_loss anchor_emb = get_embeddings(anchor_ids, anchor_mask) File "C:\Users\28996\Desktop\AI\persona_contrastive_finetuning\Contrastive_Training_LM.py", line 164, in get_embeddings outputs = model( File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\torch\nn\modules\module.py", line 1736, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\torch\nn\modules\module.py", line 1747, in _call_impl return forward_call(*args, **kwargs) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\accelerate\utils\operations.py", line 818, in forward return model_forward(*args, **kwargs) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\accelerate\utils\operations.py", line 806, in __call__ return convert_to_fp32(self.model_forward(*args, **kwargs)) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\accelerate\utils\operations.py", line 785, in convert_to_fp32 return recursively_apply(_convert_to_fp32, tensor, test_type=_is_fp16_bf16_tensor) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\accelerate\utils\operations.py", line 118, in recursively_apply { File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\accelerate\utils\operations.py", line 119, in <dictcomp> k: recursively_apply( File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\accelerate\utils\operations.py", line 126, in recursively_apply return func(data, *args, **kwargs) File "C:\Users\28996\miniconda3\envs\style_tune\lib\site-packages\accelerate\utils\operations.py", line 777, in _convert_to_fp32 return tensor.float() torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 594.00 MiB. GPU 0 has a total capacity of 8.00 GiB of which 0 bytes is free. Of the allocated memory 13.03 GiB is allocated by PyTorch, and 129.95 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) 0%| | 0/3 [00:15<?, ?it/s]

batch_indices = torch.arange(hidden_states.size(0)) # 释放不需要的中间变量 del outputs clear_memory() return hidden_states[batch_indices, seq_lengths] def compute_loss(self, model, inputs, ...

mistral 7b的sliding window attention是如何实现的，找到官方的原代码

attention_mask: Optional[torch.Tensor] = None, # 其他参数 ): # 计算query, key, value # ... # 应用滑动窗口注意力掩码 if self.sliding_window is not None: # 生成滑动窗口掩码 sliding_mask = ...

按照上述步骤，将修改好的完整代码发我

def forward(self, predictions, targets): total_loss = 0 # 1. 重量损失 - 分层处理（使用归一化后的阈值） for name in ['product_weight', 'package_weight']: # 小值样本 (0-50) 归一化后为 [0, 0.05] ...

根据虹软实现的人脸检测、追踪、识别、年龄检测、性别检测的JAVA解决方案

打开下面链接，直接免费下载资源： https://renmaiwang.cn/s/vxfyv (最新版、最全版本)根据虹软实现的人脸检测、追踪、识别、年龄检测、性别检测的JAVA解决方案

matlab YALMIP、GLPK安装资源

matlab的YALMIP、GLPK安装包，内置YALMIP、GLPK，直接将分别其添加到matlab的toolbox、路径中即可（matlab主页-设置路径-添加并包含子文件夹-YALMIP；matlab主页-设置路径-添加文件夹-github_repo）

【scratch3.0少儿编程-游戏原型-动画-项目源码】打砖块.zip

资源说明： 1：本资料仅用作交流学习参考，请切勿用于商业用途。 2：一套精品实用scratch3.0少儿编程游戏、动画源码资源，无论是入门练手还是项目复用都超实用，省去重复开发时间，让开发少走弯路！更多精品资源请访问 https://blog.csdn.net/ashyyyy/article/details/146464041

使用 OpenCV 技术实现人脸检测的方法与过程

打开下面链接，直接免费下载资源： https://renmaiwang.cn/s/o7o7f 运用 OpenCV 这一计算机视觉库来开展人脸检测相关的操作

随你记微信小程序_专为学生群体设计的便捷收支管理工具_提供快速记录日常开销与收入的功能_支持多维度数据可视化分析_帮助用户清晰掌握个人财务状况_培养理性消费习惯_无需下载安装即用即.zip

随你记微信小程序_专为学生群体设计的便捷收支管理工具_提供快速记录日常开销与收入的功能_支持多维度数据可视化分析_帮助用户清晰掌握个人财务状况_培养理性消费习惯_无需下载安装即用即.zip

一个基于PyQt5的实时人脸识别系统，使用MTCNN进行人脸检测和FaceNet进行人脸特征提取，支持人脸注册、识别和管理功能

打开下面链接，直接免费下载资源： https://renmaiwang.cn/s/2u8a8 (最新版、最全版本)一个基于PyQt5的实时人脸识别系统，使用MTCNN进行人脸检测和FaceNet进行人脸特征提取，支持人脸注册、识别和管理功能。

运用 opencv 与 tensorflow 实现实时人脸检测的方法

打开下面链接，直接免费下载资源： https://renmaiwang.cn/s/sxvdc 运用 opencv 与 tensorflow 这两种工具来开展实时性的人脸检测工作

这是一个使用 OpenCV 实现的视频人脸检测程序程序可以实时检测视频中的正面人脸和侧脸，并在检测到的人脸上绘制识别框和置信度支持从本地摄像头、视频文件或网络视频流中读取视频

打开下面链接，直接免费下载资源： https://renmaiwang.cn/s/igxjk (最新版、最全版本)这是一个使用 OpenCV 实现的视频人脸检测程序。程序可以实时检测视频中的正面人脸和侧脸，并在检测到的人脸上绘制识别框和置信度。支持从本地摄像头、视频文件或网络视频流中读取视频。

毕业设计.zip

毕业设计.zip

Excel高级使用技巧汇总

博客汇总了Excel高级使用技巧，涵盖编辑、合并、条件显示、自定义格式等多方面。如编辑技巧包含分数、序列输入方法；单元格合并可借助公式；还介绍了绘制函数图像、矩阵计算等操作，同时提及避免错误信息、宏应用等内容及常见问题解决办法。

基于OpenMV的智能小车控制项目源码与实现方案

该资料包含个人完成的课程与毕业设计项目，所有代码均经过验证并成功运行，确保功能正常后才进行发布，整体......

相关推荐

one hot编码：torch.Tensor.scatter_()函数用法详解

PyTorch中torch.tensor与torch.Tensor的区别详解

pytorch中torch.max和Tensor.view函数用法详解

mistral 7b的sliding window attention是如何实现的，找到官方的原代码

按照上述步骤，将修改好的完整代码发我

根据虹软实现的 人脸检测、追踪、识别、年龄检测、性别检测 的JAVA解决方案

matlab YALMIP、GLPK安装资源

【scratch3.0少儿编程-游戏原型-动画-项目源码】打砖块.zip

使用 OpenCV 技术实现人脸检测的方法与过程

随你记微信小程序_专为学生群体设计的便捷收支管理工具_提供快速记录日常开销与收入的功能_支持多维度数据可视化分析_帮助用户清晰掌握个人财务状况_培养理性消费习惯_无需下载安装即用即.zip

一个基于PyQt5的实时人脸识别系统，使用MTCNN进行人脸检测和FaceNet进行人脸特征提取，支持人脸注册、识别和管理功能

运用 opencv 与 tensorflow 实现实时人脸检测的方法

这是一个使用 OpenCV 实现的视频人脸检测程序 程序可以实时检测视频中的正面人脸和侧脸，并在检测到的人脸上绘制识别框和置信度 支持从本地摄像头、视频文件或网络视频流中读取视频

毕业设计.zip

Excel高级使用技巧汇总

基于OpenMV的智能小车控制项目源码与实现方案

大家在看

libssl-1_1-x64.zip

JSP SQLServer 网上购物商城 毕业论文

FolderSniffer3.51 找回因文件夹加密而丢失的文件

印能捷安装及设置教程

Dicom格式医学图像

最新推荐

根据虹软实现的 人脸检测、追踪、识别、年龄检测、性别检测 的JAVA解决方案

matlab YALMIP、GLPK安装资源

【scratch3.0少儿编程-游戏原型-动画-项目源码】打砖块.zip

使用 OpenCV 技术实现人脸检测的方法与过程

随你记微信小程序_专为学生群体设计的便捷收支管理工具_提供快速记录日常开销与收入的功能_支持多维度数据可视化分析_帮助用户清晰掌握个人财务状况_培养理性消费习惯_无需下载安装即用即.zip

Docker环境下的弹性APM服务器搭建指南

游戏开发与部署全流程指南

初级运维面试题

构建Ikiwiki的Docker容器：简易部署与使用

Unity开发实用指南：快捷键、外部工具与模型创建

根据虹软实现的人脸检测、追踪、识别、年龄检测、性别检测的JAVA解决方案

这是一个使用 OpenCV 实现的视频人脸检测程序程序可以实时检测视频中的正面人脸和侧脸，并在检测到的人脸上绘制识别框和置信度支持从本地摄像头、视频文件或网络视频流中读取视频

JSP SQLServer 网上购物商城毕业论文

根据虹软实现的人脸检测、追踪、识别、年龄检测、性别检测的JAVA解决方案