--- base_model: llm-jp/llm-jp-3-13b tags: - text-generation-inference - transformers - unsloth - llama - trl license: apache-2.0 language: - en --- # Uploaded model - **Developed by:** nagayaoh - **License:** apache-2.0 - **Finetuned from model :** llm-jp/llm-jp-3-13b This llama model was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth) and Huggingface's TRL library. [](https://github.com/unslothai/unsloth) --- # Introduction This is a result of LLM2024 competition on UTokyo seminor. * Dashboard score is 3.05 ## Including * Fine-tuned Model * README (this file) # Dataset This model used the following dataset for fine tuning only. | Language | Dataset | License | Description | |:---|:---|:---|:---| |Japanese| ichikara-instruction-003-001-1.json| CC-BY-NC-SA | [ ichikara-instruction: LLMのための日本語インストラクションデータ ](https://liat-aip.sakura.ne.jp/wp/llm%E3%81%AE%E3%81%9F%E3%82%81%E3%81%AE%E6%97%A5%E6%9C%AC%E8%AA%9E%E3%82%A4%E3%83%B3%E3%82%B9%E3%83%88%E3%83%A9%E3%82%AF%E3%82%B7%E3%83%A7%E3%83%B3%E3%83%87%E3%83%BC%E3%82%BF%E4%BD%9C%E6%88%90/llm%E3%81%AE%E3%81%9F%E3%82%81%E3%81%AE%E6%97%A5%E6%9C%AC%E8%AA%9E%E3%82%A4%E3%83%B3%E3%82%B9%E3%83%88%E3%83%A9%E3%82%AF%E3%82%B7%E3%83%A7%E3%83%B3%E3%83%87%E3%83%BC%E3%82%BF-%E5%85%AC%E9%96%8B/) | # How to build this model You can use Google colab in T4 runtime: * It takes about 18 - 40 minutes * If possible, strongly recommend you followings: * Use `{model|data}.to('cuda')` to shorten your learning duration * Use A100 * I used following code (removed): * Evalution on Learning to avoid over-learning * WandB to check parameter tuning ## Code ## This is ipynb code ``` # -*- coding: utf-8 -*- # !!! paste below code as ipynb in Google Colab # =============================================================================================== # --- Install Python Packages !pip uninstall unsloth -y !pip install --upgrade --no-cache-dir "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git" -qU !pip install --upgrade torch torchvision torchaudio -qU !pip install --upgrade xformers -qU # Install Flash Attention 2 for softcapping support import torch if torch.cuda.get_device_capability()[0] >= 8: !pip install --no-deps packaging ninja einops "flash-attn>=2.6.3" -qU # =============================================================================================== # --- Parameter definition device = "cuda" if torch.cuda.is_available() else "cpu" # --- Put data files according to path definition proj_path = "/content" input_path = proj_path + "/input/ichikara-instruction-003-001-1.json" eval_path = proj_path + "/eval/elyza-tasks-100-TV_0.jsonl" result_path = proj_path + "/results" # =============================================================================================== # Setting Parameters model_id = "llm-jp/llm-jp-3-13b" new_model_id = "llm-jp-3-13b-it" # Adaptor name dtype = None # None is OK + load_in_4bit = True # True for 13B model max_seq_length = 512 # Any length can be used because of RoPE # パラメータをPack config={ "model_id": model_id, "learning_rate": 2e-5, "per_device_train_batch_size": 4, "gradient_accumulation_steps": 4, "num_train_epochs":3, "warmup_steps": 10, "max_steps": -1, "lora_r": 32, "lora_alpha": 32, "lora_dropout": 0.05, "lora_bias": "none", "lora_use_rslora": False, "lora_loftq_config": None, "model_max_seq_length": max_seq_length, "model_dtype": dtype, "model_load_in_4bit": load_in_4bit, "seed": 3407, "max_seq_length": max_seq_length } # =============================================================================================== # Load llm-jp/llm-jp-3-13b as 4bit-quantized qLoRA from unsloth import FastLanguageModel model, tokenizer = FastLanguageModel.from_pretrained( model_name = config.model_id, dtype = config.model_dtype, load_in_4bit = config.model_load_in_4bit, trust_remote_code= True, ) model = FastLanguageModel.get_peft_model( model, target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj",], r = config.lora_r, lora_alpha = config.lora_alpha, lora_dropout = config.lora_dropout, bias = config.lora_bias, use_rslora = config.lora_use_rslora, loftq_config = config.lora_loftq_config, max_seq_length = config.max_seq_length, use_gradient_checkpointing = "unsloth", random_state = config.seed, ) # =============================================================================================== # Load dataset and spilit into train and test from datasets import load_dataset train_dataset = load_dataset("json", data_files= input_path, split="train[:80%]" ) test_dataset = load_dataset("json", data_files= input_path, split="train[80%:]") # Formatting in prompt style EOS_TOKEN = tokenizer.eos_token prompt = f"""### 指示\n{input}\n### 回答\n""" def formatting_prompts_func(examples): input = examples["text"] output = examples["output"] text = prompt.format(input, output) + EOS_TOKEN return { "formatted_text" : text, } # Retrun new field "formatted_text" # Assign Prompt style train_dataset = train_dataset.map( formatting_prompts_func, num_proc= 4 ) test_dataset = test_dataset.map( formatting_prompts_func, num_proc= 4 ) # =============================================================================================== from trl import SFTTrainer from transformers import TrainingArguments, EarlyStoppingCallback from unsloth import is_bfloat16_supported # EarlyStoppingCallback early_stopping_callback = EarlyStoppingCallback( early_stopping_patience = 3, early_stopping_threshold = 0.0 ) trainer = SFTTrainer( model = model, tokenizer = tokenizer, train_dataset = train_dataset, eval_dataset = test_dataset, max_seq_length = config.max_seq_length, dataset_text_field = "formatted_text", packing = False, callbacks=[early_stopping_callback], args = TrainingArguments( per_device_train_batch_size = config.per_device_train_batch_size, gradient_accumulation_steps = config.gradient_accumulation_steps, num_train_epochs = config.num_train_epochs, warmup_steps = config.warmup_steps, max_steps = config.max_steps, learning_rate = config.learning_rate, seed = config.seed, evaluation_strategy = "steps", eval_steps = 20, save_strategy = "steps", save_steps = 60, save_total_limit = 3, load_best_model_at_end = True, metric_for_best_model = "eval_loss", greater_is_better = False, output_dir = "outputs", report_to = "wandb", fp16 = not is_bfloat16_supported(), bf16 = is_bfloat16_supported(), group_by_length = True, logging_steps = 10, ), ) # =============================================================================================== # Trainning trainer_stats = trainer.train() # =============================================================================================== # Load Elyza-100 tasks import json eval_datasets = [] elyza_tasks_path = eval_path with open(elyza_tasks_path, "r") as f: item = "" for line in f: line = line.strip() item += line if item.endswith("}"): eval_datasets.append(json.loads(item)) item = "" # Do tasks from tqdm import tqdm FastLanguageModel.for_inference(model) model.eval() results = [] for dt in tqdm(eval_datasets): input = dt["input"] prompt = f"""### 指示\n{input}\n### 回答\n""" inputs = tokenizer([prompt], return_tensors = "pt") outputs = model.generate(**inputs, max_new_tokens = 512, use_cache = True, do_sample=False, repetition_penalty=1.2) prediction = tokenizer.decode(outputs[0], skip_special_tokens=True).split('\n### 回答')[-1] results.append({"task_id": dt["task_id"], "input": input, "output": prediction}) # =============================================================================================== # Save result as jsonl jsonl_result_path = result_path + f"/{new_model_id}_output.jsonl", with open( jsonl_result_path, 'w', encoding='utf-8') as f: for result in results: json.dump(result, f, ensure_ascii=False) f.write('\n') # Send LoRA adaptors into HugginFace modelcard page HF_TOKEN = "Please replace here by hugging face Access token" model.push_to_hub_merged( new_model_id+"_lora", tokenizer=tokenizer, save_method="lora", token=HF_TOKEN, private=False ) ```