网站建设要架服务器,九龙坡网站建设公司,青岛做优化网站哪家好,wordpress pdf view目录 问题描述#xff1a;
问题解决#xff1a; 问题描述#xff1a;
随着各种LLMs (Large Language Models#xff09;的出现#xff0c;如何调用各种LLMs的api成为了经常会遇见的问题。
问题解决#xff1a;
下面仅以生成给定sentence的复述句为例#xff0c;说明…目录 问题描述
问题解决 问题描述
随着各种LLMs (Large Language Models的出现如何调用各种LLMs的api成为了经常会遇见的问题。
问题解决
下面仅以生成给定sentence的复述句为例说明如何应用。
data: 存放在.txt文件中按行存放每一行是一个句子
⚠️ 只能在本地运行不可以在服务器上运行并且要打开。
代码实现
import openai
import time
import os
import random from tqdm import tqdm
# openai.api_key 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
# openai.api_key sk-huUlpXTcDnvNevXgB1399aCa9dB247F599D81737B7CfD927 # 实验室共有openai.api_key YOUR API KEY # GPT4.0
openai.api_base https://api.ngapi.top/v1def get_response(prompt, temperature0.5, max_tokens2048):completion openai.ChatCompletion.create(# modelgpt-3.5-turbo,modelgpt-4,temperature0,top_p0,# max_tokensmax_tokens,messages[{role: user, content: f{prompt}}])return completion
def generate_prompt(all_sentences): # all_asps是一个listprompts []for cur_sent in all_sentences: # all_compared_pair是一个list:[asp,sub,obj]# prompt Please generate just one synonymous sentence for the given sentence. The current given sentence is : {}.format(cur_sent)prompt fGenerate a paraphrased version of the following sentence: {cur_sent}prompts.append(prompt)return promptsimport redef process_sentence(sentence):# 定义要添加空格的特殊字符, 比如’sspecial_chars [,, ., \, ’, “, ”, (, ), [, ], {, }, :, ;, ?, !] # - 因为sub,obj中存在很多以-为连字符的sub,obj所以原始句子中这部分不可以加空格# 在特殊字符前添加空格for char in special_chars:if char (: # 特别的左括号是在后面加空格sentence sentence re.sub(rf([{char}]), r\1 , sentence)else:sentence re.sub(rf([{char}]), r \1, sentence)return sentencedef get_res(all_sentences, wirte_path):if os.path.exists(wirte_path):res_file open(wirte_path)begin len(res_file.readlines()) 1 # 断点续传res_file.close()else:begin 0with open(wirte_path, a, buffering1) as res_file:count 0for p in tqdm(generate_prompt(all_sentences), totallen(all_sentences)):print(p)count 1if count begin:continuetry:res get_response(p)[choices][0][message][content]# print(res, res)processed_res process_sentence(res) # 处理成以单词标点符号等为分隔符空格# print(processed_res, processed_res)if The server is overloaded or not ready yet in processed_res:time.sleep(30)res_file.write(#### \n)else:res_file.write(processed_res \n)except Exception as e:if isinstance(e, openai.error.RateLimitError):time.sleep(30) # 处理拥挤的函数就只是睡着而已res get_response(p)[choices][0][message][content]processed_res process_sentence(res) # res_file.write(processed_res \n)else:print(e)exit(-1)# print(the number of prompt is :, count)
from pdb import set_trace as stopdataset_list [14lap,14res, 15res, 16res]
name_list [train, test, dev]# name_list [dev]
# dataset_list [14lap]for dataset in dataset_list:for name in name_list:# print(dataste, dataset)# print(name, name)sentence_path f/Users/qtxu/Workplace/Chatgpt/ATE_Task/{dataset}/{name}_sentence.txtwrite_path f/Users/qtxu/Workplace/Chatgpt/ATE_Task/{dataset}/{name}_paraphrase_sentence.txtwith open(sentence_path,r) as fr:lines fr.readlines()all_sentences []for line in lines:all_sentences.append(line)# print(all_sentences, all_sentences)get_res(all_sentences, write_path)
