魔搭社区每周速递（1.5-1.18）

!pip install vector-quantize-pytorch==1.18.5 !pip install vocos==0.1.0!pip install transformers==4.44.2

import torchfrom PIL import Imagefrom modelscope import AutoModel, AutoTokenizer
# load omni model default, the default init_vision/init_audio/init_tts is True# if load vision-only model, please set init_audio=False and init_tts=False# if load audio-only model, please set init_vision=Falsemodel = AutoModel.from_pretrained(    'openbmb/MiniCPM-o-2_6',    trust_remote_code=True,    attn_implementation='sdpa', # sdpa or flash_attention_2    torch_dtype=torch.bfloat16,    init_vision=True,    init_audio=True,    init_tts=True)

model = model.eval().cuda()tokenizer = AutoTokenizer.from_pretrained('OpenBMB/MiniCPM-o-2_6', trust_remote_code=True)
# In addition to vision-only mode, tts processor and vocos also needs to be initializedmodel.init_tts()model.tts.float()import mathimport numpy as npfrom PIL import Imagefrom moviepy.editor import VideoFileClipimport tempfileimport librosaimport soundfile as sf
def get_video_chunk_content(video_path, flatten=True):    video = VideoFileClip(video_path)    print('video_duration:', video.duration)
    with tempfile.NamedTemporaryFile(suffix=".wav", delete=True) as temp_audio_file:        temp_audio_file_path = temp_audio_file.name        video.audio.write_audiofile(temp_audio_file_path, codec="pcm_s16le", fps=16000)        audio_np, sr = librosa.load(temp_audio_file_path, sr=16000, mono=True)    num_units = math.ceil(video.duration)
    # 1 frame + 1s audio chunk    contents= []    for i in range(num_units):        frame = video.get_frame(i+1)        image = Image.fromarray((frame).astype(np.uint8))        audio = audio_np[sr*i:sr*(i+1)]        if flatten:            contents.extend(["<unit>", image, audio])        else:            contents.append(["<unit>", image, audio])
    return contents
video_path="/mnt/workspace/video.mp4"sys_msg = model.get_sys_prompt(mode='omni', language='en')# if use voice clone prompt, please set ref_audio# ref_audio_path = '/path/to/ref_audio'# ref_audio, _ = librosa.load(ref_audio_path, sr=16000, mono=True)# sys_msg = model.get_sys_prompt(ref_audio=ref_audio, mode='omni', language='en')
contents = get_video_chunk_content(video_path)msg = {"role":"user", "content": contents}msgs = [sys_msg, msg]
# please set generate_audio=True and output_audio_path to save the tts resultgenerate_audio = Trueoutput_audio_path = '/mnt/workspace/4.wav'
res = model.chat(    msgs=msgs,    tokenizer=tokenizer,    sampling=True,    temperature=0.5,    max_new_tokens=4096,    omni_input=True, # please set omni_input=True when omni inference    use_tts_template=True,    generate_audio=generate_audio,    output_audio_path=output_audio_path,    max_slice_nums=1,    use_image_id=False,    return_dict=True)print(res)

import torchfrom modelscope import AutoTokenizer, AutoModelForCausalLM
model_dir = "Shanghai_AI_Laboratory/internlm3-8b-instruct"#model = AutoModelForCausalLM(model_dir, trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)# Set `torch_dtype=torch.float16` to load model in float16, otherwise it will be loaded as float32 and might cause OOM Error.model = AutoModelForCausalLM.from_pretrained(model_dir, trust_remote_code=True, torch_dtype=torch.float16)# (Optional) If on low resource devices, you can load model in 4-bit or 8-bit to further save GPU memory via bitsandbytes.  # InternLM3 8B in 4bit will cost nearly 8GB GPU memory.  # pip install -U bitsandbytes  # 8-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_8bit=True)  # 4-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_4bit=True)model = model.eval()model = model.cuda()

system_prompt = """You are an AI assistant whose name is InternLM (书生·浦语).- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.- InternLM (书生·浦语) can understand and communicate fluently in the language chosen by the user such as English and 中文."""messages = [    {"role": "system", "content": system_prompt},    {"role": "user", "content": "Please tell me five scenic spots in Shanghai"}, ]tokenized_chat = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").cuda()
generated_ids = model.generate(tokenized_chat, max_new_tokens=1024, temperature=1, repetition_penalty=1.005, top_k=40, top_p=0.8)
generated_ids = [    output_ids[len(input_ids):] for input_ids, output_ids in zip(tokenized_chat, generated_ids)].cuda()response = tokenizer.batch_decode(generated_ids)[0]print(response)

!wget "https://modelscope.oss-cn-beijing.aliyuncs.com/resource/cloudflared-linux-amd64.deb"!dpkg -i cloudflared-linux-amd64.deb
!git clone https://www.modelscope.cn/AI-ModelScope/ComfyUI-MajicFlus.git
%cd /mnt/workspace/ComfyUI-MajicFlusimport subprocessimport threadingimport timeimport socketimport urllib.request
def iframe_thread(port):  while True:      time.sleep(0.5)      sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)      result = sock.connect_ex(('127.0.0.1', port))      if result == 0:        break      sock.close()  print("\nComfyUI finished loading, trying to launch cloudflared (if it gets stuck here cloudflared is having issues)\n")
  p = subprocess.Popen(["cloudflared", "tunnel", "--url", "http://127.0.0.1:{}".format(port)], stdout=subprocess.PIPE, stderr=subprocess.PIPE)  for line in p.stderr:    l = line.decode()    if "trycloudflare.com " in l:      print("This is the URL to access ComfyUI:", l[l.find("http"):], end='')    #print(l, end='')

threading.Thread(target=iframe_thread, daemon=True, args=(8188,)).start()
!python main.py --dont-print-server

<|im_start|>system<|im_sep|>You are a medieval knight and must provide explanations to modern people.<|im_end|><|im_start|>user<|im_sep|>How should I explain the Internet?<|im_end|><|im_start|>assistant<|im_sep|>

import transformersfrom modelscope import snapshot_download
model_dir = snapshot_download("LLM-Research/phi-4")
pipeline = transformers.pipeline(    "text-generation",    model=model_dir,    model_kwargs={"torch_dtype": "auto"},    device_map="auto",)
messages = [    {"role": "system", "content": "You are a medieval knight and must provide explanations to modern people."},    {"role": "user", "content": "How should I explain the Internet?"},]
outputs = pipeline(messages, max_new_tokens=128)print(outputs[0]["generated_text"][-1])

conda create -n memo python=3.10 -yconda activate memoconda install -c conda-forge ffmpeg -ypip install -e .

python inference.py --config configs/inference.yaml --input_image <IMAGE_PATH> --input_audio <AUDIO_PATH> --output_dir <SAVE_PATH>

python inference.py --config configs/inference.yaml --input_image assets/examples/dicaprio.jpg --input_audio assets/examples/speech.wav --output_dir outputs

from valley_eagle_chat import ValleyEagleChatfrom modelscope import snapshot_downloadimport urllib.request
# 需要把模型文件中的config.json的eagle_vision_tower和mm_vision_tower改为本地路径
model_dir = snapshot_download("bytedance-research/Valley-Eagle-7B")!modelscope download --model=Qwen/Qwen2-VL-7B-Instruct --local_dir=./Qwen2-VL-7B-Instruct!modelscope download --model=AI-ModelScope/siglip-so400m-patch14-384 --local_dir=./siglip-so400m-patch14-384model = ValleyEagleChat(    model_path=model_dir,    padding_side = 'left',)
url = 'http://p16-goveng-va.ibyteimg.com/tos-maliva-i-wtmo38ne4c-us/4870400481414052507~tplv-wtmo38ne4c-jpeg.jpeg'
img = urllib.request.urlopen(url=url, timeout=5).read()
request = {    "chat_history": [        {'role': 'system', 'content': 'You are Valley, developed by ByteDance. Your are a helpfull Assistant.'},        {'role': 'user', 'content': 'Describe the given image.'},    ],    "images": [img],}
result = model(request)print(f"\n>>> Assistant:\n")print(result)

from valley_eagle_chat import ValleyEagleChatimport decordimport requestsimport numpy as npfrom torchvision import transforms
model = ValleyEagleChat(    model_path=model_dir,    padding_side = 'left',)
url = 'https://videos.pexels.com/video-files/29641276/12753127_1920_1080_25fps.mp4'video_file = './video.mp4'response = requests.get(url)if response.status_code == 200:    with open("video.mp4", "wb") as f:        f.write(response.content)else:    print("download error!")    exit(1)
video_reader = decord.VideoReader(video_file)decord.bridge.set_bridge("torch")video = video_reader.get_batch(    np.linspace(0,  len(video_reader) - 1, 8).astype(np.int_)).byte()print([transforms.ToPILImage()(image.permute(2, 0, 1)).convert("RGB") for image in video])
request = {    "chat_history": [        {'role': 'system', 'content': 'You are Valley, developed by ByteDance. Your are a helpfull Assistant.'},        {'role': 'user', 'content': 'Describe the given video.'},    ],    "images": [transforms.ToPILImage()(image.permute(2, 0, 1)).convert("RGB") for image in video],}result = model(request)print(f"\n>>> Assistant:\n")print(result)