File size: 16,734 Bytes
7453da0
cbbd024
19d9994
c657020
9738ed3
5e703c7
 
540056e
b16b354
19d9994
5e703c7
 
b16b354
 
 
 
 
 
 
bba094e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22bd95b
 
 
 
 
 
19d9994
 
 
 
 
 
 
b16b354
cd48b2f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
540056e
7125d26
 
 
 
 
 
 
 
 
 
 
540056e
7125d26
5301c9c
7125d26
 
 
 
5301c9c
 
a2c09c5
5301c9c
 
a15b3ce
99bd104
 
c657020
 
5e6dba7
 
 
 
 
 
9738ed3
 
 
55e712f
9738ed3
 
 
 
 
 
 
c657020
a15b3ce
bf18300
a15b3ce
 
bf18300
a15b3ce
 
bf18300
a15b3ce
9738ed3
540056e
cf39162
0a67e21
 
 
 
 
 
 
 
 
396214f
f83630e
396214f
 
d1d7297
396214f
f83630e
396214f
 
 
 
 
 
 
 
 
 
 
 
 
7125d26
 
 
5e703c7
7125d26
 
 
 
5eb9bd5
5e703c7
7125d26
 
 
 
 
 
 
8f90609
319b833
8f90609
 
 
319b833
8f90609
 
 
 
 
1c32bc8
 
 
 
34b13a8
03c8ef5
bba094e
1c32bc8
 
 
 
13e8b80
 
 
 
 
 
 
 
 
 
19d9994
 
 
 
 
 
 
 
 
 
 
 
396214f
80aa4e5
508045d
 
 
9e731de
 
 
72aa658
508045d
6122bcd
32bf766
cbbd024
aef38d7
 
 
 
455006b
 
26ae92c
32bf766
57faca1
26ae92c
 
 
 
 
 
6122bcd
26ae92c
 
e377751
26ae92c
ac75eb4
26ae92c
 
e377751
 
 
6122bcd
508045d
fc0768e
508045d
 
6122bcd
f7b9ef5
508045d
80aa4e5
 
 
aef38d7
682c6da
 
80c21c9
cd48b2f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b27e1a3
ff0b641
b27e1a3
906ee65
 
 
b27e1a3
cd48b2f
 
682c6da
 
 
 
26ae92c
396214f
7125d26
 
 
 
 
fd37796
8f90609
 
 
1c32bc8
 
 
13e8b80
 
 
19d9994
 
 
8f4dc2d
cd48b2f
 
 
 
8f4dc2d
 
 
 
 
 
 
fd37796
8f4dc2d
 
 
 
 
 
13e8b80
 
 
19d9994
 
 
cd48b2f
 
8f4dc2d
cd48b2f
 
b9bed89
 
b89ea66
b9bed89
3127104
b9bed89
 
b89ea66
fc13525
 
 
 
d36a529
 
 
 
b9bed89
 
 
e377751
b9bed89
e377751
b9bed89
72aa658
b9bed89
 
776a974
 
e377751
776a974
cd48b2f
e377751
b16b354
e377751
b16b354
 
 
 
 
 
13e8b80
19d9994
 
b16b354
b27e1a3
 
b16b354
e377751
 
 
 
 
 
cd48b2f
 
1ec1afa
 
88f4ccf
7942f17
 
1ec1afa
 
 
 
e377751
776a974
e377751
5bddbaf
7125d26
fc13525
 
5bddbaf
e377751
8f4dc2d
e377751
396214f
 
776a974
cd48b2f
 
e377751
b9bed89
b16b354
 
 
 
 
 
8f4dc2d
 
 
cd48b2f
 
8f4dc2d
b16b354
b9bed89
 
 
cd48b2f
906ee65
 
b9bed89
 
5bddbaf
2ef1055
cd48b2f
 
1e18cd1
b9bed89
 
7125d26
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
import gradio as gr
import spaces
import os
import json
import re
import random
import numpy as np
from gradio_client import Client

hf_token = os.environ.get("HF_TOKEN")
MAX_SEED = np.iinfo(np.int32).max

def check_api(model_name):
    if model_name == "MAGNet":
        try :
            client = Client("https://fffiloni-magnet.hf.space/")
            return "api ready"
        except : 
            return "api not ready yet"
    elif model_name == "AudioLDM-2":
        try :
            client = Client("https://haoheliu-audioldm2-text2audio-text2music.hf.space/")
            return "api ready"
        except : 
            return "api not ready yet"
    elif model_name == "Riffusion":
        try :
            client = Client("https://fffiloni-spectrogram-to-music.hf.space/")
            return "api ready"
        except : 
            return "api not ready yet"
    elif model_name == "Mustango":
        try :
            client = Client("https://declare-lab-mustango.hf.space/")
            return "api ready"
        except : 
            return "api not ready yet"
    elif model_name == "MusicGen":
        try :
            client = Client("https://facebook-musicgen.hf.space/")
            return "api ready"
        except : 
            return "api not ready yet"
    elif model_name == "Stable Audio Open":
        try:
            client = Client("fffiloni/Stable-Audio-Open-A10", hf_token=hf_token)
            return "api ready"
        except:
            return "api not ready yet"

        
from moviepy.editor import *
import cv2

def extract_firstframe(video_in):
    vidcap = cv2.VideoCapture(video_in)
    success,image = vidcap.read()
    count = 0
    while success:
        if count == 0:
            cv2.imwrite("first_frame.jpg", image)     # save first extracted frame as jpg file named first_frame.jpg
        else:
            break   # exit loop after saving first frame
        success,image = vidcap.read()
        print ('Read a new frame: ', success)
        count += 1
    print ("Done extracted first frame!")
    return "first_frame.jpg"

def extract_audio(video_in):
    input_video = video_in
    output_audio = 'audio.wav'
    
    # Open the video file and extract the audio
    video_clip = VideoFileClip(input_video)
    audio_clip = video_clip.audio
    
    # Save the audio as a .wav file
    audio_clip.write_audiofile(output_audio, fps=44100)  # Use 44100 Hz as the sample rate for .wav files  
    print("Audio extraction complete.")

    return 'audio.wav'



def get_caption(image_in):
    kosmos2_client = Client("https://ydshieh-kosmos-2.hf.space/")
    kosmos2_result = kosmos2_client.predict(
        image_in,	# str (filepath or URL to image) in 'Test Image' Image component
        "Detailed",	# str in 'Description Type' Radio component
        fn_index=4
    )

    print(f"KOSMOS2 RETURNS: {kosmos2_result}")

    with open(kosmos2_result[1], 'r') as f:
        data = json.load(f)
    
    reconstructed_sentence = []
    for sublist in data:
        reconstructed_sentence.append(sublist[0])

    full_sentence = ' '.join(reconstructed_sentence)
    #print(full_sentence)

    # Find the pattern matching the expected format ("Describe this image in detail:" followed by optional space and then the rest)...
    pattern = r'^Describe this image in detail:\s*(.*)$'
    # Apply the regex pattern to extract the description text.
    match = re.search(pattern, full_sentence)
    if match:
        description = match.group(1)
        print(description)
    else:
        print("Unable to locate valid description.")

    # Find the last occurrence of "."
    #last_period_index = full_sentence.rfind('.')

    # Truncate the string up to the last period
    #truncated_caption = full_sentence[:last_period_index + 1]

    # print(truncated_caption)
    #print(f"\n—\nIMAGE CAPTION: {truncated_caption}")
    
    return description

def get_caption_from_MD(image_in):
    client = Client("https://vikhyatk-moondream1.hf.space/")
    result = client.predict(
		image_in,	# filepath  in 'image' Image component
		"Describe precisely the image.",	# str  in 'Question' Textbox component
		api_name="/answer_question"
    )
    print(result)
    return result

def get_magnet(prompt):

    client = Client("https://fffiloni-magnet.hf.space/")
    result = client.predict(
        "facebook/magnet-small-10secs",	# Literal['facebook/magnet-small-10secs', 'facebook/magnet-medium-10secs', 'facebook/magnet-small-30secs', 'facebook/magnet-medium-30secs', 'facebook/audio-magnet-small', 'facebook/audio-magnet-medium']  in 'Model' Radio component
        "",	# str  in 'Model Path (custom models)' Textbox component
        prompt,	# str  in 'Input Text' Textbox component
        3,	# float  in 'Temperature' Number component
        0.9,	# float  in 'Top-p' Number component
        10,	# float  in 'Max CFG coefficient' Number component
        1,	# float  in 'Min CFG coefficient' Number component
        20,	# float  in 'Decoding Steps (stage 1)' Number component
        10,	# float  in 'Decoding Steps (stage 2)' Number component
        10,	# float  in 'Decoding Steps (stage 3)' Number component
        10,	# float  in 'Decoding Steps (stage 4)' Number component
        "prod-stride1 (new!)",	# Literal['max-nonoverlap', 'prod-stride1 (new!)']  in 'Span Scoring' Radio component
        api_name="/predict_full"
    )
    print(result)
    return result[1]

def get_audioldm(prompt):
    client = Client("https://haoheliu-audioldm2-text2audio-text2music.hf.space/")
    seed = random.randint(0, MAX_SEED)
    result = client.predict(
        prompt,	# str in 'Input text' Textbox component
        "Low quality.",	# str in 'Negative prompt' Textbox component
        10,	# int | float (numeric value between 5 and 15) in 'Duration (seconds)' Slider component
        6.5,	# int | float (numeric value between 0 and 7) in 'Guidance scale' Slider component
        seed,	# int | float in 'Seed' Number component
        3,	# int | float (numeric value between 1 and 5) in 'Number waveforms to generate' Slider component
        fn_index=1
    )
    print(result)
    audio_result = extract_audio(result)
    return audio_result

def get_riffusion(prompt):
    client = Client("https://fffiloni-spectrogram-to-music.hf.space/")
    result = client.predict(
		prompt,	# str  in 'Musical prompt' Textbox component
		"",	# str  in 'Negative prompt' Textbox component
		None,	# filepath  in 'parameter_4' Audio component
		10,	# float (numeric value between 5 and 10) in 'Duration in seconds' Slider component
		api_name="/predict"
    )
    print(result)
    return result[1]

def get_mustango(prompt):
    client = Client("https://declare-lab-mustango.hf.space/")
    result = client.predict(
		prompt,	# str  in 'Prompt' Textbox component
		200,	# float (numeric value between 100 and 200) in 'Steps' Slider component
		6,	# float (numeric value between 1 and 10) in 'Guidance Scale' Slider component
							api_name="/predict"
    )
    print(result)
    return result

def get_musicgen(prompt):
    client = Client("https://facebook-musicgen.hf.space/")
    result = client.predict(
        prompt,	# str  in 'Describe your music' Textbox component
        None,	# str (filepath or URL to file) in 'File' Audio component
        fn_index=0
    )
    print(result)
    return result[1]

def get_stable_audio_open(prompt):
    client = Client("fffiloni/Stable-Audio-Open-A10", hf_token=hf_token)
    result = client.predict(
		prompt=prompt,
		seconds_total=10,
		steps=100,
		cfg_scale=7,
		api_name="/predict"
    )
    print(result)
    return result
    
import re
import torch
from transformers import pipeline

zephyr_model = "HuggingFaceH4/zephyr-7b-beta"
mixtral_model = "mistralai/Mixtral-8x7B-Instruct-v0.1"

pipe = pipeline("text-generation", model=zephyr_model, torch_dtype=torch.bfloat16, device_map="auto")

standard_sys = f"""
You are a musician AI whose job is to help users create their own music which its genre will reflect the character or scene from an image described by users.
In particular, you need to respond succintly with few musical words, in a friendly tone, write a musical prompt for a music generation model.

For example, if a user says, "a picture of a man in a black suit and tie riding a black dragon", provide immediately a musical prompt corresponding to the image description. 
Immediately STOP after that. It should be EXACTLY in this format:
"A grand orchestral arrangement with thunderous percussion, epic brass fanfares, and soaring strings, creating a cinematic atmosphere fit for a heroic battle"
"""

mustango_sys = f"""
You are a musician AI whose job is to help users create their own music which its genre will reflect the character or scene from an image described by users.
In particular, you need to respond succintly with few musical words, in a friendly tone, write a musical prompt for a music generation model, you MUST include chords progression.

For example, if a user says, "a painting of three old women having tea party", provide immediately a musical prompt corresponding to the image description. 
Immediately STOP after that. It should be EXACTLY in this format:
"The song is an instrumental. The song is in medium tempo with a classical guitar playing a lilting melody in accompaniment style. The song is emotional and romantic. The song is a romantic instrumental song. The chord sequence is Gm, F6, Ebm. The time signature is 4/4. This song is in Adagio. The key of this song is G minor."
"""

@spaces.GPU(enable_queue=True)
def get_musical_prompt(user_prompt, chosen_model):

    """
    if chosen_model == "Mustango" :
        agent_maker_sys = standard_sys
    else :
        agent_maker_sys = standard_sys
    """
    agent_maker_sys = standard_sys
    
    instruction = f"""
<|system|>
{agent_maker_sys}</s>
<|user|>
"""
    
    prompt = f"{instruction.strip()}\n{user_prompt}</s>"    
    outputs = pipe(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
    pattern = r'\<\|system\|\>(.*?)\<\|assistant\|\>'
    cleaned_text = re.sub(pattern, '', outputs[0]["generated_text"], flags=re.DOTALL)
    
    print(f"SUGGESTED Musical prompt: {cleaned_text}")
    return cleaned_text.lstrip("\n")


def blend_vmsc(video_in, audio_result):
    audioClip = AudioFileClip(audio_result)
    print(f"AUD: {audioClip.duration}")
    clip = VideoFileClip(video_in)
    print(f"VID: {clip.duration}")
    if clip.duration < audioClip.duration :
        audioClip = audioClip.subclip((0.0), (clip.duration))
    elif clip.duration > audioClip.duration :
        clip = clip.subclip((0.0), (audioClip.duration))
    final_clip = clip.set_audio(audioClip)
    # Set the output codec
    codec = 'libx264'
    audio_codec = 'aac'
    final_clip.write_videofile('final_video_with_music.mp4', codec=codec, audio_codec=audio_codec)
    return "final_video_with_music.mp4"
    
def infer(video_in, chosen_model, api_status):
    if video_in == None :
        raise gr.Error("Please provide a video input")

    if chosen_model == [] :
        raise gr.Error("Please pick a model")

    if api_status == "api not ready yet" :
        raise gr.Error("This model is not ready yet, you can pick another one instead :)")
    
    image_in = extract_firstframe(video_in)
    
    gr.Info("Getting image caption with Kosmos2...")
    user_prompt = get_caption(image_in)
    
    gr.Info("Building a musical prompt according to the image caption ...")
    musical_prompt = get_musical_prompt(user_prompt, chosen_model)

    if chosen_model == "MAGNet" :
        gr.Info("Now calling MAGNet for music...")
        music_o = get_magnet(musical_prompt)
    elif chosen_model == "AudioLDM-2" :
        gr.Info("Now calling AudioLDM-2 for music...")
        music_o = get_audioldm(musical_prompt)
    elif chosen_model == "Riffusion" :
        gr.Info("Now calling Riffusion for music...")
        music_o = get_riffusion(musical_prompt)
    elif chosen_model == "Mustango" :
        gr.Info("Now calling Mustango for music...")
        music_o = get_mustango(musical_prompt)
    elif chosen_model == "MusicGen" :
        gr.Info("Now calling MusicGen for music...")
        music_o = get_musicgen(musical_prompt)
    elif chosen_model == "Stable Audio Open" :
        gr.Info("Now calling Stable Audio Open for music...")
        music_o = get_stable_audio_open(musical_prompt)

    final_res = blend_vmsc(video_in, music_o)
    return gr.update(value=musical_prompt, interactive=True), gr.update(visible=True), music_o, final_res

def retry(video_in, chosen_model, caption):
    musical_prompt = caption

    if chosen_model == "MAGNet" :
        gr.Info("Now calling MAGNet for music...")
        music_o = get_magnet(musical_prompt)
    elif chosen_model == "AudioLDM-2" :
        gr.Info("Now calling AudioLDM-2 for music...")
        music_o = get_audioldm(musical_prompt)
    elif chosen_model == "Riffusion" :
        gr.Info("Now calling Riffusion for music...")
        music_o = get_riffusion(musical_prompt)
    elif chosen_model == "Mustango" :
        gr.Info("Now calling Mustango for music...")
        music_o = get_mustango(musical_prompt)
    elif chosen_model == "MusicGen" :
        gr.Info("Now calling MusicGen for music...")
        music_o = get_musicgen(musical_prompt)
    elif chosen_model == "Stable Audio Open" :
        gr.Info("Now calling Stable Audio Open for music...")
        music_o = get_stable_audio_open(musical_prompt)
    final_res = blend_vmsc(video_in, music_o)
    return music_o, final_res

demo_title = "Video to Music"
description = "Get music from a video shot, compare text-to-music models"

css = """
#col-container {
    margin: 0 auto;
    max-width: 980px;
    text-align: left;
}
#inspi-prompt textarea {
    font-size: 20px;
    line-height: 24px;
    font-weight: 600;
}
/* fix examples gallery width on mobile */
div#component-11 > .gallery > .gallery-item > .container > img {
    width: auto!important;
}
"""

with gr.Blocks(css=css) as demo:
    
    with gr.Column(elem_id="col-container"):
    
        gr.HTML(f"""
        <h2 style="text-align: center;">{demo_title}</h2>
        <p style="text-align: center;">{description}</p>
        """)
        
        with gr.Row():
            
            with gr.Column():
                video_in = gr.Video(sources=["upload"], label="Video input")
                
                with gr.Row():
                    
                    chosen_model = gr.Dropdown(
                        label = "Choose a model",
                        choices = [
                            "MAGNet",
                            "AudioLDM-2",
                            "Riffusion",
                            "Mustango",
                            "MusicGen",
                            "Stable Audio Open"
                        ],
                        value = None,
                        filterable = False
                    )
                    
                    check_status = gr.Textbox(
                        label="API status",
                        interactive=False
                    )
                
                submit_btn = gr.Button("Make music from my shot !")

                gr.Examples(
                    examples = [
                        "vid_examples/big-sur.mp4",
                        "vid_examples/snow-dogs.mp4",
                        "vid_examples/ocean-turtle.mp4"
                    ],
                    inputs = [video_in],
                    cache_examples=False   
                )
            
            with gr.Column():
            
                caption = gr.Textbox(
                    label = "Inspirational musical prompt",
                    interactive = False,
                    elem_id = "inspi-prompt"
                )
                
                retry_btn = gr.Button("Retry with edited prompt", visible=False)
                
                result = gr.Audio(
                    label = "Music"
                )

                video_o = gr.Video(label="Video with soundFX")
        

    chosen_model.change(
        fn = check_api,
        inputs = chosen_model,
        outputs = check_status,
        queue = False
    )

    retry_btn.click(
        fn = retry,
        inputs = [video_in, chosen_model, caption],
        outputs = [result, video_o]
    )
    
    submit_btn.click(
        fn = infer,
        inputs = [
            video_in,
            chosen_model,
            check_status
        ],
        outputs =[
            caption,
            retry_btn,
            result,
            video_o
        ],
    )

demo.queue(max_size=16).launch(show_api=False)