MOS Survey Tool

[swarajdalmia]

Getting a reliable MOS score is really essential to evaluate any TTS system.
I was wondering if you guys had any links to any tool i could use to carry out a good MOS survey.

• Google forms doesn't allow me to upload audios
• I'm looking for a tool that randomises the audio's so as to deal with bias of priors and allows for a few different ratings for every audio

Please let me know if you have any leads on how i should go about this.

[a-froghyar]

https://github.com/audiolabs/webMUSHRA

…

On 12. May 2021, at 09:06, Swaraj Dalmia ***@***.***> wrote:  Getting a reliable MOS score is really essential to evaluate any TTS system. I was wondering if you guys had any links to any tool i could use to carry out a good MOS survey. Google forms doesn't allow me to upload audios I'm looking for a tool that randomises the audio's so as to deal with bias of priors and allows for a few different ratings for every audio Please let me know if you have any leads on how i should go about this. — You are receiving this because you are subscribed to this thread. Reply to this email directly, view it on GitHub, or unsubscribe.

[eminentgu]

I tried webMUSHRA but it is a bit difficulty for me to adapt and I want to do MOS tests without reference audio, which has no template.So currently my solution is to write it myself with Gradio, which is 100% python language and is widely used in ML fields, so ig it would be much easier for anyone in AI field to quickly launch a MOS test with Gradio.

#demo FYI
import gradio as gr

class MOSApp:
    MOS = {
        1: "1-Bad", 1.5: "1.5", 2: "2-Poor", 2.5: "2.5", 3: "3-Fair",
        3.5: "3.5", 4: "4-Good", 4.5: "4.5", 5: "5-Excellent"
    }

    def __init__(self):
        self.current_files = ['1.wav', '2.wav']

    def initialize_state(self):
        return {
            "index": 0,
            "selected_MOS": [],
            "tester_id": "",
            "data_store": {}
        }


    def submit_options(self, option, state):
        if not state["tester_id"]:
            return None, "Please enter your tester ID first.", -1, state

        if state["index"] < len(self.current_files) - 1:
            state["selected_MOS"].append(option)
            state["index"] += 1
            current_audio = self.current_files[state["index"]]
            return current_audio, f"#### you are rating {state['index']+1} out of {str(len(self.current_files))},after submitting Scroll UP to Step 2 to hear the new audio", -1, state
        elif state["index"] == len(self.current_files) - 1:
            state["selected_MOS"].append(option)
            state["data_store"][state["tester_id"]] = state["selected_MOS"]
            print(f"Tester ID: {state['tester_id']}, Scores: {state['selected_MOS']}")
            with open('res.txt','a') as f:
                text = state['tester_id'] + '\t'
                for s in state['selected_MOS']:
                    text += str(s) + '\t'
                text += '\n'
                f.write(text)
            state["index"] += 1
            return None, "## Thank you for your feedback! The return code is XXXXX", -1, state
        else:
            return None, "## Invalid submission! You can only submitted once per id! The return code is XXXXXX", -1, state    

    def set_tester_id(self, id, state):
        if id:
            state["tester_id"] = id
            state["selected_MOS"] = []
            state["index"] = 0
            return (
                f"## Your ID: {state['tester_id']}", 
                state, 
                self.current_files[0],  # Reset audio to first file
                f"#### you are rating {state['index']+1} out of {str(len(self.current_files))},after submitting Scroll UP to Step 2 to hear the new audio"  # Reset current file display
            )
        else:
            return "## Please enter a valid ID!", state, None, ""

    def create_interface(self):
        with gr.Blocks() as demo:
            state = gr.State(self.initialize_state())

            gr.Markdown("# TTS quality subjective survey demo with Gradio")

            gr.Markdown("## Step 1 Bind your tester id ONCE at the very beginning")
            
            with gr.Row():
                tester_id_input = gr.Textbox(label="Enter Tester ID")
                set_id_button = gr.Button("Set ID")
                id_display = gr.Markdown()
            
            gr.Markdown("------")
            gr.Markdown("## Step 2 Listen Carefully to the following audio: ")
            display_audio = gr.Audio(self.current_files[0], type='filepath')

            gr.Markdown("------")
            gr.Markdown("## Step3 Answer the following questions basing on the audio you hear.")
            score_description = gr.Markdown("""
                ### 1.  How would you rate the overall quality of the voice, in terms of its naturalness, intelligibility, and pronunciation?
                | Score | How natural / human | How Robotic |
                |-------|---------------------|-------------|
                | 5 Excellent | Completely natural speech | Imperceptible |
                | 4 Good | Mostly natural speech | Just perceptible but not annoying |
                | 3 Fair | Equally natural and unnatural | Perceptible and slightly annoying |
                | 2 Poor | Mostly unnatural speech | Annoying, but not objectionable |
                | 1 Bad | Completely unnatural speech | Very annoying and objectionable |

                ### Rating the Confidence Level of Each Recording
                """)
            options = gr.Slider(minimum=1, maximum=5, step=0.5,
                                value=-1, container=False, interactive=True) #label="Rate the overall quality of audio on a scale from 1 to 5",
            
            gr.HTML(self.get_slider_labels_html())
            
            with gr.Row():
                submit = gr.Button("Submit")
                #output = gr.Textbox(label="Your score will appear here")
            current_file = gr.Markdown("#### Enter your ID before you start or it won't be saved")

            set_id_button.click(
                self.set_tester_id, 
                inputs=[tester_id_input, state], 
                outputs=[id_display, state, display_audio, current_file]
            )
            submit.click(
                self.submit_options, 
                inputs=[options, state], 
                outputs=[display_audio, current_file, options, state]
            )

        return demo

    @staticmethod
    def get_slider_labels_html():
        return """
        <style>
            .slider-labels {
                display: flex;
                justify-content: space-between;
                margin-top: -10px;
                font-size: 12px;
            }
            .slider-labels div {
                text-align: center;
                width: 5%;
            }
            .slider-labels div:first-child {
                text-align: left;
            }
            .slider-labels div:last-child {
                text-align: right;
            }
        </style>
        <div class="slider-labels">
            <div>1 Bad</div>
            <div>1.5</div>
            <div>2 Poor</div>
            <div>2.5</div>
            <div>3 Fair</div>
            <div>3.5</div>
            <div>4 Good</div>
            <div>4.5</div>
            <div>5 Excellent</div>
        </div>
        """

if __name__ == "__main__":
    app = MOSApp()
    demo = app.create_interface()
    demo.launch(server_name="0.0.0.0", server_port=25565)