@Article{info:doi/10.2196/58158,
author="Chen, Xi
and Wang, Li
and You, MingKe
and Liu, WeiZhi
and Fu, Yu
and Xu, Jie
and Zhang, Shaoting
and Chen, Gang
and Li, Kang
and Li, Jian",
title="Evaluating and Enhancing Large Language Models' Performance in Domain-Specific Medicine: Development and Usability Study With DocOA",
journal="J Med Internet Res",
year="2024",
month="Jul",
day="22",
volume="26",
pages="e58158",
keywords="large language model; retrieval-augmented generation; domain-specific benchmark framework; osteoarthritis management",
abstract="Background: The efficacy of large language models (LLMs) in domain-specific medicine, particularly for managing complex diseases such as osteoarthritis (OA), remains largely unexplored. Objective: This study focused on evaluating and enhancing the clinical capabilities and explainability of LLMs in specific domains, using OA management as a case study. Methods: A domain-specific benchmark framework was developed to evaluate LLMs across a spectrum from domain-specific knowledge to clinical applications in real-world clinical scenarios. DocOA, a specialized LLM designed for OA management integrating retrieval-augmented generation and instructional prompts, was developed. It can identify the clinical evidence upon which its answers are based through retrieval-augmented generation, thereby demonstrating the explainability of those answers. The study compared the performance of GPT-3.5, GPT-4, and a specialized assistant, DocOA, using objective and human evaluations. Results: Results showed that general LLMs such as GPT-3.5 and GPT-4 were less effective in the specialized domain of OA management, particularly in providing personalized treatment recommendations. However, DocOA showed significant improvements. Conclusions: This study introduces a novel benchmark framework that assesses the domain-specific abilities of LLMs in multiple aspects, highlights the limitations of generalized LLMs in clinical contexts, and demonstrates the potential of tailored approaches for developing domain-specific medical LLMs. ",
issn="1438-8871",
doi="10.2196/58158",
url="https://www.jmir.org/2024/1/e58158",
url="https://doi.org/10.2196/58158",
url="http://www.ncbi.nlm.nih.gov/pubmed/38833165"
}