@Article{info:doi/10.2196/jmir.4582,
author="Kuang, Jinqiu
and Argo, Lauren
and Stoddard, Greg
and Bray, Bruce E
and Zeng-Treitler, Qing",
title="Assessing Pictograph Recognition: A Comparison of Crowdsourcing and Traditional Survey Approaches",
journal="J Med Internet Res",
year="2015",
month="Dec",
day="17",
volume="17",
number="12",
pages="e281",
keywords="crowdsourcing; patient discharge summaries; Amazon Mechanical Turk; pictograph recognition; cardiovascular",
abstract="Background: Compared to traditional methods of participant recruitment, online crowdsourcing platforms provide a fast and low-cost alternative. Amazon Mechanical Turk (MTurk) is a large and well-known crowdsourcing service. It has developed into the leading platform for crowdsourcing recruitment. Objective: To explore the application of online crowdsourcing for health informatics research, specifically the testing of medical pictographs. Methods: A set of pictographs created for cardiovascular hospital discharge instructions was tested for recognition. This set of illustrations (n=486) was first tested through an in-person survey in a hospital setting (n=150) and then using online MTurk participants (n=150). We analyzed these survey results to determine their comparability. Results: Both the demographics and the pictograph recognition rates of online participants were different from those of the in-person participants. In the multivariable linear regression model comparing the 2 groups, the MTurk group scored significantly higher than the hospital sample after adjusting for potential demographic characteristics (adjusted mean difference 0.18, 95{\%} CI 0.08-0.28, P<.001). The adjusted mean ratings were 2.95 (95{\%} CI 2.89-3.02) for the in-person hospital sample and 3.14 (95{\%} CI 3.07-3.20) for the online MTurk sample on a 4-point Likert scale (1=totally incorrect, 4=totally correct). Conclusions: The findings suggest that crowdsourcing is a viable complement to traditional in-person surveys, but it cannot replace them. ",
issn="1438-8871",
doi="10.2196/jmir.4582",
url="http://www.jmir.org/2015/12/e281/",
url="https://doi.org/10.2196/jmir.4582",
url="http://www.ncbi.nlm.nih.gov/pubmed/26678085"
}