{"id":"https://openalex.org/W2004914077","doi":"https://doi.org/10.1145/2390803.2390815","title":"Pushing the limits of mechanical turk","display_name":"Pushing the limits of mechanical turk","publication_year":2012,"publication_date":"2012-10-29","ids":{"openalex":"https://openalex.org/W2004914077","doi":"https://doi.org/10.1145/2390803.2390815","mag":"2004914077"},"language":"en","primary_location":{"is_oa":false,"landing_page_url":"https://doi.org/10.1145/2390803.2390815","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false},"type":"article","type_crossref":"proceedings-article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006513718","display_name":"Luke Gottlieb","orcid":null},"institutions":[{"id":"https://openalex.org/I1297971548","display_name":"International Computer Science Institute","ror":"https://ror.org/01ewh7m12","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1297971548"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Luke Gottlieb","raw_affiliation_strings":["International Computer Science Institute, Berkeley, CA, USA."],"affiliations":[{"raw_affiliation_string":"International Computer Science Institute, Berkeley, CA, USA.","institution_ids":["https://openalex.org/I1297971548"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069814835","display_name":"Jaeyoung Choi","orcid":"https://orcid.org/0000-0001-9118-8050"},"institutions":[{"id":"https://openalex.org/I1297971548","display_name":"International Computer Science Institute","ror":"https://ror.org/01ewh7m12","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1297971548"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jaeyoung Choi","raw_affiliation_strings":["International Computer Science Institute, Berkeley, CA, USA."],"affiliations":[{"raw_affiliation_string":"International Computer Science Institute, Berkeley, CA, USA.","institution_ids":["https://openalex.org/I1297971548"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036676657","display_name":"Pascal Kelm","orcid":null},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Pascal Kelm","raw_affiliation_strings":["Technische Universitat Berlin, Berlin, Germany"],"affiliations":[{"raw_affiliation_string":"Technische Universitat Berlin, Berlin, Germany","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113970295","display_name":"Thomas Sikora","orcid":null},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Thomas Sikora","raw_affiliation_strings":["Technische Universitat Berlin, Berlin, Germany"],"affiliations":[{"raw_affiliation_string":"Technische Universitat Berlin, Berlin, Germany","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062323483","display_name":"Gerald Friedland","orcid":"https://orcid.org/0000-0002-9400-6539"},"institutions":[{"id":"https://openalex.org/I1297971548","display_name":"International Computer Science Institute","ror":"https://ror.org/01ewh7m12","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1297971548"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gerald Friedland","raw_affiliation_strings":["International Computer Science Institute, Berkeley, CA, USA."],"affiliations":[{"raw_affiliation_string":"International Computer Science Institute, Berkeley, CA, USA.","institution_ids":["https://openalex.org/I1297971548"]}]}],"institution_assertions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.515,"has_fulltext":true,"fulltext_origin":"ngrams","cited_by_count":15,"citation_normalized_percentile":{"value":0.732716,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":89},"biblio":{"volume":null,"issue":null,"first_page":"23","last_page":"28"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9999,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9999,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11980","display_name":"Human Mobility and Location-Based Analysis","score":0.9918,"subfield":{"id":"https://openalex.org/subfields/3313","display_name":"Transportation"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11819","display_name":"Data-Driven Disease Surveillance","score":0.9821,"subfield":{"id":"https://openalex.org/subfields/2713","display_name":"Epidemiology"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.643748}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.8219365},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.8117126},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.80083096},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.643748},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5016546},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46582195},{"id":"https://openalex.org/C146047270","wikidata":"https://www.wikidata.org/wiki/Q469666","display_name":"Human\u2013machine system","level":2,"score":0.42312396},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.40979385},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.40400308},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.38168988},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3450042},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07125893},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"is_oa":false,"landing_page_url":"https://doi.org/10.1145/2390803.2390815","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false}],"best_oa_location":null,"sustainable_development_goals":[],"grants":[],"datasets":[],"versions":[],"referenced_works_count":8,"referenced_works":["https://openalex.org/W2092719662","https://openalex.org/W2110764733","https://openalex.org/W2131938770","https://openalex.org/W2151401338","https://openalex.org/W2184725583","https://openalex.org/W2554839354","https://openalex.org/W3124258878","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W3196817267","https://openalex.org/W2392921965","https://openalex.org/W2383111961","https://openalex.org/W2377979023","https://openalex.org/W2365952365","https://openalex.org/W2361861616","https://openalex.org/W2358755282","https://openalex.org/W2263699433","https://openalex.org/W222919325","https://openalex.org/W2218034408"],"abstract_inverted_index":{"In":[0],"this":[1,34],"article":[2,89],"we":[3,7,37,94,103],"review":[4],"the":[5,16,20,26,91,100,107,119],"methods":[6],"have":[8],"developed":[9],"for":[10,15,33,45,66,80,86],"finding":[11],"Mechanical":[12,58,74],"Turk":[13,59,75],"participants":[14],"manual":[17],"annotation":[18,116],"of":[19,22],"geo-location":[21],"random":[23],"videos":[24],"from":[25,55],"web.":[27],"We":[28],"require":[29],"high":[30],"quality":[31],"annotations":[32],"project,":[35],"as":[36],"are":[38],"attempting":[39],"to":[40,48,105,111],"establish":[41],"a":[42,56,72],"human":[43],"baseline":[44],"future":[46],"comparison":[47],"machine":[49],"systems.":[50],"This":[51,88],"task":[52,60,76,117],"is":[53,64,77],"different":[54],"standard":[57,73],"in":[61,118],"that":[62,102],"it":[63],"difficult":[65,83],"both":[67],"humans":[68,81],"and":[69,82,99],"machines,":[70],"whereas":[71],"usually":[78],"easy":[79],"or":[84],"impossible":[85],"machines.":[87],"discusses":[90],"varied":[92],"difficulties":[93],"encountered":[95],"while":[96],"qualifying":[97],"annotators":[98],"steps":[101],"took":[104],"select":[106],"individuals":[108],"most":[109],"likely":[110],"do":[112],"well":[113],"at":[114],"our":[115],"future.":[120]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2004914077","counts_by_year":[{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":2},{"year":2015,"cited_by_count":3},{"year":2014,"cited_by_count":3},{"year":2013,"cited_by_count":3}],"updated_date":"2024-12-10T04:46:43.660722","created_date":"2016-06-24"}
  <script>parent.window.postMessage('ixistenz.url:https://api.openalex.org/works/doi:10.1145%2F2390803.2390815', '*'); </script> <div id='browserdiffcontainer' style='position: fixed; right: 10px; top: 10vh; min-width: 60px;  border: 1px solid white; background: rgb(251, 189, 0);; opacity: 0.9; font-family: helvetica, arial, sans serif;  font-size: 12px; z-index: 20001; padding-bottom: 20px;'><div onclick="toggle('browserdiffcontainerlist'); return false;" style='background: rgb(151, 89, 0);; color: white; '><strong>&nbsp; <span class='glyphicon glyphicon-minus'></span> NODES</strong></div><div id='browserdiffcontainerlist' style='padding-left: 10px; padding-right: 5px; max-height: 50vh; overflow: auto; '><div><a href='#diffid123' style='color: white;' onClick="parent.window.postMessage('ixistenz.opendiff:123', '*');; return false;" style='text-decoration: none; '>INTERN </a> <a style='color: white;' href='#diffid123'><sup>9</sup></a></div><div><a href='#diffid115' style='color: white;' onClick="parent.window.postMessage('ixistenz.opendiff:115', '*');; return false;" style='text-decoration: none; '>Project </a> <a style='color: white;' href='#diffid115'><sup>2</sup></a></div></div></div>
<script>
function toggle( divname ) {
  var x = document.getElementById( divname );
  if (x.style.display === 'none') {
    x.style.display = 'block';
  } else {
    x.style.display = 'none';
  }
} </script>