Commit 0cba6d623803a1f809dfe40dd627676454382446

Authored by waldispuhl
1 parent 4e112b3551
Exists in master

new refs & comments

Showing 2 changed files with 246 additions and 193 deletions Side-by-side Diff

CHIPLAYpaper/MarketPaper.tex View file @ 0cba6d6
... ... @@ -37,6 +37,7 @@
37 37 % line to: "\documentclass[chi_draft]{sigchi}". You can then place todo notes by using the "\todo{...}"
38 38 % command. Make sure to disable the draft option again before submitting your final document.
39 39 \usepackage{todonotes}
  40 +\usepackage{ulem}
40 41  
41 42 % Paper metadata (use plain text, for PDF inclusion and later
42 43 % re-using, if desired). Use \emtpyauthor when submitting for review
43 44  
... ... @@ -154,12 +155,13 @@
154 155  
155 156 However, such constraints are necessarily limiting the capacity of the system to harness the cognitive power of crowds and make full benefit of collective intelligence. For instance, iterative combinations of crowdsourced contributions can help enhancing creativity \cite{DBLP:conf/chi/YuN11}. Similarly, the presence of a broad spectrum of expertise in a crowdsourcing community has been shown to increase innovation and the advance of knowledge in the group~\cite{Dankulov2015}. The usefulness of parallelizing workflows has also been suggested for tasks accepting broad varieties of answers \cite{DBLP:conf/chi/Little10}.
156 157  
157   -The benefits of developing recommendation systems or coordination methods in collaborative environments has been demonstrated \cite{DBLP:conf/cscw/KitturK08,DBLP:conf/cscw/DowKKH12,DBLP:conf/chi/ZhangLMGPH12}. Therefore, in order to gain expressivity and improve their performance, the next generation of human-computation systems will certainly need to implement mechanisms to promote and control the collaboration between workers. Nonetheless, before transitioning to this model, it is important to first estimate the potential gains in productivity, and quantify the usefulness of the mechanisms and incentives to promote collaborative solving and prevent groupthink. A first step in this direction was made recently: mathematical models have been used to simulate the interactions between
158   -the participants of crowdsourcing projects in order to estimate the optimal group size depending on the difficulty of the problem~\cite{Guazzini2015}, and analyze the effect of repeated encounters on the reactive strategies of competing groups that can attack and sabotage each other~\cite{Oishi2014}.
  158 +The benefits of developing recommendation systems or coordination methods in collaborative environments has been demonstrated \cite{DBLP:conf/cscw/KitturK08,DBLP:conf/cscw/DowKKH12,DBLP:conf/chi/ZhangLMGPH12}. Therefore, in order to gain expressivity and improve their performance, the next generation of human-computation systems will certainly need to implement mechanisms to promote and control the collaboration between workers. Nonetheless, before transitioning to this model, it is important to first estimate the potential gains in productivity, and quantify the usefulness of the mechanisms and incentives to promote collaborative solving and prevent groupthink. A first step in this direction was made recently: mathematical models have been used to simulate the interactions between the participants of crowdsourcing projects in order to estimate the optimal group size depending on the difficulty of the problem~\cite{Guazzini2015}, and analyze the effect of repeated encounters on the reactive strategies of competing groups that can attack and sabotage each other~\cite{Oishi2014}.
159 159 %and analyze the relationships
160 160 %between the number of participants in a crowdsourcing group and the difficulty of the problem in order to find optimal size of groups [ref], and also the possible reactions
161 161 %between competing crowdsourcing groups that can attack and sabotage each other [ref].
162 162  
  163 +{\color{blue} Computer games are now a widely used and effective media to complete crowdsourcing tasks \cite{krause2015play}. Multiple studies already investigated the impact of design on human-computing tasks \cite{Krause:2010:FPE,ipeirotis2014quizz,kaufman2016investigating}. Incidentally, they are also an excellent and potentially very powerful framework to study collaborative processes.}
  164 +
163 165 Historically, computation on graphs has proven to be a good model to study the performance of humans in solving complex combinatorial problems \cite{Kearns:2006aa}. Experiments have been conducted to evaluate the dynamics of crowds collaborating at solving graph problems \cite{DBLP:journals/cacm/Kearns12} but still, little is known about the efficiency of the various modes of interaction.
164 166  
165 167 In this paper, we propose a formal framework to study human collaborative solving. We embed a combinatorial graph problem into a novel multiplayer game-with-a-purpose \cite{DBLP:conf/chi/AhnD04,DBLP:conf/aaai/HoCH07}, which will be used to engage participants and analyze collective performances. More precisely, we design a market game in which players can sell and buy solutions or bits of information, and couple this platform with (i) a skills system to enhance the efficiency of specific gaming strategies and (ii) a challenge system to guide the work of the crowd.
... ... @@ -186,7 +188,7 @@
186 188 %(e.g. increasing the diversity of intermediate solutions)
187 189 but that the efficiency of a skill is reduced if it is designed to help solve one of the primary objectives of the game. Finally, we observed that a precise parametrization of challenges (i.e. finding an appropriate difficulty, nor too easy, nor too difficult) is required to result in an improvement of the quality of the collective work.
188 190  
189   -{\color{red}Our game is freely available at \texttt{URL:TBA}}, and can be used as a platform for further independent studies.
  191 +{\color{blue}Our game is freely available at \texttt{http://csb.cs.mcgill.ca/market-game}}, and can be used as a platform for further independent studies.
190 192  
191 193 \section{Problem}
192 194  
... ... @@ -468,9 +470,7 @@
468 470 %coming from the experience gained by the players if they played a second time.
469 471 Before starting each game session, the players were shown a document explaining
470 472 the rules of the game and the interface.
471   -{\color{red} The participants were told that they were playing a human computing game that aims to solve a graph problem, but the mathematical problem in question was
472   - not described to the players (to keep the instructions as simple as possible), only the rules of the game.}
473   -They were also asked to fill in a questionnaire so that we could get information on the participants, such as their age,
  473 +{\color{red} The participants were told that they were playing a human computing game that aims to solve a graph problem, but the mathematical problem in question was not described to the players \sout{(to keep the instructions as simple as possible), only the rules of the game.}} They were also asked to fill in a questionnaire so that we could get information on the participants, such as their age,
474 474 their abilities at puzzle solving and their experience with video games for example. For all the experiments, the game session lasted 45 minutes.
475 475  
476 476 \subsection{Generating the graph}
... ... @@ -853,7 +853,7 @@
853 853 Game time & 10.00 & 4.11\\\hline
854 854 \end{tabular}
855 855 \end{center}
856   -\caption{Average statistics on the top 12 players vs the others}\label{tab_playerStats}
  856 +\caption{Average statistics on the top 12 players vs the others {\color{blue}[What are the units for game time?]}}\label{tab_playerStats}
857 857 \end{table}
858 858  
859 859 {\color{red} We also looked at the mean age, mean video game time per week, and mean puzzle solving self evaluation for each of the 12 groups. We used
860 860  
... ... @@ -895,11 +895,11 @@
895 895 of the game and find winning strategies faster than the average participant.
896 896  
897 897 {\color{red} In the context of a human computing game, where participants are contributing for free, players' perception and statisfaction with the gameplay is very important.
898   - We asked the players in the questionnaire to rate their experience with the game on a scale of 1 to 10 and the average score was 7.16.
  898 + We asked the players in the questionnaire to rate their experience with the game on a scale of 1 to 10 and the average score was 7.16 {\color{blue} [Is it good?]}.
899 899 Moreover, we also gathered some feedback when we were talking to the participants after the sessions.
900 900 As mentioned earlier, people who played the game with the market enjoyed it a lot more than the ones who played without it.
901   - Since the game has no end in itself (we decided to limit the sessions to 45 minutes, but you can play forever), some players mentioned that it is very addictive
902   - (some wanted us to release the game publicly so they could play at home). Competing against other players to be at the top of the leaderboard was also a powerful
  901 + Since the game has no end in itself (we decided to limit the sessions to 45 minutes, \sout{but you can play forever}), some players mentioned that it is very addictive
  902 + \sout{(some wanted us to release the game publicly so they could play at home)}. Competing against other players to be at the top of the leaderboard was also a powerful
903 903 motivation for the players. Obviously, we also had some negative comments: some players found the game hard to understand, while some very good players found it too easy.
904 904 Overall, it is quite interesting to see that players enjoyed the game in its current state without any extensive effort put into visual aesthetics.
905 905  
... ... @@ -917,7 +917,7 @@
917 917 The authors would also like to thank Jean-Fran\c{c}ois Bourbeau, Mathieu Blanchette, Derek Ruths and Edward Newell for their help with the initial design of the game,
918 918 and Alexandre Leblanc for his helpful advice on the statistical tests.
919 919 The authors wish to thank Silvia Juliana Leon Mantilla and Shu Hayakawa for their help with the organization of the game sessions and the recruitment of participants.
920   -{\color{red}This work was made possible by grants from NSERC and FRQNT.}
  920 +{\color{red}This work was made possible by grants from NSERC, Genome Canada and FRQNT.}
921 921  
922 922 % REFERENCES FORMAT
923 923 % References must be the same font size as other body text.
CHIPLAYpaper/references.bib View file @ 0cba6d6
... ... @@ -2,13 +2,61 @@
2 2 %% http://bibdesk.sourceforge.net/
3 3  
4 4  
5   -%% Created for Jerome Waldispuhl at 2016-04-18 16:03:55 +0200
  5 +%% Created for Jerome Waldispuhl at 2016-07-28 01:07:48 -0400
6 6  
7 7  
8 8 %% Saved with string encoding Unicode (UTF-8)
9 9  
10 10  
11 11  
  12 +@inproceedings{Krause:2010:FPE,
  13 + Acmid = {1837893},
  14 + Address = {New York, NY, USA},
  15 + Author = {Krause, Markus and Takhtamysheva, Aneta and Wittstock, Marion and Malaka, Rainer},
  16 + Booktitle = {Proceedings of the ACM SIGKDD Workshop on Human Computation},
  17 + Date-Added = {2016-07-28 03:37:55 +0000},
  18 + Date-Modified = {2016-07-28 04:04:08 +0000},
  19 + Doi = {10.1145/1837885.1837893},
  20 + Isbn = {978-1-4503-0222-7},
  21 + Location = {Washington DC},
  22 + Numpages = {4},
  23 + Pages = {22--25},
  24 + Publisher = {ACM},
  25 + Series = {HCOMP '10},
  26 + Title = {Frontiers of a Paradigm: Exploring Human Computation with Digital Games},
  27 + Url = {http://doi.acm.org/10.1145/1837885.1837893},
  28 + Year = {2010},
  29 + Bdsk-Url-1 = {http://doi.acm.org/10.1145/1837885.1837893},
  30 + Bdsk-Url-2 = {http://dx.doi.org/10.1145/1837885.1837893}}
  31 +
  32 +@inproceedings{kaufman2016investigating,
  33 + Author = {Kaufman, Geoff and Flanagan, Mary and Punjasthitkul, Sukdith},
  34 + Booktitle = {Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems},
  35 + Date-Added = {2016-07-28 03:34:35 +0000},
  36 + Date-Modified = {2016-07-28 03:34:35 +0000},
  37 + Organization = {ACM},
  38 + Pages = {4122--4128},
  39 + Title = {Investigating the Impact of'Emphasis Frames' and Social Loafing on Player Motivation and Performance in a Crowdsourcing Game},
  40 + Year = {2016}}
  41 +
  42 +@inproceedings{ipeirotis2014quizz,
  43 + Author = {Ipeirotis, Panagiotis G and Gabrilovich, Evgeniy},
  44 + Booktitle = {Proceedings of the 23rd international conference on World wide web},
  45 + Date-Added = {2016-07-28 03:34:03 +0000},
  46 + Date-Modified = {2016-07-28 03:34:03 +0000},
  47 + Organization = {ACM},
  48 + Pages = {143--154},
  49 + Title = {Quizz: targeted crowdsourcing with a billion (potential) users},
  50 + Year = {2014}}
  51 +
  52 +@inproceedings{krause2015play,
  53 + Author = {Krause, Markus and Kizilcec, Ren{\'e}},
  54 + Booktitle = {Third AAAI Conference on Human Computation and Crowdsourcing},
  55 + Date-Added = {2016-07-28 03:32:45 +0000},
  56 + Date-Modified = {2016-07-28 03:32:45 +0000},
  57 + Title = {To Play or Not to Play: Interactions between Response Quality and Task Complexity in Games and Paid Crowdsourcing},
  58 + Year = {2015}}
  59 +
12 60 @article{Kawrykow:2012aa,
13 61 Abstract = {BACKGROUND: Comparative genomics, or the study of the relationships of genome structure and function across different species, offers a powerful tool for studying evolution, annotating genomes, and understanding the causes of various genetic disorders. However, aligning multiple sequences of DNA, an essential intermediate step for most types of analyses, is a difficult computational task. In parallel, citizen science, an approach that takes advantage of the fact that the human brain is exquisitely tuned to solving specific types of problems, is becoming increasingly popular. There, instances of hard computational problems are dispatched to a crowd of non-expert human game players and solutions are sent back to a central server.
14 62 METHODOLOGY/PRINCIPAL FINDINGS: We introduce Phylo, a human-based computing framework applying "crowd sourcing" techniques to solve the Multiple Sequence Alignment (MSA) problem. The key idea of Phylo is to convert the MSA problem into a casual game that can be played by ordinary web users with a minimal prior knowledge of the biological context. We applied this strategy to improve the alignment of the promoters of disease-related genes from up to 44 vertebrate species. Since the launch in November 2010, we received more than 350,000 solutions submitted from more than 12,000 registered users. Our results show that solutions submitted contributed to improving the accuracy of up to 70% of the alignment blocks considered.
... ... @@ -29,7 +77,6 @@
29 77 Volume = {7},
30 78 Year = {2012},
31 79 Bdsk-Url-1 = {http://dx.doi.org/10.1371/journal.pone.0031362}}
32   -
33 80  
34 81 @article{Kwak:2013aa,
35 82 Abstract = {Citizen science games such as Galaxy Zoo, Foldit, and Phylo aim to harness the intelligence and processing power generated by crowds of online gamers to solve scientific problems. However, the selection of the data to be analyzed through these games is under the exclusive control of the game designers, and so are the results produced by gamers. Here, we introduce Open-Phylo, a freely accessible crowd-computing platform that enables any scientist to enter our system and use crowds of gamers to assist computer programs in solving one of the most fundamental problems in genomics: the multiple sequence alignment problem.},
36 83  
37 84  
38 85  
39 86  
40 87  
41 88  
42 89  
43 90  
44 91  
45 92  
46 93  
47 94  
48 95  
49 96  
... ... @@ -49,209 +96,215 @@
49 96 Volume = {14},
50 97 Year = {2013},
51 98 Bdsk-Url-1 = {http://dx.doi.org/10.1186/gb-2013-14-10-r116}}
52   -
  99 +
53 100 @article{Guazzini2015,
54   - author = {Guazzini, Andrea and Vilone, Daniele and Donati, Camillo and Nardi, Annalisa and Levnajić, Zoran},
55   - title = {Modeling crowdsourcing as collective problem solving},
56   - journal = {Scientific Reports},
57   - volume = {5},
58   - number = {16557},
59   - year = {2015},
60   - articleno = {16557},
61   - url = {http://dx.doi.org/10.1038/srep16557},
62   - doi = {10.1038/srep16557},
63   -}
  101 + Articleno = {16557},
  102 + Author = {Guazzini, Andrea and Vilone, Daniele and Donati, Camillo and Nardi, Annalisa and Levnaji{\'c}, Zoran},
  103 + Doi = {10.1038/srep16557},
  104 + Journal = {Scientific Reports},
  105 + Number = {16557},
  106 + Title = {Modeling crowdsourcing as collective problem solving},
  107 + Url = {http://dx.doi.org/10.1038/srep16557},
  108 + Volume = {5},
  109 + Year = {2015},
  110 + Bdsk-Url-1 = {http://dx.doi.org/10.1038/srep16557}}
64 111  
65 112 @article{Oishi2014,
66   - author = {Oishi, Koji and Cebrian, Manuel and Abeliuk, Andres and Masuda, Naoki},
67   - title = {Iterated crowdsourcing dilemma game},
68   - journal = {Scientific Reports},
69   - volume = {4},
70   - number = {4100},
71   - year = {2014},
72   - articleno = {4100},
73   - url = {http://dx.doi.org/10.1038/srep04100},
74   - doi = {10.1038/srep04100},
75   -}
  113 + Articleno = {4100},
  114 + Author = {Oishi, Koji and Cebrian, Manuel and Abeliuk, Andres and Masuda, Naoki},
  115 + Doi = {10.1038/srep04100},
  116 + Journal = {Scientific Reports},
  117 + Number = {4100},
  118 + Title = {Iterated crowdsourcing dilemma game},
  119 + Url = {http://dx.doi.org/10.1038/srep04100},
  120 + Volume = {4},
  121 + Year = {2014},
  122 + Bdsk-Url-1 = {http://dx.doi.org/10.1038/srep04100}}
76 123  
77 124 @article{Dankulov2015,
78   - author = {Dankulov, Marija Mitrovi\'{c} and Melnik, Roderick and Tadi\'{c}, Bosiljka},
79   - title = {The dynamics of meaningful social interactions and the emergence of collective knowledge},
80   - journal = {Scientific Reports},
81   - volume = {5},
82   - number = {12197},
83   - year = {2015},
84   - articleno = {12197},
85   - url = {http://dx.doi.org/10.1038/srep12197},
86   - doi = {10.1038/srep12197},
87   -}
  125 + Articleno = {12197},
  126 + Author = {Dankulov, Marija Mitrovi\'{c} and Melnik, Roderick and Tadi\'{c}, Bosiljka},
  127 + Doi = {10.1038/srep12197},
  128 + Journal = {Scientific Reports},
  129 + Number = {12197},
  130 + Title = {The dynamics of meaningful social interactions and the emergence of collective knowledge},
  131 + Url = {http://dx.doi.org/10.1038/srep12197},
  132 + Volume = {5},
  133 + Year = {2015},
  134 + Bdsk-Url-1 = {http://dx.doi.org/10.1038/srep12197}}
88 135  
89 136 @inproceedings{Chang10,
90   - author = {Hu, Chang and Bederson, Benjamin B. and Resnik, Philip},
91   - title = {Translation by Iterative Collaboration Between Monolingual Users},
92   - booktitle = {Proceedings of the ACM SIGKDD Workshop on Human Computation},
93   - series = {HCOMP '10},
94   - year = {2010},
95   - isbn = {978-1-4503-0222-7},
96   - location = {Washington DC},
97   - pages = {54--55},
98   - numpages = {2},
99   - url = {http://doi.acm.org/10.1145/1837885.1837902},
100   - doi = {10.1145/1837885.1837902},
101   - acmid = {1837902},
102   - publisher = {ACM},
103   - address = {New York, NY, USA},
104   - keywords = {crowdsourcing, distributed human computation, human computation, machine translation, monolingual, translation, translation interface, wisdom of crowds},
105   -}
  137 + Acmid = {1837902},
  138 + Address = {New York, NY, USA},
  139 + Author = {Hu, Chang and Bederson, Benjamin B. and Resnik, Philip},
  140 + Booktitle = {Proceedings of the ACM SIGKDD Workshop on Human Computation},
  141 + Doi = {10.1145/1837885.1837902},
  142 + Isbn = {978-1-4503-0222-7},
  143 + Keywords = {crowdsourcing, distributed human computation, human computation, machine translation, monolingual, translation, translation interface, wisdom of crowds},
  144 + Location = {Washington DC},
  145 + Numpages = {2},
  146 + Pages = {54--55},
  147 + Publisher = {ACM},
  148 + Series = {HCOMP '10},
  149 + Title = {Translation by Iterative Collaboration Between Monolingual Users},
  150 + Url = {http://doi.acm.org/10.1145/1837885.1837902},
  151 + Year = {2010},
  152 + Bdsk-Url-1 = {http://doi.acm.org/10.1145/1837885.1837902},
  153 + Bdsk-Url-2 = {http://dx.doi.org/10.1145/1837885.1837902}}
106 154  
107 155 @article{Phrasedetectives,
108   - author = {Poesio, Massimo and Chamberlain, Jon and Kruschwitz, Udo and Robaldo, Livio and Ducceschi, Luca},
109   - title = {Phrase Detectives: Utilizing Collective Intelligence for Internet-scale Language Resource Creation},
110   - journal = {ACM Trans. Interact. Intell. Syst.},
111   - issue_date = {April 2013},
112   - volume = {3},
113   - number = {1},
114   - month = apr,
115   - year = {2013},
116   - issn = {2160-6455},
117   - pages = {3:1--3:44},
118   - articleno = {3},
119   - numpages = {44},
120   - url = {http://doi.acm.org/10.1145/2448116.2448119},
121   - doi = {10.1145/2448116.2448119},
122   - acmid = {2448119},
123   - publisher = {ACM},
124   - address = {New York, NY, USA},
125   - keywords = {Web cooperation, anaphora, corpus annotation, games with a purpose, human language technology, resource creation},
126   -}
  156 + Acmid = {2448119},
  157 + Address = {New York, NY, USA},
  158 + Articleno = {3},
  159 + Author = {Poesio, Massimo and Chamberlain, Jon and Kruschwitz, Udo and Robaldo, Livio and Ducceschi, Luca},
  160 + Doi = {10.1145/2448116.2448119},
  161 + Issn = {2160-6455},
  162 + Issue_Date = {April 2013},
  163 + Journal = {ACM Trans. Interact. Intell. Syst.},
  164 + Keywords = {Web cooperation, anaphora, corpus annotation, games with a purpose, human language technology, resource creation},
  165 + Month = apr,
  166 + Number = {1},
  167 + Numpages = {44},
  168 + Pages = {3:1--3:44},
  169 + Publisher = {ACM},
  170 + Title = {Phrase Detectives: Utilizing Collective Intelligence for Internet-scale Language Resource Creation},
  171 + Url = {http://doi.acm.org/10.1145/2448116.2448119},
  172 + Volume = {3},
  173 + Year = {2013},
  174 + Bdsk-Url-1 = {http://doi.acm.org/10.1145/2448116.2448119},
  175 + Bdsk-Url-2 = {http://dx.doi.org/10.1145/2448116.2448119}}
127 176  
128   -
129 177 @inproceedings{Picbreeder,
130   - author = {Secretan, Jimmy and Beato, Nicholas and D Ambrosio, David B. and Rodriguez, Adelein and Campbell, Adam and Stanley, Kenneth O.},
131   - title = {Picbreeder: Evolving Pictures Collaboratively Online},
132   - booktitle = {Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
133   - series = {CHI '08},
134   - year = {2008},
135   - isbn = {978-1-60558-011-1},
136   - location = {Florence, Italy},
137   - pages = {1759--1768},
138   - numpages = {10},
139   - url = {http://doi.acm.org/10.1145/1357054.1357328},
140   - doi = {10.1145/1357054.1357328},
141   - acmid = {1357328},
142   - publisher = {ACM},
143   - address = {New York, NY, USA},
144   - keywords = {collaborative interactive evolution, interactive evolutionary computation},
145   -}
  178 + Acmid = {1357328},
  179 + Address = {New York, NY, USA},
  180 + Author = {Secretan, Jimmy and Beato, Nicholas and D Ambrosio, David B. and Rodriguez, Adelein and Campbell, Adam and Stanley, Kenneth O.},
  181 + Booktitle = {Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
  182 + Doi = {10.1145/1357054.1357328},
  183 + Isbn = {978-1-60558-011-1},
  184 + Keywords = {collaborative interactive evolution, interactive evolutionary computation},
  185 + Location = {Florence, Italy},
  186 + Numpages = {10},
  187 + Pages = {1759--1768},
  188 + Publisher = {ACM},
  189 + Series = {CHI '08},
  190 + Title = {Picbreeder: Evolving Pictures Collaboratively Online},
  191 + Url = {http://doi.acm.org/10.1145/1357054.1357328},
  192 + Year = {2008},
  193 + Bdsk-Url-1 = {http://doi.acm.org/10.1145/1357054.1357328},
  194 + Bdsk-Url-2 = {http://dx.doi.org/10.1145/1357054.1357328}}
146 195  
147 196 @inproceedings{Packplay,
148   - author = {Green, Nathan and Breimyer, Paul and Kumar, Vinay and Samatova, Nagiza F.},
149   - title = {PackPlay: Mining Semantic Data in Collaborative Games},
150   - booktitle = {Proceedings of the Fourth Linguistic Annotation Workshop},
151   - series = {LAW IV '10},
152   - year = {2010},
153   - isbn = {978-1-932432-72-5},
154   - location = {Uppsala, Sweden},
155   - pages = {227--234},
156   - numpages = {8},
157   - url = {http://dl.acm.org/citation.cfm?id=1868720.1868757},
158   - acmid = {1868757},
159   - publisher = {Association for Computational Linguistics},
160   - address = {Stroudsburg, PA, USA},
161   -}
  197 + Acmid = {1868757},
  198 + Address = {Stroudsburg, PA, USA},
  199 + Author = {Green, Nathan and Breimyer, Paul and Kumar, Vinay and Samatova, Nagiza F.},
  200 + Booktitle = {Proceedings of the Fourth Linguistic Annotation Workshop},
  201 + Isbn = {978-1-932432-72-5},
  202 + Location = {Uppsala, Sweden},
  203 + Numpages = {8},
  204 + Pages = {227--234},
  205 + Publisher = {Association for Computational Linguistics},
  206 + Series = {LAW IV '10},
  207 + Title = {PackPlay: Mining Semantic Data in Collaborative Games},
  208 + Url = {http://dl.acm.org/citation.cfm?id=1868720.1868757},
  209 + Year = {2010},
  210 + Bdsk-Url-1 = {http://dl.acm.org/citation.cfm?id=1868720.1868757}}
162 211  
163 212 @inproceedings{Waisda,
164   - author = {Hildebrand, Michiel and Brinkerink, Maarten and Gligorov, Riste and van Steenbergen, Martijn and Huijkman, Johan and Oomen, Johan},
165   - title = {Waisda?: Video Labeling Game},
166   - booktitle = {Proceedings of the 21st ACM International Conference on Multimedia},
167   - series = {MM '13},
168   - year = {2013},
169   - isbn = {978-1-4503-2404-5},
170   - location = {Barcelona, Spain},
171   - pages = {823--826},
172   - numpages = {4},
173   - url = {http://doi.acm.org/10.1145/2502081.2502221},
174   - doi = {10.1145/2502081.2502221},
175   - acmid = {2502221},
176   - publisher = {ACM},
177   - address = {New York, NY, USA},
178   - keywords = {Waisda, crowdsourcing, gwap, video tagging},
179   -}
  213 + Acmid = {2502221},
  214 + Address = {New York, NY, USA},
  215 + Author = {Hildebrand, Michiel and Brinkerink, Maarten and Gligorov, Riste and van Steenbergen, Martijn and Huijkman, Johan and Oomen, Johan},
  216 + Booktitle = {Proceedings of the 21st ACM International Conference on Multimedia},
  217 + Doi = {10.1145/2502081.2502221},
  218 + Isbn = {978-1-4503-2404-5},
  219 + Keywords = {Waisda, crowdsourcing, gwap, video tagging},
  220 + Location = {Barcelona, Spain},
  221 + Numpages = {4},
  222 + Pages = {823--826},
  223 + Publisher = {ACM},
  224 + Series = {MM '13},
  225 + Title = {Waisda?: Video Labeling Game},
  226 + Url = {http://doi.acm.org/10.1145/2502081.2502221},
  227 + Year = {2013},
  228 + Bdsk-Url-1 = {http://doi.acm.org/10.1145/2502081.2502221},
  229 + Bdsk-Url-2 = {http://dx.doi.org/10.1145/2502081.2502221}}
180 230  
181 231 @inproceedings{Tagatune,
182   - author = {Law, Edith and von Ahn, Luis},
183   - title = {Input-agreement: A New Mechanism for Collecting Data Using Human Computation Games},
184   - booktitle = {Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
185   - series = {CHI '09},
186   - year = {2009},
187   - isbn = {978-1-60558-246-7},
188   - location = {Boston, MA, USA},
189   - pages = {1197--1206},
190   - numpages = {10},
191   - url = {http://doi.acm.org/10.1145/1518701.1518881},
192   - doi = {10.1145/1518701.1518881},
193   - acmid = {1518881},
194   - publisher = {ACM},
195   - address = {New York, NY, USA},
196   - keywords = {games with a purpose, human computation, tagging},
197   -}
  232 + Acmid = {1518881},
  233 + Address = {New York, NY, USA},
  234 + Author = {Law, Edith and von Ahn, Luis},
  235 + Booktitle = {Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
  236 + Doi = {10.1145/1518701.1518881},
  237 + Isbn = {978-1-60558-246-7},
  238 + Keywords = {games with a purpose, human computation, tagging},
  239 + Location = {Boston, MA, USA},
  240 + Numpages = {10},
  241 + Pages = {1197--1206},
  242 + Publisher = {ACM},
  243 + Series = {CHI '09},
  244 + Title = {Input-agreement: A New Mechanism for Collecting Data Using Human Computation Games},
  245 + Url = {http://doi.acm.org/10.1145/1518701.1518881},
  246 + Year = {2009},
  247 + Bdsk-Url-1 = {http://doi.acm.org/10.1145/1518701.1518881},
  248 + Bdsk-Url-2 = {http://dx.doi.org/10.1145/1518701.1518881}}
198 249  
199 250 @inproceedings{Visimu,
200   - author = {Wang, Borui and Chen, Jingshu},
201   - title = {Visimu: A Game for Music Color Label Collection},
202   - booktitle = {Proceedings of the Adjunct Publication of the 26th Annual ACM Symposium on User Interface Software and Technology},
203   - series = {UIST '13 Adjunct},
204   - year = {2013},
205   - isbn = {978-1-4503-2406-9},
206   - location = {St. Andrews, Scotland, United Kingdom},
207   - pages = {93--94},
208   - numpages = {2},
209   - url = {http://doi.acm.org/10.1145/2508468.2514726},
210   - doi = {10.1145/2508468.2514726},
211   - acmid = {2514726},
212   - publisher = {ACM},
213   - address = {New York, NY, USA},
214   - keywords = {color perception, game with a purpose, music information retrieval, music visualization},
215   -}
  251 + Acmid = {2514726},
  252 + Address = {New York, NY, USA},
  253 + Author = {Wang, Borui and Chen, Jingshu},
  254 + Booktitle = {Proceedings of the Adjunct Publication of the 26th Annual ACM Symposium on User Interface Software and Technology},
  255 + Doi = {10.1145/2508468.2514726},
  256 + Isbn = {978-1-4503-2406-9},
  257 + Keywords = {color perception, game with a purpose, music information retrieval, music visualization},
  258 + Location = {St. Andrews, Scotland, United Kingdom},
  259 + Numpages = {2},
  260 + Pages = {93--94},
  261 + Publisher = {ACM},
  262 + Series = {UIST '13 Adjunct},
  263 + Title = {Visimu: A Game for Music Color Label Collection},
  264 + Url = {http://doi.acm.org/10.1145/2508468.2514726},
  265 + Year = {2013},
  266 + Bdsk-Url-1 = {http://doi.acm.org/10.1145/2508468.2514726},
  267 + Bdsk-Url-2 = {http://dx.doi.org/10.1145/2508468.2514726}}
216 268  
217   -
218 269 @article{Kisskissban,
219   - author = {Ho, Chien-Ju and Chang, Tao-Hsuan and Lee, Jong-Chuan and Hsu, Jane Yung-jen and Chen, Kuan-Ta},
220   - title = {KissKissBan: A Competitive Human Computation Game for Image Annotation},
221   - journal = {SIGKDD Explor. Newsl.},
222   - issue_date = {June 2010},
223   - volume = {12},
224   - number = {1},
225   - month = nov,
226   - year = {2010},
227   - issn = {1931-0145},
228   - pages = {21--24},
229   - numpages = {4},
230   - url = {http://doi.acm.org/10.1145/1882471.1882475},
231   - doi = {10.1145/1882471.1882475},
232   - acmid = {1882475},
233   - publisher = {ACM},
234   - address = {New York, NY, USA},
235   - keywords = {ESP game, amazon mechanical turk, games with a purpose, human computation, image annotation},
236   -}
  270 + Acmid = {1882475},
  271 + Address = {New York, NY, USA},
  272 + Author = {Ho, Chien-Ju and Chang, Tao-Hsuan and Lee, Jong-Chuan and Hsu, Jane Yung-jen and Chen, Kuan-Ta},
  273 + Doi = {10.1145/1882471.1882475},
  274 + Issn = {1931-0145},
  275 + Issue_Date = {June 2010},
  276 + Journal = {SIGKDD Explor. Newsl.},
  277 + Keywords = {ESP game, amazon mechanical turk, games with a purpose, human computation, image annotation},
  278 + Month = nov,
  279 + Number = {1},
  280 + Numpages = {4},
  281 + Pages = {21--24},
  282 + Publisher = {ACM},
  283 + Title = {KissKissBan: A Competitive Human Computation Game for Image Annotation},
  284 + Url = {http://doi.acm.org/10.1145/1882471.1882475},
  285 + Volume = {12},
  286 + Year = {2010},
  287 + Bdsk-Url-1 = {http://doi.acm.org/10.1145/1882471.1882475},
  288 + Bdsk-Url-2 = {http://dx.doi.org/10.1145/1882471.1882475}}
237 289  
238 290 @inproceedings{Peekaboom,
239   - author = {von Ahn, Luis and Liu, Ruoran and Blum, Manuel},
240   - title = {Peekaboom: A Game for Locating Objects in Images},
241   - booktitle = {Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
242   - series = {CHI '06},
243   - year = {2006},
244   - isbn = {1-59593-372-7},
245   - location = {Montr\&\#233;al, Qu\&\#233;bec, Canada},
246   - pages = {55--64},
247   - numpages = {10},
248   - url = {http://doi.acm.org/10.1145/1124772.1124782},
249   - doi = {10.1145/1124772.1124782},
250   - acmid = {1124782},
251   - publisher = {ACM},
252   - address = {New York, NY, USA},
253   - keywords = {computer vision, distributed knowledge acquisition, object recognition, object segmentation, web-based games},
254   -}
  291 + Acmid = {1124782},
  292 + Address = {New York, NY, USA},
  293 + Author = {von Ahn, Luis and Liu, Ruoran and Blum, Manuel},
  294 + Booktitle = {Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
  295 + Doi = {10.1145/1124772.1124782},
  296 + Isbn = {1-59593-372-7},
  297 + Keywords = {computer vision, distributed knowledge acquisition, object recognition, object segmentation, web-based games},
  298 + Location = {Montr\&\#233;al, Qu\&\#233;bec, Canada},
  299 + Numpages = {10},
  300 + Pages = {55--64},
  301 + Publisher = {ACM},
  302 + Series = {CHI '06},
  303 + Title = {Peekaboom: A Game for Locating Objects in Images},
  304 + Url = {http://doi.acm.org/10.1145/1124772.1124782},
  305 + Year = {2006},
  306 + Bdsk-Url-1 = {http://doi.acm.org/10.1145/1124772.1124782},
  307 + Bdsk-Url-2 = {http://dx.doi.org/10.1145/1124772.1124782}}
255 308  
256 309 @inproceedings{DBLP:conf/aaai/HoCH07,
257 310 Author = {Chien{-}Ju Ho and Tsung{-}Hsiang Chang and Jane Yung{-}jen Hsu},