Commit c1248208cbd9bb3863817becfef94364ea7a6dc2

Authored by waldispuhl
1 parent c43470c5d1
Exists in master

submission to HCOMP

Showing 5 changed files with 675 additions and 672 deletions Side-by-side Diff

paper/HCOMP2015.bib View file @ c124820
  1 +%% This BibTeX bibliography file was created using BibDesk.
  2 +%% http://bibdesk.sourceforge.net/
  3 +
  4 +
  5 +%% Created for Jerome Waldispuhl at 2015-05-02 01:20:36 -0400
  6 +
  7 +
  8 +%% Saved with string encoding Unicode (UTF-8)
  9 +
  10 +
  11 +
  12 +@article{Lorenz:2011aa,
  13 + Abstract = {Social groups can be remarkably smart and knowledgeable when their averaged judgements are compared with the judgements of individuals. Already Galton [Galton F (1907) Nature 75:7] found evidence that the median estimate of a group can be more accurate than estimates of experts. This wisdom of crowd effect was recently supported by examples from stock markets, political elections, and quiz shows [Surowiecki J (2004) The Wisdom of Crowds]. In contrast, we demonstrate by experimental evidence (N = 144) that even mild social influence can undermine the wisdom of crowd effect in simple estimation tasks. In the experiment, subjects could reconsider their response to factual questions after having received average or full information of the responses of other subjects. We compare subjects' convergence of estimates and improvements in accuracy over five consecutive estimation periods with a control condition, in which no information about others' responses was provided. Although groups are initially "wise," knowledge about estimates of others narrows the diversity of opinions to such an extent that it undermines the wisdom of crowd effect in three different ways. The "social influence effect" diminishes the diversity of the crowd without improvements of its collective error. The "range reduction effect" moves the position of the truth to peripheral regions of the range of estimates so that the crowd becomes less reliable in providing expertise for external observers. The "confidence effect" boosts individuals' confidence after convergence of their estimates despite lack of improved accuracy. Examples of the revealed mechanism range from misled elites to the recent global financial crisis.},
  14 + Author = {Lorenz, Jan and Rauhut, Heiko and Schweitzer, Frank and Helbing, Dirk},
  15 + Date-Added = {2015-05-02 05:20:34 +0000},
  16 + Date-Modified = {2015-05-02 05:20:34 +0000},
  17 + Doi = {10.1073/pnas.1008636108},
  18 + Journal = {Proc Natl Acad Sci U S A},
  19 + Journal-Full = {Proceedings of the National Academy of Sciences of the United States of America},
  20 + Mesh = {Commerce; Game Theory; Humans; Intelligence; Judgment; Models, Statistical; Politics; Regression Analysis; Research Design; Social Behavior; Switzerland},
  21 + Month = {May},
  22 + Number = {22},
  23 + Pages = {9020-5},
  24 + Pmc = {PMC3107299},
  25 + Pmid = {21576485},
  26 + Pst = {ppublish},
  27 + Title = {How social influence can undermine the wisdom of crowd effect},
  28 + Volume = {108},
  29 + Year = {2011},
  30 + Bdsk-Url-1 = {http://dx.doi.org/10.1073/pnas.1008636108}}
  31 +
  32 +@article{Kearns:2006aa,
  33 + Abstract = {Theoretical work suggests that structural properties of naturally occurring networks are important in shaping behavior and dynamics. However, the relationships between structure and behavior are difficult to establish through empirical studies, because the networks in such studies are typically fixed. We studied networks of human subjects attempting to solve the graph or network coloring problem, which models settings in which it is desirable to distinguish one's behavior from that of one's network neighbors. Networks generated by preferential attachment made solving the coloring problem more difficult than did networks based on cyclical structures, and "small worlds" networks were easier still. We also showed that providing more information can have opposite effects on performance, depending on network structure.},
  34 + Author = {Kearns, Michael and Suri, Siddharth and Montfort, Nick},
  35 + Date-Added = {2015-05-02 04:41:32 +0000},
  36 + Date-Modified = {2015-05-02 04:41:32 +0000},
  37 + Doi = {10.1126/science.1127207},
  38 + Journal = {Science},
  39 + Journal-Full = {Science (New York, N.Y.)},
  40 + Mesh = {Game Theory; Group Processes; Humans; Interpersonal Relations; Social Behavior; Systems Theory},
  41 + Month = {Aug},
  42 + Number = {5788},
  43 + Pages = {824-7},
  44 + Pmid = {16902134},
  45 + Pst = {ppublish},
  46 + Title = {An experimental study of the coloring problem on human subject networks},
  47 + Volume = {313},
  48 + Year = {2006},
  49 + Bdsk-Url-1 = {http://dx.doi.org/10.1126/science.1127207}}
  50 +
  51 +@article{DBLP:journals/cacm/Kearns12,
  52 + Author = {Michael Kearns},
  53 + Bibsource = {dblp computer science bibliography, http://dblp.org},
  54 + Biburl = {http://dblp.uni-trier.de/rec/bib/journals/cacm/Kearns12},
  55 + Date-Added = {2015-05-02 04:40:15 +0000},
  56 + Date-Modified = {2015-05-02 04:40:15 +0000},
  57 + Doi = {10.1145/2347736.2347753},
  58 + Journal = {Commun. {ACM}},
  59 + Number = {10},
  60 + Pages = {56--67},
  61 + Timestamp = {Sun, 21 Oct 2012 17:30:12 +0200},
  62 + Title = {Experiments in social computation},
  63 + Url = {http://doi.acm.org/10.1145/2347736.2347753},
  64 + Volume = {55},
  65 + Year = {2012},
  66 + Bdsk-Url-1 = {http://doi.acm.org/10.1145/2347736.2347753},
  67 + Bdsk-Url-2 = {http://dx.doi.org/10.1145/2347736.2347753}}
  68 +
  69 +
  70 +@article{Buhrmester01012011,
  71 +author = {Buhrmester, Michael and Kwang, Tracy and Gosling, Samuel D.},
  72 +title = {Amazon's Mechanical Turk: A New Source of Inexpensive, Yet High-Quality, Data?},
  73 +volume = {6},
  74 +number = {1},
  75 +pages = {3-5},
  76 +year = {2011},
  77 +doi = {10.1177/1745691610393980},
  78 +abstract ={Amazon’s Mechanical Turk (MTurk) is a relatively new website that contains the major elements required to conduct research: an integrated participant compensation system; a large participant pool; and a streamlined process of study design, participant recruitment, and data collection. In this article, we describe and evaluate the potential contributions of MTurk to psychology and other social sciences. Findings indicate that (a) MTurk participants are slightly more demographically diverse than are standard Internet samples and are significantly more diverse than typical American college samples; (b) participation is affected by compensation rate and task length, but participants can still be recruited rapidly and inexpensively; (c) realistic compensation rates do not affect data quality; and (d) the data obtained are at least as reliable as those obtained via traditional methods. Overall, MTurk can be used to obtain high-quality data inexpensively and rapidly.},
  79 +URL = {http://pps.sagepub.com/content/6/1/3.abstract},
  80 +eprint = {http://pps.sagepub.com/content/6/1/3.full.pdf+html},
  81 +journal = {Perspectives on Psychological Science}
  82 +}
  83 +
  84 +@article{Paolacci,
  85 +author = {Paolacci, Gabriele and Chandler, Jesse and Ipeirotis, Panagiotis G.},
  86 +title = {Running Experiments on Amazon Mechanical Turk},
  87 +volume = {5},
  88 +number = {5},
  89 +pages = {411-419},
  90 +year = {2010},
  91 +journal = {Judgment and Decision Making}
  92 +}
  93 +
  94 +@online{Crowdcrafting,
  95 + author = {Lombrana, Daniel and Reimer, Marvin and Dominguez, Alejandro and Doherty, James and Correa, Jorge and Sanchez-Puga, Clara and Suarez Perez, Alvaro},
  96 + title = {http://crowdcrafting.org/},
  97 + year = 2015,
  98 + url = {http://crowdcrafting.org/},
  99 + urldate = {2015-05-01}
  100 +}
paper/MarketPaper.aux View file @ c124820
1   -\relax
2   -\newlabel{fig_interface}{{1}{2}}
3   -\newlabel{fig_totalXP}{{2}{3}}
4   -\newlabel{fig_averageSeqLength}{{3}{4}}
5   -\newlabel{tab_buyoutKing}{{1}{4}}
6   -\newlabel{tab_masterTrader}{{2}{4}}
7   -\newlabel{tab_colorExpert}{{3}{4}}
8   -\newlabel{tab_colorExpert2}{{4}{5}}
9   -\newlabel{tab_seqCollector}{{5}{5}}
10   -\newlabel{tab_seqCollector2}{{6}{5}}
11   -\newlabel{fig_sellBuySNP}{{4}{5}}
12   -\newlabel{fig_minNbCols}{{5}{6}}
13   -\newlabel{fig_minSeqLength}{{6}{6}}
paper/MarketPaper.log View file @ c124820
1   -This is pdfTeX, Version 3.1415926-2.5-1.40.14 (MiKTeX 2.9) (preloaded format=pdflatex 2015.4.9) 1 MAY 2015 17:59
2   -entering extended mode
3   -**MarketPaper
4   -(C:\Users\Olivier\Documents\MarketGame\paper\Version3\MarketPaper.tex
5   -LaTeX2e <2011/06/27>
6   -Babel <v3.8m> and hyphenation patterns for english, afrikaans, ancientgreek, ar
7   -abic, armenian, assamese, basque, bengali, bokmal, bulgarian, catalan, coptic,
8   -croatian, czech, danish, dutch, esperanto, estonian, farsi, finnish, french, ga
9   -lician, german, german-x-2013-05-26, greek, gujarati, hindi, hungarian, iceland
10   -ic, indonesian, interlingua, irish, italian, kannada, kurmanji, latin, latvian,
11   - lithuanian, malayalam, marathi, mongolian, mongolianlmc, monogreek, ngerman, n
12   -german-x-2013-05-26, nynorsk, oriya, panjabi, pinyin, polish, portuguese, roman
13   -ian, russian, sanskrit, serbian, slovak, slovenian, spanish, swedish, swissgerm
14   -an, tamil, telugu, turkish, turkmen, ukenglish, ukrainian, uppersorbian, usengl
15   -ishmax, welsh, loaded.
16   -("C:\Program Files (x86)\MiKTeX 2.9\tex\latex\base\article.cls"
17   -Document Class: article 2007/10/19 v1.4h Standard LaTeX document class
18   -("C:\Program Files (x86)\MiKTeX 2.9\tex\latex\base\size10.clo"
19   -File: size10.clo 2007/10/19 v1.4h Standard LaTeX file (size option)
20   -)
21   -\c@part=\count79
22   -\c@section=\count80
23   -\c@subsection=\count81
24   -\c@subsubsection=\count82
25   -\c@paragraph=\count83
26   -\c@subparagraph=\count84
27   -\c@figure=\count85
28   -\c@table=\count86
29   -\abovecaptionskip=\skip41
30   -\belowcaptionskip=\skip42
31   -\bibindent=\dimen102
32   -)
33   -(C:\Users\Olivier\Documents\MarketGame\paper\Version3\aaai.sty
34   -Conference Style for AAAI for LaTeX 2e -- version of 1 January 2015
35   -\titlebox=\skip43
36   -) ("C:\Program Files (x86)\MiKTeX 2.9\tex\latex\psnfss\times.sty"
37   -Package: times 2005/04/12 PSNFSS-v9.2a (SPQR)
38   -)
39   -("C:\Program Files (x86)\MiKTeX 2.9\tex\latex\psnfss\helvet.sty"
40   -Package: helvet 2005/04/12 PSNFSS-v9.2a (WaS)
41   -
42   -("C:\Program Files (x86)\MiKTeX 2.9\tex\latex\graphics\keyval.sty"
43   -Package: keyval 1999/03/16 v1.13 key=value parser (DPC)
44   -\KV@toks@=\toks14
45   -))
46   -("C:\Program Files (x86)\MiKTeX 2.9\tex\latex\psnfss\courier.sty"
47   -Package: courier 2005/04/12 PSNFSS-v9.2a (WaS)
48   -)
49   -("C:\Program Files (x86)\MiKTeX 2.9\tex\latex\graphics\graphicx.sty"
50   -Package: graphicx 1999/02/16 v1.0f Enhanced LaTeX Graphics (DPC,SPQR)
51   -
52   -("C:\Program Files (x86)\MiKTeX 2.9\tex\latex\graphics\graphics.sty"
53   -Package: graphics 2009/02/05 v1.0o Standard LaTeX Graphics (DPC,SPQR)
54   -
55   -("C:\Program Files (x86)\MiKTeX 2.9\tex\latex\graphics\trig.sty"
56   -Package: trig 1999/03/16 v1.09 sin cos tan (DPC)
57   -)
58   -("C:\Program Files (x86)\MiKTeX 2.9\tex\latex\00miktex\graphics.cfg"
59   -File: graphics.cfg 2007/01/18 v1.5 graphics configuration of teTeX/TeXLive
60   -)
61   -Package graphics Info: Driver file: pdftex.def on input line 91.
62   -
63   -("C:\Program Files (x86)\MiKTeX 2.9\tex\latex\pdftex-def\pdftex.def"
64   -File: pdftex.def 2011/05/27 v0.06d Graphics/color for pdfTeX
65   -
66   -("C:\Program Files (x86)\MiKTeX 2.9\tex\generic\oberdiek\infwarerr.sty"
67   -Package: infwarerr 2010/04/08 v1.3 Providing info/warning/error messages (HO)
68   -)
69   -("C:\Program Files (x86)\MiKTeX 2.9\tex\generic\oberdiek\ltxcmds.sty"
70   -Package: ltxcmds 2011/11/09 v1.22 LaTeX kernel commands for general use (HO)
71   -)
72   -\Gread@gobject=\count87
73   -))
74   -\Gin@req@height=\dimen103
75   -\Gin@req@width=\dimen104
76   -)
77   -(C:\Users\Olivier\Documents\MarketGame\paper\Version3\MarketPaper.aux)
78   -LaTeX Font Info: Checking defaults for OML/cmm/m/it on input line 21.
79   -LaTeX Font Info: ... okay on input line 21.
80   -LaTeX Font Info: Checking defaults for T1/cmr/m/n on input line 21.
81   -LaTeX Font Info: ... okay on input line 21.
82   -LaTeX Font Info: Checking defaults for OT1/cmr/m/n on input line 21.
83   -LaTeX Font Info: ... okay on input line 21.
84   -LaTeX Font Info: Checking defaults for OMS/cmsy/m/n on input line 21.
85   -LaTeX Font Info: ... okay on input line 21.
86   -LaTeX Font Info: Checking defaults for OMX/cmex/m/n on input line 21.
87   -LaTeX Font Info: ... okay on input line 21.
88   -LaTeX Font Info: Checking defaults for U/cmr/m/n on input line 21.
89   -LaTeX Font Info: ... okay on input line 21.
90   -LaTeX Font Info: Try loading font information for OT1+ptm on input line 21.
91   -
92   -("C:\Program Files (x86)\MiKTeX 2.9\tex\latex\psnfss\ot1ptm.fd"
93   -File: ot1ptm.fd 2001/06/04 font definitions for OT1/ptm.
94   -)
95   -("C:\Program Files (x86)\MiKTeX 2.9\tex\context\base\supp-pdf.mkii"
96   -[Loading MPS to PDF converter (version 2006.09.02).]
97   -\scratchcounter=\count88
98   -\scratchdimen=\dimen105
99   -\scratchbox=\box26
100   -\nofMPsegments=\count89
101   -\nofMParguments=\count90
102   -\everyMPshowfont=\toks15
103   -\MPscratchCnt=\count91
104   -\MPscratchDim=\dimen106
105   -\MPnumerator=\count92
106   -\makeMPintoPDFobject=\count93
107   -\everyMPtoPDFconversion=\toks16
108   -)
109   -LaTeX Font Info: Font shape `OT1/ptm/bx/n' in size <14.4> not available
110   -(Font) Font shape `OT1/ptm/b/n' tried instead on input line 29.
111   -LaTeX Font Info: External font `cmex10' loaded for size
112   -(Font) <7> on input line 29.
113   -LaTeX Font Info: External font `cmex10' loaded for size
114   -(Font) <5> on input line 29.
115   -LaTeX Font Info: Font shape `OT1/ptm/bx/n' in size <12> not available
116   -(Font) Font shape `OT1/ptm/b/n' tried instead on input line 29.
117   -LaTeX Font Info: Try loading font information for OMS+ptm on input line 29.
118   - ("C:\Program Files (x86)\MiKTeX 2.9\tex\latex\psnfss\omsptm.fd"
119   -File: omsptm.fd
120   -)
121   -LaTeX Font Info: Font shape `OMS/ptm/m/n' in size <9> not available
122   -(Font) Font shape `OMS/cmsy/m/n' tried instead on input line 29.
123   -LaTeX Font Info: Font shape `OT1/ptm/bx/n' in size <10> not available
124   -(Font) Font shape `OT1/ptm/b/n' tried instead on input line 30.
125   -LaTeX Font Info: Font shape `OT1/ptm/bx/n' in size <10.95> not available
126   -(Font) Font shape `OT1/ptm/b/n' tried instead on input line 54.
127   -
128   -<Figs/interface_mod.png, id=1, 941.01563pt x 545.78906pt>
129   -File: Figs/interface_mod.png Graphic file (type png)
130   -
131   -<use Figs/interface_mod.png>
132   -Package pdftex.def Info: Figs/interface_mod.png used on input line 65.
133   -(pdftex.def) Requested size: 505.89pt x 293.41418pt.
134   - [1{C:/ProgramData/MiKTeX/2.9/pdftex/config/pdftex.map}
135   -
136   -
137   -
138   -] [2 <C:/Users/Olivier/Documents/MarketGame/paper/Version3/Figs/interface_mod.p
139   -ng>] <Figs/totalXP_session.pdf, id=18, 361.35pt x 216.81pt>
140   -File: Figs/totalXP_session.pdf Graphic file (type pdf)
141   -
142   -<use Figs/totalXP_session.pdf>
143   -Package pdftex.def Info: Figs/totalXP_session.pdf used on input line 127.
144   -(pdftex.def) Requested size: 252.94499pt x 151.76596pt.
145   -
146   -Overfull \hbox (13.55061pt too wide) in paragraph at lines 127--132
147   - []
148   - []
149   -
150   -<Figs/averageSeqLength.pdf, id=19, 361.35pt x 216.81pt>
151   -File: Figs/averageSeqLength.pdf Graphic file (type pdf)
152   -
153   -<use Figs/averageSeqLength.pdf>
154   -Package pdftex.def Info: Figs/averageSeqLength.pdf used on input line 166.
155   -(pdftex.def) Requested size: 252.94499pt x 151.76596pt.
156   -
157   -Overfull \hbox (13.55061pt too wide) in paragraph at lines 166--172
158   - []
159   - []
160   -
161   -[3 <C:/Users/Olivier/Documents/MarketGame/paper/Version3/Figs/totalXP_session.p
162   -df>]
163   -Overfull \hbox (7.67398pt too wide) in paragraph at lines 199--204
164   - []
165   - []
166   -
167   -
168   -Overfull \hbox (7.67398pt too wide) in paragraph at lines 225--230
169   - []
170   - []
171   -
172   -
173   -Overfull \hbox (20.45439pt too wide) in paragraph at lines 249--254
174   - []
175   - []
176   -
177   -[4 <C:/Users/Olivier/Documents/MarketGame/paper/Version3/Figs/averageSeqLength.
178   -pdf>]
179   -Overfull \hbox (26.68245pt too wide) in paragraph at lines 270--275
180   - []
181   - []
182   -
183   -
184   -Overfull \hbox (8.5042pt too wide) in paragraph at lines 286--291
185   - []
186   - []
187   -
188   -
189   -Overfull \hbox (5.57208pt too wide) in paragraph at lines 306--311
190   - []
191   - []
192   -
193   -
194   -LaTeX Warning: `h' float specifier changed to `ht'.
195   -
196   -<Figs/sellBuySNP.pdf, id=44, 361.35pt x 216.81pt>
197   -File: Figs/sellBuySNP.pdf Graphic file (type pdf)
198   - <use Figs/sellBuySNP.pdf>
199   -Package pdftex.def Info: Figs/sellBuySNP.pdf used on input line 335.
200   -(pdftex.def) Requested size: 252.94499pt x 151.76596pt.
201   -
202   -Overfull \hbox (13.55061pt too wide) in paragraph at lines 335--339
203   - []
204   - []
205   -
206   -[5 <C:/Users/Olivier/Documents/MarketGame/paper/Version3/Figs/sellBuySNP.pdf>]
207   -<Figs/minNbCols.pdf, id=57, 361.35pt x 216.81pt>
208   -File: Figs/minNbCols.pdf Graphic file (type pdf)
209   - <use Figs/minNbCols.pdf>
210   -Package pdftex.def Info: Figs/minNbCols.pdf used on input line 355.
211   -(pdftex.def) Requested size: 252.94499pt x 151.76596pt.
212   -
213   -Overfull \hbox (13.55061pt too wide) in paragraph at lines 355--360
214   - []
215   - []
216   -
217   -<Figs/minSeqLength.pdf, id=58, 361.35pt x 216.81pt>
218   -File: Figs/minSeqLength.pdf Graphic file (type pdf)
219   -
220   -<use Figs/minSeqLength.pdf>
221   -Package pdftex.def Info: Figs/minSeqLength.pdf used on input line 378.
222   -(pdftex.def) Requested size: 252.94499pt x 151.76596pt.
223   -
224   -Overfull \hbox (13.55061pt too wide) in paragraph at lines 378--383
225   - []
226   - []
227   -
228   -[6 <C:/Users/Olivier/Documents/MarketGame/paper/Version3/Figs/minNbCols.pdf> <C
229   -:/Users/Olivier/Documents/MarketGame/paper/Version3/Figs/minSeqLength.pdf
230   -
231   -pdfTeX warning: pdflatex (file C:/Users/Olivier/Documents/MarketGame/paper/Vers
232   -ion3/Figs/minSeqLength.pdf): PDF inclusion: multiple pdfs with page group inclu
233   -ded in a single page
234   ->] (C:\Users\Olivier\Documents\MarketGame\paper\Version3\MarketPaper.aux) )
235   -Here is how much of TeX's memory you used:
236   - 1276 strings out of 493921
237   - 17873 string characters out of 3144871
238   - 75014 words of memory out of 3000000
239   - 4586 multiletter control sequences out of 15000+200000
240   - 16774 words of font info for 32 fonts, out of 3000000 for 9000
241   - 841 hyphenation exceptions out of 8191
242   - 27i,10n,21p,250b,248s stack positions out of 5000i,500n,10000p,200000b,50000s
243   -{C:/
244   -Program Files (x86)/MiKTeX 2.9/fonts/enc/dvips/fontname/8r.enc}<C:/Program File
245   -s (x86)/MiKTeX 2.9/fonts/type1/public/amsfonts/cm/cmmi10.pfb><C:/Program Files
246   -(x86)/MiKTeX 2.9/fonts/type1/public/amsfonts/cm/cmmi7.pfb><C:/Program Files (x8
247   -6)/MiKTeX 2.9/fonts/type1/public/amsfonts/cm/cmr10.pfb><C:/Program Files (x86)/
248   -MiKTeX 2.9/fonts/type1/public/amsfonts/cm/cmsy10.pfb><C:/Program Files (x86)/Mi
249   -KTeX 2.9/fonts/type1/public/amsfonts/cm/cmsy7.pfb><C:/Program Files (x86)/MiKTe
250   -X 2.9/fonts/type1/public/amsfonts/cm/cmsy9.pfb><C:/Program Files (x86)/MiKTeX 2
251   -.9/fonts/type1/urw/times/utmb8a.pfb><C:/Program Files (x86)/MiKTeX 2.9/fonts/ty
252   -pe1/urw/times/utmr8a.pfb><C:/Program Files (x86)/MiKTeX 2.9/fonts/type1/urw/tim
253   -es/utmri8a.pfb>
254   -Output written on MarketPaper.pdf (6 pages, 206869 bytes).
255   -PDF statistics:
256   - 108 PDF objects out of 1000 (max. 8388607)
257   - 0 named destinations out of 1000 (max. 500000)
258   - 31 words of extra memory for PDF output out of 10000 (max. 10000000)
259   -
paper/MarketPaper.pdf View file @ c124820

No preview for this file type

paper/MarketPaper.tex View file @ c124820
... ... @@ -8,6 +8,7 @@
8 8 \frenchspacing
9 9 \setlength{\pdfpagewidth}{8.5in}
10 10 \setlength{\pdfpageheight}{11in}
  11 +\usepackage[numbers]{natbib}
11 12  
12 13 %ADDED
13 14 \usepackage[pdftex]{graphicx}
14 15  
15 16  
... ... @@ -29,11 +30,26 @@
29 30 \maketitle
30 31 \begin{abstract}
31 32 \begin{quote}
32   -Abstract.
  33 +Using a human computing game to solve a problem that has a large search space is not straightforward. The difficulty of using such an approach
  34 +comes from the following facts: (1) it would be overwhelming for a player to show him or her the complete search space and at the same time,
  35 +(2) it is impossible to find an optimal solution without considering all the available data. In this paper, we present a human computing
  36 +game that uses a market, skills and a challenge system to help the players solve a graph problem in a collaborative manner. The results obtained
  37 +during five game sessions of 10 players show that the market helps players to build larger solutions. We also show that a skill and a
  38 +challenge system can be used to influence and guide the players towards producing better solutions.
33 39 \end{quote}
34 40 \end{abstract}
35 41  
  42 +\section{Introduction}
  43 +Human-computation and crowd-sourcing are now perceived as valuable techniques to help solving difficult computational problems. In order to make the best use of human skills in these systems, it is important to be able to characterize the expertise and performance of humans as individual and even more importantly as groups.
36 44  
  45 +Currently, popular crowd-computing platform such as Amazon Mechanical Turk~\cite{Buhrmester01012011, Paolacci} or Crowdcrafting~\cite{Crowdcrafting} are based on similar divide-and-conquer architectures, where the initial problem is decomposed into smaller sub-tasks that are distributed to individual workers and then aggregated to build a solution. In particular, these systems prevent any interaction between workers in order to prevent groupthink phenomena and bias in the solution \cite{Lorenz:2011aa}.
  46 +
  47 +However, such constraints are necessarily limiting the capacity of the system to harness the cognitive power of crowds and make full benefit of collective intelligence. In order to gain expressivity and improve their performance, the next generation of human-computation platforms will undoubtedly need to relax these constraints and build market systems in which workers can collaborate. Nonetheless, before transitioning to this new model, it is important to first estimate the gain of productivity and quantify the usefulness of the mechanisms and incentives to promote collaborative solving and prevent groupthink.
  48 +
  49 +Historically, computation on graphs has proven to be a good model to study the performance of humans in solving complex combinatorial problems \cite{Kearns:2006aa}. Experiments have been conducted to evaluate the dynamics of crowds collaborating at solving graph problems \cite{DBLP:journals/cacm/Kearns12} but still, little is known about the efficiency of various modes of interaction.
  50 +
  51 +In this paper, we propose a formal framework to study human collaborative solving. We design a market system coupled with skills and a challenge system to help the players solve combinatorial graph problems. In order to prevent any bias, we implement this system as a game that makes abstraction the graphical nature of the underlying problem.
  52 +
37 53 \section{Problem}
38 54  
39 55 The game was implemented to solve a graph problem, which is the problem of finding maximal cliques in a multigraph.
40 56  
41 57  
... ... @@ -54,12 +70,84 @@
54 70 \subsection{Goal of the game}
55 71  
56 72 The main objective of the game is to build {\em sequences} ({\em i.e.} sets) of circles that are as long as possible and contain as many colors in common
57   -as possible.
  73 +as possible. Circles used by the players to build the sequences either come random packages bought from the system or they come from another player
  74 +through the market. The sequences can then be sold to the system for a certain amount of game money, which is determined by a scoring function that takes into
  75 +account the length and the number of colors in common of the sequence.
58 76  
59 77 \subsection{Scoring function}
60 78  
  79 +The score of a sequence sold to the system is equal to $baseScore_{n} * seqLength^2$, where $baseScore_{n}$ is a base score depending on the number
  80 +of colors in common (see Table~\ref{tab_baseScore}) and $seqLength$ is the length of the sequence. The base scores were calculated based on the exact solution
  81 +for the graph that was generated for the tests (see section Generating the graph for a description of the graph that was used) in such a way to give a reward
  82 +that is proportional to the difficulty of building the sequence. More precisely,
  83 +we calculated the average length $L_n$ of all solutions for each $n$ number of colors. The base score is simply the reciprocal of this average ($1/L_n$) multiplied
  84 +by a balancing factor (505 in our case). The balancing factor was chosen in order to get a score of 500 for a sequence of length 10 with only one color in common,
  85 +which is exactly the price of two random packages of circles.
  86 +
  87 +\begin{table}[h]
  88 +\caption{Value of the base score depending on the number of colors in common}\label{tab_baseScore}
  89 +\begin{center}
  90 +\begin{tabular}{cc}\hline
  91 +Number of colors & Base score\\
  92 +0 & 0\\
  93 +1 & 5\\
  94 +2 & 14\\
  95 +3 & 26\\
  96 +4 & 40\\
  97 +5 & 55\\
  98 +6 & 72\\\hline
  99 +\end{tabular}
  100 +\end{center}
  101 +\end{table}
  102 +
61 103 \subsection{Game interface}
62 104  
  105 +The game client and the server were built in Java 1.7.
  106 +As shown in Figure~\ref{fig_interface}, the game interface can be divided into 3 parts: the player information panel, the game panel and the market panel.
  107 +
  108 +\subsubsection{Player information panel}
  109 +
  110 +This panel simply contains information on the player's wallet, the current level of the player and has three buttons, allowing the player to open
  111 +dialogs showing information on the current challenge, the skills (see section Skills for a description of the available skills) and the leaderboard.
  112 +One experience point is given to the player for each game dollar that he/she wins. The player can lose game money, but cannot lose experience points
  113 +(experience points can only go up).
  114 +
  115 +\subsubsection{Game panel}
  116 +
  117 +The first component of the game panel is the 'My sequence' panel, which shows the current sequence that is being built by the player. The maximum size
  118 +of a sequence is 10. Colors in common in the
  119 +sequence are indicated by a thick black border surrounding the colors in the circles. Players can use the arrows to switch between the different sequence slots
  120 +(2 sequence slots are available at the start of the game). The current value of the sequence is shown at the right, and the price for adding one more circle
  121 +with the same colors in common is shown right below in gray. Finally, the sell button allows the player to sell the current sequence to the system: the sequence then
  122 +disappears and the money is given to the player.
  123 +
  124 +The second component is the 'My hand' panel, which can contain up to 20 circles. Players can add a circle to the sequence by clicking on it. Circles are
  125 +represented by their colors and by a price label (in a black box). The price corresponds to the current value of the circle on the market. Clicking on
  126 +the price label sells the circle to the highest bidder on the market. Circles that are bought from a random package or from other players are sent to
  127 +the hand.
  128 +
  129 +The 'Awaiting to get sold' is where the circles are sent just before being sold to the highest bidder. If the bid disappears before the transaction is completed,
  130 +the 'sold' circle will stay there. The player can then click on it to cancel the selling and put it back in the hand.
  131 +
  132 +Finally, the bottom panel is a news feed, showing information on the game state, like the remaining time to complete the challenge and the last transactions
  133 +completed by the player for example.
  134 +
  135 +\subsubsection{Market panel}
  136 +
  137 +At the top of the market panel, buttons allow the player to create bids for circles or to buy random packages (or bags) of circles.
  138 +The 'Random bag' costs \$250 and contains 5 circles with fewer colors. The 'Premium bag' costs \$500 and contains 5 circles
  139 +with a higher chance of getting circles with many colors.
  140 +
  141 +Right below the buttons is the 'Automatic bids' panel, which allows the player to get automatic bids for circles corresponding to the sequences that
  142 +he or she is building. A percentage of profit for the price of the automatic bids can be set with the slider.
  143 +The profit is defined as the money the player would make by adding one more SNP with the same colors in the current sequence (difference between the gray and black prices).
  144 +
  145 +The 'My bids' panel shows all the bids that the player currently has on the market. The bid price is shown below the circle (in the black box). On the right side
  146 +of the circle is the number of sequences with the same colors that the player can buy from other players (in the blue box).
  147 +Clicking on the blue box opens a window showing the list of sequences that can be bought. Buying a sequence from another player is called a 'buyout'.
  148 +
  149 +Finally, the last panel at the bottom shows the last circle or sequence that was bought by the player.
  150 +
63 151 \begin{figure*}[htbp]
64 152 \begin{center}
65 153 \includegraphics[width=\textwidth]{Figs/interface_mod.png}
66 154  
67 155  
... ... @@ -72,10 +160,51 @@
72 160  
73 161 \subsection{Market}
74 162  
  163 +The market has three functions: allow the players to exchange circles through a bidding system, allow players to buy sequences built and sold by other
  164 +players so that they can be improved, and merge together sequences of length 10 to create super circles that are then put back in the game.
  165 +
  166 +For every subset of colors, the server has a list of all the bids that are currently on the market. The value of
  167 +the highest bid on the market is shown below every circle under the possession of the players. When a circle is sold by a player, it is sent through
  168 +the server to the highest bidder.
  169 +
  170 +Buyouts work differently. Players cannot bid on sequences, but the server holds for two minutes all the sequence that have been sold by the players.
  171 +During those two minutes, other players can buy the sequences for a price that is equal to 150\% of the initial score of the sequence. When a buyout
  172 +is made, the bonus game money is sent to the player who initially sold the sequence to the system.
  173 +
  174 +Finally, the game system creates a super circle every time a sequence of length 10 is sold by a player. A super circle of level 2 (representing 10 circles)
  175 +counts as two circles when put in a sequence. Super circles can be of any level (a sequence of 10 super circles of level 2 form a super circle of level 3, and so on).
  176 +The objective is to remove the limitation of the maximum sequence size imposed by the game interface.
  177 +
75 178 \subsection{Skills}
76 179  
  180 +Four different skills were implemented in the game. One skill point is awarded to a player when he or she levels up, which can then be put in any
  181 +of the four skills. The maximum level of each skill is equal to six (there are six levels of bonuses). Each skill was put in the game as a way to
  182 +guide the player in doing actions that are beneficial to the system or to the other players:
  183 +
  184 +\begin{itemize}
  185 +\item {\em Buyout King}: lowers the price of buying a sequence from another player;
  186 +\item {\em Color Expert}: gives a bonus to selling sequences that have more than one color in common;
  187 +\item {\em Sequence Collector}: gives an additional sequence slot;
  188 +\item {\em Master Trader}: gives a bonus to selling SNPs to other players.
  189 +\end{itemize}
  190 +
77 191 \subsection{Challenge system}
78 192  
  193 +We implemented a challenge system that analyzes the recent actions of the players and creates a new challenge every five minutes. The five challenge
  194 +types are:
  195 +
  196 +\begin{itemize}
  197 +\item {\em Sell/buy circles}: requires the players to sell or buy circles;
  198 +\item {\em Buyout sequences}: requires the players to buy sequences from other players;
  199 +\item {\em Minimum number of colors}: requires the players to sell sequences with at least a certain number of colors in common;
  200 +\item {\em Minimum sequence length}: requires the players to sell sequences with a minimum sequence length;
  201 +\item {\em Specific colors in common}: requires the players to sell sequences with a specific subset of colors in common.
  202 +\end{itemize}
  203 +
  204 +Basically, the system continuously monitors the activities of the players and decreases or increases the probabilities of each challenge type.
  205 +The next challenge is then selected using a multinomial sampling on these probabilities. The number of times $T$ that the challenge-related action must be
  206 +completed is selected randomly between 3 and 5. The prize that is awarded for completing the challenge is equal to $1500 * T$.
  207 +
79 208 \section{Experiments}
80 209  
81 210 We recruited 50 people in total to test our game. We divided the participants into groups of 10 and made each of the following four tests with a different group.
... ... @@ -396,6 +525,52 @@
396 525 This can be explained by the fact that it was the hardest challenge. All the other challenges are more general and can be completed by
397 526 doing actions that are not specific to a certain subset of colors. Even if the market should be helpful in finding circles with the required
398 527 subset of colors, it seems highly probable that the players felt that this type of challenge was too hard and never tried to complete it.
  528 +
  529 +\subsection{Understanding what makes a good player}
  530 +
  531 +Based on the questionnaire filled by the players before playing the game, and the global leaderboard of all the players from all the sessions put together,
  532 +we tried to find similarities between the top players. Table~\ref{tab_playerStats} shows the most interesting differences between the top six players
  533 +and the rest of the players. In the questionnaires, players had to indicate their age category (between 21 and 25 for example), their own evaluation
  534 +of their puzzle solving abilities and a range of hours of time spent playing video games every week. The mean age of the two groups of players
  535 +was calculated by taking the middle point of the age categories. The average age of the top 6 players was about 5 years younger than the one of
  536 +the other players. For the puzzle solving self evaluation, the players could choose a level between 1 and 5 (5 being the strongest). The average
  537 +level of the top 6 players was 3.83, compared to 2.81 for the others. As with the age categories, we computed averages of time spent playing
  538 +video games every week using the middle point of the categories. The top six players were playing roughly 3 times more every week than the
  539 +rest of the players.
  540 +
  541 +\begin{table}[h]
  542 +\caption{Average statistics on the top six players vs the others}\label{tab_playerStats}
  543 +\begin{center}
  544 +\begin{tabular}{ccc}\hline
  545 + & Top 6 players & Others\\
  546 +Age & 25.50 & 30.33\\
  547 +Self evaluation & 3.83 & 2.81\\
  548 +Game time & 10.42 & 3.20\\\hline
  549 +\end{tabular}
  550 +\end{center}
  551 +\end{table}
  552 +
  553 +\section{Conclusion}
  554 +
  555 +We implemented a human computing game that uses a market, skills and challenges in order to solve a problem collaboratively. The problem that is solved
  556 +by the players in our game is a graph problem that can be easily translated into a color matching game. The total number of colors used in the tests was small
  557 +enough so that we were able to compute an exact solution and evaluate the performance of the players. We organized five game sessions of 10 players with
  558 +different game conditions and to our surprise, the great variability in the participants' skills made it impossible to make direct comparisons between the tests
  559 +in regards to the percentage of the solutions found. However, our tests showed that the market is a useful tool to help players build better solutions
  560 +(longer sequences, in our case). Our
  561 +results also show that skills and challenges systems are helpful tools to inform, influence and guide the players in doing specific actions that are
  562 +beneficial to the system and other players.
  563 +Finally, based on the game sessions that we organized, it seems that younger players who play video games on a regular basis are able to understand the rules
  564 +of the game and find winning strategies faster than the average participant.
  565 +
  566 +\section{Acknowledgments}
  567 +
  568 +The authors would like to thank Jean-Fran\c{c}ois Bourbeau, Mathieu Blanchette, Derek Ruths and Edward Newell for their help with the initial design of the game.
  569 +The authors would also like to thank Silvia Juliana Leon Mantilla for her help with the organization of the game sessions and the recruitment of participants.
  570 +
  571 +\bibliographystyle{unsrt}
  572 +\bibliography{HCOMP2015}
  573 +
399 574  
400 575 \end{document}