-
Notifications
You must be signed in to change notification settings - Fork 3
/
index.html
1219 lines (1009 loc) · 44.1 KB
/
index.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>Autoreject - An Automatic Review Generator</title>
<link href='https://fonts.googleapis.com/css?family=Fira+Mono:400,500,700|Fira+Sans|Staatliches' rel='stylesheet' type='text/css'>
<link rel="stylesheet" href="custom.css">
</head>
<body onload="init()">
<script src="md5.min.js" type="text/javascript"></script>
<h1>Autoreject — An automatic review generator</h1>
<p>
<em>"Detailed reviews at the touch of a button ... Every reviewer should bookmark this immediately."</em> — Science and Review 24:2, April 2019
</p>
<p>
April Fool's Day is over! Before you use this site, check out the <a href="https://github.com/uds-se/autoreject">project page</a>.
</p>
<div id="container" class="flex-container">
<div>
<form id="paper" action="#" method="generate_review" onreset="reset_form()">
<h2>The Paper <span class="checkmark" id="paper_check"></span></h2>
<h3>Title and Area <span class="checkmark" id="title_check"></h3>
<label for="title">Paper title:</label> <input id="title" type="text" placeholder="The Potrzebie System of Weights and Measures" size=52 oninput="update_review()" autofocus><br>
<label for="area">Main Area:</label> <input id="area" type="text" placeholder="Computer Science" size=20 oninput="update_review()">
<label for="field">Subarea:</label> <input id="field" type="text" placeholder="Software Engineering" size=20 oninput="update_review()" required><br>
<script type="text/javascript">
"use strict"
function todo(comment)
{
return "<<< To complete the review, please " + comment + ".\n\n"
}
function review_title(accept)
{
var title = document.getElementById("title").value
var anon = document.getElementById("anon").checked
var anon_i = document.getElementById("anon").indeterminate
var author = document.getElementById("author").value
var institution = document.getElementById("institution").value
var area = document.getElementById("area").value
var field = document.getElementById("field").value
var s = ""
s += "Dear "
if (!anon_i && !anon && author)
{
if (author && institution)
{
s += author + " (" + institution + "),"
}
else
{
s += author + ","
}
}
else
{
s += "Author, "
}
s += "\n\n"
s += "Thank you very much for your submission"
if (title)
{
s += "\n\n '" + title + "'\n\n"
}
else
{
s += ". "
}
if (all_decisions())
{
if (accept)
{
if (!area) {
area = "Science"
}
if (!field) {
field = area
}
s += "We are happy to inform you that your paper was accepted. The venue received a great many submissions of high quality. Each submission was thoroughly and extensively reviewed by our expert panels. After week-long online and offline discussions, we selected a small subset of papers to be accepted. Your paper was one of them.\n\n"
s += "Your paper was not only selected for acceptance but also unanimously voted as best paper of the venue, with a score of 10.0 out of a maximum possible 10.0. As such, your paper will see expedited publishing in the International Journal of " + area + ", the highest-impact journal in the field.\n\n"
s += "We would also be honored if you could attend the Best Paper Award ceremony during the convention, where you will receive the 100,000 GBP " + field + " Achievement Award sponsored by the Ponzi Science Fund. The Award will be given in name and presence of Their Royal Highnesses The Duke and Duchess of Sussex. They have followed your work for a long time, and very much look forward to meet you. The Award comes with a golden plaque commemorating your achievements and a lifetime Amazon Prime subscription.\n\n"
s += "As it comes to your paper, it is perfect as is. Still, we have enclosed detailed reviews of your paper below. We hope that these will help you in your future work and look forward to the final version of your paper!\n\n"
}
else
{
s += "We are sorry to inform you that your paper was rejected. The venue received a great many submissions of high quality. Each submission was thoroughly and extensively reviewed by our expert panels. After week-long online and offline discussions, we selected a small subset of papers to be accepted. Unfortunately, your paper was not one of them.\n\n"
s += "We have enclosed detailed reviews of your paper below. We hope that these will help you in your scientific work and look forward to your future submissions!\n\n"
}
}
else
{
s += "We have enclosed detailed reviews of your paper below.\n\n"
}
return s
}
function review_head(accept)
{
var r = ""
if (accept)
{
r += "# Reviewer 1\n\nFor a long time, this reviewer has not been as excited about a paper as this one. This is a great paper that will stand the test of time. There are a few minor flaws, admitted, but it will mark our community for years to come.\n\n"
}
else
{
r += "# Reviewer 2\n\nGenerally speaking, this is interesting work which may well find some readers in the community. However, in its present form, the paper suffers from a number of issues, which are outlined below.\n\n"
}
return r
}
function review_field(accept)
{
var area = document.getElementById("area").value
var field = document.getElementById("field").value
if (!area && !field) {
return ""
}
var s = "## Area\n\n"
if (!area && field)
area = field
if (accept)
{
s += area + " was, is, and will be one of the hottest areas of current scientific research. Its impact on society cannot be overstated; and this paper condenses the very best of the field.\n\n"
return s
}
if (area)
{
s += area + " is not new. The past decade has seen a flurry of papers in the area, which would indicate progress and interest. However, the impact _outside_ of " + area + " has been very limited, as is easily reflected in the ever-decreasing number of citations. Unfortunately, " + area + " finds itself in a sandwich position between a more theoretical community and a more practical community, which can both claim a bigger intellectual challenge and a greater relevance in practical settings. "
}
if (area && field && area != field)
{
s += "This is especially true for the field of " + field + ", where the last big progress is now decades ago. "
}
s += "As refreshing it feels to finally again see progress claimed in the field, the present paper must be read and evaluated in this context.\n\n"
return s
}
</script>
<h3>Author <span class="checkmark" id="author_check"></span></h3>
<label for="author">Main Author:</label> <input id="author" type="text" placeholder="Donald E. Knuth" size=20 oninput="update_review()">
<label for="institution">Institution:</label> <input id="institution" type="text" placeholder="Stanford" size=20 oninput="update_review()"> <br>
<input id="anon" type="checkbox" onchange="update_review()"> <label for="anon">The submission is anonymous; Author and institution are guesses</label><br>
<input id="rebuttal" onchange="update_author(this)" type="checkbox"> <label for="rebuttal">The authors have provided a substantial rebuttal</label><br> <!-- no effect -->
<script type="text/javascript">
function privileged()
{
var author = document.getElementById("author").value
var institution = document.getElementById("institution").value
return md5(author) == 'e4de39488a8c655c7d001f5c9d72bc1b' || md5(institution) == 'ad3b397e9872725538929fccf26921a3'
}
function update_author(box) {
box.indeterminate = false
update_review()
}
function review_author(accept)
{
var author = document.getElementById("author").value
var institution = document.getElementById("institution").value
var title = document.getElementById("title").value
var anon = document.getElementById("anon").checked
var area = document.getElementById("area").value
if (!area)
{
area = "the field"
}
var s = ""
if (anon && !accept)
{
if (author || institution)
{
s += "## Originality\n\nThe central problem of the paper is its novelty with respect to related work. Just last month, this reviewer attended a talk "
if (title)
{
s += "on " + title + " "
}
if (author)
{
s += "by " + author + " "
}
if (institution)
{
s += "at " + institution + " "
}
s += "where a very similar solution to the exact same problem was presented."
if (author)
{
s += " The paper is actually online at\n\n"
s += " http://scigen.csail.mit.edu/cgi-bin/scigen.cgi?author=" + encodeURI(author)
}
s += "\n\nPlease relate your work carefully against this existing work, precisely highlighting the advances.\n\n"
}
}
else // not anonymous or accept
{
if (author || institution)
{
s += "## Originality\n\nThis paper adds to a long line of research "
if (author)
{
s += "by " + author + " "
}
if (institution)
{
s += "at " + institution + " "
}
if (accept)
{
s += "which is some of the finest research in " + area + ". The present submission can be seen as the cornerstone of this decade-long effort, bringing this work to full completion.\n\n"
}
else
{
s += "which _has_ had some impact in well-delimited fields. The central problem is how the present submission can still add something original to this body of work.\n\n"
}
}
}
return s
}
</script>
<h3>Approach <span class="checkmark" id="approach_check"></span></h3>
<input id="novel" type="checkbox" onchange="update_approach(this)"> <label for="novel">The approach is novel.</label><br>
<input id="incremental" onchange="update_approach(this)" type="checkbox"> <label for="incremental">The approach extends earlier work.</label><br>
<input id="sound" onchange="update_approach(this)" type="checkbox"> <label for="sound">The approach has no false positives.</label><br>
<input id="complete" onchange="update_approach(this)" type="checkbox"> <label for="complete">The approach has no false negatives.</label><br>
<input id="formal" onchange="update_approach(this)" type="checkbox"> <label for="formal">The approach is formally defined.</label><br>
<input id="scales" onchange="update_approach(this)" type="checkbox"> <label for="scales">The approach scales.</label><br>
<input id="inuse" onchange="update_approach(this)" type="checkbox"> <label for="inuse">The approach is in use outside of academia.</label>
<script type="text/javascript">
"use strict"
function update_approach(box) {
box.indeterminate = false
update_review()
}
function any_approach_set() {
var novel_i = document.getElementById("novel").indeterminate
var incremental_i = document.getElementById("incremental").indeterminate
var sound_i = document.getElementById("sound").indeterminate
var complete_i = document.getElementById("complete").indeterminate
var formal_i = document.getElementById("formal").indeterminate
var scales_i = document.getElementById("scales").indeterminate
var inuse_i = document.getElementById("inuse").indeterminate
return !novel_i || !incremental_i || !sound_i || !complete_i || !scales_i || !inuse_i
}
function review_approach(accept) {
var novel = document.getElementById("novel").checked
var incremental = document.getElementById("incremental").checked
var sound = document.getElementById("sound").checked
var complete = document.getElementById("complete").checked
var formal = document.getElementById("formal").checked
var scales = document.getElementById("scales").checked
var inuse = document.getElementById("inuse").checked
var novel_i = document.getElementById("novel").indeterminate
var incremental_i = document.getElementById("incremental").indeterminate
var sound_i = document.getElementById("sound").indeterminate
var complete_i = document.getElementById("complete").indeterminate
var formal_i = document.getElementById("formal").indeterminate
var scales_i = document.getElementById("scales").indeterminate
var inuse_i = document.getElementById("inuse").indeterminate
if (!any_approach_set())
{
return todo("comment on the approach")
}
var s = "## Approach\n\n"
if (accept)
{
s += "The approach is absolutely revolutionary. It is amazing how the authors manage to fit all the details with all clarity within the page limit, yet can come up with realistic examples and a full-fledged evaluation. None of the results leaves the slightest question open. Kudos!\n\n"
return s
}
if (!novel_i)
{
if (novel) {
s += "The approach is novel in the sense that the exact same approach has not been examined before.\n\n"
}
if (!novel) {
s += "As the authors state themselves, their approach is not novel. As much as the community appreciates replication of existing work, there simply is not enough to learn from the present paper.\n\n"
}
}
if (!incremental_i) {
if (incremental) {
s += "A central problem with the approach is that the difference with respect to earlier work is simply too small. An incremental approach such as this one may be appreciated in practice, as it may easily integrate into existing processes; but for a research paper, the community expects bigger leaps.\n\n"
} else {
s += "As much as the community may appreciate how much this paper stays away from the scientific mainstream, the problem is that the approach requires too many changes in practice to have a chance to succeed. Without a viable strategy that lets users migrate to the new approach, its practical value is at best limited.\n\n"
}
}
if (!sound_i && !sound) {
s += "The approach suffers from a large number of false positives. In my experience, this is exactly the central problem of this and related approaches in the field; and it is this problem which - for more than a decade! - has undermined the reputation of such approaches in practice. Unless the number of false positives is significantly reduced, this reviewer sees no chance for the approach to succeed.\n\n"
}
if (!complete_i && !complete) {
s += "The large number of false negatives is another serious concern. In my experience, this is exactly the central problem of this and related approaches in the field; and it is this problem which - for more than a decade! - has undermined the reputation of such approaches in practice. Unless the number of false negatives is significantly reduced, this reviewer sees no chance for the approach to succeed.\n\n"
}
if (!sound_i && !complete_i && !sound && !complete) {
s += "Also, with the approach apparently being neither sound nor complete, how is this ever to succeed in practice?\n\n"
}
if (!sound_i && !complete_i && sound && complete) {
s += "At first sight, the approach shows an impressive accuracy - almost too good to be true. This accuracy, however, can be achieved only in very controlled environments, as outlined below; the authors will have to demonstrate their approach on realistic data sets to establish any credibility.\n\n"
}
if (!formal_i) {
if (formal) {
s += "The paper suffers from an excess of formalism. Several important definitions are introduced formally only, without a chance to clearly understand their motivation or usage. The authors should be encouraged to make more use of informal definitions, building on the well-established formal definitions in the community.\n\n"
}
else
{
s += "The paper suffers from a lack of precision. Several important definitions are introduced informally, without a chance to clearly reproduce - or even understand! - the approach at hand. The authors should be encouraged to make more use of formal definitions, using the tools of the trade in the community.\n\n"
}
}
if (!scales_i) {
if (scales) {
s += "It is good to see the approach scale to real-life settings. The price of scalability, however, is having to cope with a multitude of details, which the paper only glosses over, without ever providing a complete picture. The authors would be better served to provide a tangible abstraction of their approach; maybe it only works in limited settings, but at least, for these, we can understand how and why.\n\n"
}
else
{
s += "For readers, it is helpful to see the approach being demonstrated in a limited setting that facilitates control and comprehension. However, real-world settings are much more detailed, and much more challenging. If the approach is ever to succeed in practice, having a demonstrator that copes with real-world conditions is a must; the authors would be encouraged to work with industry to get access to appropriate settings and data sets.\n\n"
}
}
if (!inuse_i)
{
if (inuse)
{
s += "The authors should be happy to see their approach being used outside of academia. However, this challenges the novelty of the present submission: Not only is the approach is no longer superior to the state of the art, it even is not longer superior to the state of the practice - ironically, because it now defines what the current state of practice is. The submission had better be framed as an experience report and target non-academic readers.\n\n"
}
else
{
s += "Given the above problems of the approach, it is not surprising that its adoption so far is limited. It would be wiser to wait with publication until the approach has become more mature and the authors have gathered more experience in practical settings.\n\n"
}
}
return s
}
</script>
<h3>Evaluation <span class="checkmark" id="evaluation_check"></span></h3>
<input id="subjects" oninput="update_evaluation()" type="number" placeholder="10" min="0" max="999" pattern="\d*"> <label for="subjects">Number of subjects the approach was evaluated on.</label><br>
<input id="working" oninput="update_evaluation()" type="number" placeholder="5" min="0" max="999" pattern="\d*"> <label for="working">Number of subjects it shows improvements on.</label><br>
<input id="comparisons" oninput="update_evaluation()" type="number" placeholder="3" min="0" max="999" pattern="\d*"> <label for="comparisons">Number of approaches it was compared against.</label><br>
<input id="students" onchange="update_evaluation()" type="checkbox"> <label for="students">The evaluation involves students.</label><br>
<script type="text/javascript">
"use strict"
function update_evaluation() {
update_review()
}
function any_evaluation_set() {
var subjects = document.getElementById("subjects").value
var working = document.getElementById("working").value
var comparisons = document.getElementById("comparisons").value
var students_i = document.getElementById("students").indeterminate
return subjects || working || comparisons || !students_i
}
function review_evaluation(accept) {
var subjects = document.getElementById("subjects").value
var working = document.getElementById("working").value
var comparisons = document.getElementById("comparisons").value
var students = document.getElementById("students").checked
var students_i = document.getElementById("students").indeterminate
var title = document.getElementById("title").value
var author = document.getElementById("author").value
var institution = document.getElementById("institution").value
var anon = document.getElementById("anon").checked
var area = document.getElementById("area").value
if (!any_evaluation_set()){
return todo("comment on the evaluation")
}
var s = "## Evaluation\n\n"
if (accept)
{
s += "The evaluation shows the full benefits of the approach. The authors do a great effort in explaining the results; their novel usage of statistics clearly demonstrates how the few outliers can be easily attributed to noise and measurement effects. Overall, this is solid and significant work.\n\n"
return s
}
if (subjects.length > 0)
{
if (parseInt(subjects) == 0)
{
s += "As nice as it is to see the approach defined and its properties discussed, it _has_ to be evaluated whether it works on actual subjects. Without a detailed evaluation, we can never know whether the claimed properties also hold under less abstract and less favorable circumstances.\n\n"
}
if (parseInt(subjects) == 1)
{
s += "Evaluating the approach on one subject serves well as proof of concept. However, to capture the wide range of possible subjects, a more widespread evaluation will be necessary to dispel doubts on the generality of the approach.\n\n"
}
if (parseInt(subjects) > 1)
{
s += "Evaluating the approach on " + subjects + " subjects is a promising first step. However, the number of subjects appears far too small with respect to the limited effects expected and observed. The number must be at least double; with the given limited control of variability, a number of " + (subjects * 3) + " would be even more appropriate.\n\n"
}
}
else
{
s += "This reviewer read until Page 2, and the evaluation seems to be okay.\n\n"
}
if (working.length > 0)
{
if (parseInt(working) == 0)
{
s += "These concerns are the more troubling as apparently, the approach for now does not show *any* improvements. Seriously, why submit a paper if you do not have a success to share? The causes for these failures must be thoroughly investigated, probably in a revised version of this paper.\n\n"
}
else if (parseInt(working) < parseInt(subjects))
{
s += "The fact that the approach produces inconsistent findings across the " + subjects + " subjects is a cause for concern. How can you claim superiority of your approach if it is clearly inferior on " + (subjects - working) + " subjects? Do not call these results 'promising' - this reads like an euphemism for failure. You have to precisely identify and evaluate under which circumstances your approach is better, and how prevalent (and likely) these circumstances are.\n\n"
}
}
if (comparisons.length > 0)
{
if (comparisons == 0)
{
if (!area)
{
area = "the field"
}
s += "Your experiments do not compare against related approaches. However, there is a multitude of approaches in " + area + " that might be applicable. "
}
if (comparisons > 0)
{
s += "Comparing the approach against " + comparisons + " competing approaches is necessary given how incremental your approach is. However, the number of approaches compared against appears far too small with respect to the great many competing approaches and the limited effects expected and observed. "
}
if (title)
{
s += "In particular, you have to compare against the latest work on " + title + " by R. E. Viewer, which was just uploaded to ArXiV. "
}
if (anon && (author || institution))
{
s += "Also, compare against the recent work "
if (author) {
s += "by " + author + " "
}
if (institution) {
s += "at " + institution + " "
}
s += "- one of the few who still do research in the field."
}
s += "\n\n"
}
if (!students_i)
{
if (students) {
s += "The fact that your evaluation involves students is also a cause for concerns - not only for ethical reasons, but also for the sake of generality. Generally speaking, results achieved with students cannot generalize to results achieved with the general public"
if (area)
{
s += " - especially not in " + area
}
s += ". You have to run your evaluation and experiments involving a wider range of subjects.\n\n"
}
if (!students) {
s += "Given how simple your approach is, it is a pity it was not evaluated in a teaching context involving students - this could have compensated for it being less suitable for practical purposes. For the next revision of your paper, include an evaluation involving students.\n\n"
}
}
return s
}
</script>
<h3>Limitations <span class="checkmark" id="limitations_check"></span></h3>
<input id="threats" type="checkbox" onchange="update_limitations()"> <label for="threats">The paper discusses threats to validity.</label><br>
<input id="future_work" type="checkbox" onchange="update_limitations()"> <label for="future_work">The paper details future work.</label>
<script type="text/javascript">
"use strict"
function update_limitations() {
update_review()
}
function any_limitations_set() {
var threats_i = document.getElementById("threats").indeterminate
var future_work_i = document.getElementById("future_work").indeterminate
return !threats_i || !future_work_i
}
function review_limitations(accept)
{
var threats = document.getElementById("threats").checked
var future_work = document.getElementById("future_work").checked
var threats_i = document.getElementById("threats").indeterminate
var future_work_i = document.getElementById("future_work").indeterminate
var area = document.getElementById("area").value
var field = document.getElementById("field").value
if (!any_limitations_set()) {
return todo("comment on limitations")
}
if (!area) {
area = "this area"
}
if (!field) {
field = "this field"
}
var s = "## Limitations\n\n"
if (accept)
{
s += "The work has no limitations.\n\n"
return s
}
if (!threats_i) {
if (threats)
{
s += "The section on 'threats to validity' pretty much sums everything up: The approach cannot claim external validity (no surprise, given the evaluation results); on top, _the authors themselves_ list important threats to internal and construct validity.\n\n"
}
else
{
s += "The paper does not have an explicit section on threats to validity. Yet, as discussed above, these threats are manifold. With respect to external validity, we already have the evaluation results; regarding internal and/or construct validity, the authors themselves would be well-advised to think hard about why their approach might have issues.\n\n"
}
}
if (!future_work_i)
{
if (future_work)
{
s += "The section on 'future work' fits into this very picture: There clearly is still a lot of additional work to do until this approach can begin to show the maturity that would be expected from published scientific results. "
}
else
{
s += "The fact that the authors themselves apparently cannot list future work fits into this very picture. This is a clear indication of work being stuck in an impasse; if one ever needed a living proof how " + field + " has become the laughingstock of the " + area + " community, or how " + area + " is more and more becoming a dead branch of science, here it is. "
}
}
if (!threats_i && !future_work_i) {
s += "With so many threats and limitations, it is unclear whether the paper can be published at all, even in a thoroughly revised form."
}
s += "\n\n"
return s
}
</script>
<h3>Reproducibility <span class="checkmark" id="reproducibility_check"></span></h3>
<input id="open_source" onchange="update_reproducibility()" type="checkbox"> <label for="open_source">The code is available as open source</label>.<br>
<input id="open_data" onchange="update_reproducibility()" type="checkbox"> <label for="open_data">The raw data is available as open data.</label><br>
<script type="text/javascript">
"use strict"
function update_reproducibility() {
update_review()
}
function any_reproducibility_set() {
var open_source_i = document.getElementById("open_source").indeterminate
var open_data_i = document.getElementById("open_data").indeterminate
return !open_source_i || !open_data_i
}
function review_reproducibility(accept)
{
var open_source = document.getElementById("open_source").checked
var open_data = document.getElementById("open_data").checked
var open_source_i = document.getElementById("open_source").indeterminate
var open_data_i = document.getElementById("open_data").indeterminate
var area = document.getElementById("area").value
var field = document.getElementById("field").value
if (!any_reproducibility_set())
{
return todo("comment on reproducibility")
}
var s = "## Reproducibility\n\n"
if (accept)
{
s += "Thanks to the clear writing and the instructions provided, this reviewer was able to reproduce all results within minutes using nothing but a TI-59 calculator. Never before has an approach be so simple, yet so novel and convincing.\n\n"
return s
}
if (!open_source_i && open_source)
{
if (!area) {
area = "the field"
}
s += "We rarely see open source packages in " + area + "; so the extra effort by the authors in making their code available is well-appreciated. However, faced with dozens of files with unknown extensions (.R, .ipynb) which would not even open in Word, this reviewer quickly gave up making sense of it all. The assumption was that README.md would contain instructions, but again, this reviewer was not able to open it. In the future, be sure your code can be accessed using standard Office tools and formats.\n\n"
}
if (!open_data_i && open_data)
{
s += "This reviewer "
if (open_source) {
s += "also "
}
s += "appreciates that the authors provide a data package to reproduce results. However, attempting to open the package resulted in a message\n\n unzip: command not found\n\nConsequently, this reviewer was unable to assess the package, and henceforth treated the package as if it were non-existent. For the future, please be more careful in stating the precise package requirements, assuming less exotic tools and environments.\n\n"
}
if ((!open_source_i && !open_source) || (!open_data_i && !open_data))
{
s += "Your "
if (!open_source && !open_data) {
s += "code and data are "
}
else if (!open_source) {
s += "code is "
}
else if (!open_data) {
s += "data is "
}
s += "not available as open source. Being unavailable for the general public, there is no way for readers (or this reviewer) to validate your claims, which is a strict requirement for a scientific contribution. Please submit it as an electronic appendix with the next revision.\n\n"
}
return s
}
</script>
<h3>Presentation <span class="checkmark" id="presentation_check"></span></h3>
<input id="grammar" onchange="update_presentation()" type="checkbox"> <label for="grammar">The paper has no grammatical errors.</label><br>
<input id="typos" onchange="update_presentation()" type="checkbox"> <label for="typos">The paper has no spelling errors.</label><br>
<input id="layout" onchange="update_presentation()" type="checkbox"> <label for="layout">The paper has no typesetting or layout errors.</label><br>
<input id="biblio" onchange="update_presentation()" type="checkbox"> <label for="biblio">The bibliography is complete.</label><br>
<script type="text/javascript">
"use strict"
function update_presentation() {
update_review()
}
function any_presentation_set() {
var grammar_i = document.getElementById("grammar").indeterminate
var typos_i = document.getElementById("typos").indeterminate
var layout_i = document.getElementById("layout").indeterminate
var biblio_i = document.getElementById("biblio").indeterminate
return !grammar_i || !typos_i || !layout_i || !biblio_i
}
function review_presentation(accept)
{
var grammar = document.getElementById("grammar").checked
var typos = document.getElementById("typos").checked
var layout = document.getElementById("layout").checked
var biblio = document.getElementById("biblio").checked
var grammar_i = document.getElementById("grammar").indeterminate
var typos_i = document.getElementById("typos").indeterminate
var layout_i = document.getElementById("layout").indeterminate
var biblio_i = document.getElementById("biblio").indeterminate
if (!any_presentation_set())
{
return todo("comment on presentation")
}
var initial_s = "## Presentation\n\n"
var s = initial_s
if (accept)
{
s += "Beautiful. 'Let the heavens rejoice, let the earth be glad; let them say among the nations, \"The LORD reigns!\"'\n\n"
return s
}
if (grammar && typos && layout)
{
s += "The paper is excellently written. However, even the best of presentations cannot save flawed material.\n\n"
}
if (!layout_i && !layout)
{
s += "The typography of your paper is a disgrace. Respect the style instructions as given by the publisher. Respect paper lengths. Do not cheat with super-small fonts. "
s += "Learn what good typography is. Learn how to use LaTeX, and how to use LaTeX properly. Do not use multi-letter identifiers in LaTeX math mode. Distinguish - (hyphen) between elements of compound words, -- (en-dash) in ranges, $-$ (minus) for math, --- (em-dash) for digressions in a sentence. use the correct quotes (`` and ''). In BibTeX, capitalize titles properly. Use \\dots rather than ... . This reviewer stops here.\n\n"
}
if (!grammar_i && !grammar || !typos_i && !typos)
{
s += "The paper has numerous presentation issues. "
if (!typos_i && !typos) {
s += "The paper contains numerous typos - Page 2, for instance, has a period '.' instead of a comma ','. This is a sign of careless proofreading, and ultimately disrespect - if the authors do not care about their paper, why should the reader care? At least _try_ to produce a polished version for submission.\n\n"
}
if (!grammar_i && !grammar) {
s += "The grammar"
if (!typos_i && !typos) {
s += " also"
}
s += " needs to be throughly checked and proof read. This reviewer is happy to point to this excellent resource that will help you out:\n\n https://www.talkenglish.com/grammar/grammar.aspx\n\n"
}
}
if (!layout_i && !layout || !grammar_i && !grammar || !typos_i && !typos)
{
s += "Keep in mind that a high number of presentation issues eventually will hinder the readers and reviewers to understand what your work is about. Should you find misunderstandings in the above review, ask yourself what you could have done to avoid these.\n\n"
}
if (!grammar_i || !typos_i || !layout_i) {
if (s == initial_s) {
s += "This reviewer read until Page 2, and the presentation seemed to be okay.\n\n"
}
}
return s
}
function review_bibliography(accept)
{
var biblio = document.getElementById("biblio").checked
var biblio_i = document.getElementById("biblio").indeterminate
var area = document.getElementById("area").value
if (!area) {
area = "the Field"
}
var s = ""
if (accept) {
if (!biblio_i)
{
s += "## Bibliography\n\nThe bibliography by itself could make a publication of its own. Each and every paper is carefully commented and related to the author's works. This reviewer especially liked the citations of the collected works of R. E. Viewer, who should have had a best paper award in 1999, and whose long-standing contributions to the community and the field are still under-appreciated.\n\n"
}
return s
}
if (!biblio_i && !biblio) {
s += "## Bibliography\n\nYour bibliography misses the following references:\n\n" +
"[1] R. E. Viewer. 30 Years of Research in " + area + ". A memoir." +
"\n Viewer & Sons, London, 2018.\n" +
"[2] R. E. Viewer. Introduction to High-Impact Research." +
"\n Viewer & Sons, London, 2005.\n" +
"[3] R. E. Viewer. More Citations through Reviews." +
"\n Viewer & Sons, London, 1991.\n" +
"[4] D. E. Coy. How not to be Identified as a Reviewer." +
"\n Viewer & Sons, London, 1993.\n" +
"\nBe sure to read and cite these seminal works; this will help in getting your paper accepted. Good luck!\n\n"
}
return s
}
</script>
<p>
<input type="reset" value="Reset Form">
</form>
</div>
<div> <!-- placeholder between form and review -->
</div>
<div>
<h2>The Review <span class="checkmark" id="review_check"></span></h2>
<textarea id="review" rows="53" cols="72" readonly>
Please enable JavaScript in your browser.
</textarea>
<p><button id="copy" onclick="copy_review()">Copy to Clipboard</button>
<span id="copied"></span><br>
</div>
</div>
<script type="text/javascript">
"use strict"
function review_abstract(accept)
{
var title = document.getElementById("title").value
var area = document.getElementById("area").value
var field = document.getElementById("field").value
var author = document.getElementById("author").value
var anon = document.getElementById("anon").checked
var novel = document.getElementById("novel").checked
var incremental = document.getElementById("incremental").checked
var subjects = document.getElementById("subjects").value
var working = document.getElementById("working").value
var layout = document.getElementById("incremental").checked
var grammar = document.getElementById("grammar").checked
var typos = document.getElementById("typos").checked
if (!all_decisions())
{
return ""
}
if (!field) {
field = "research"
}
if (!area) {
area = "the field"
}
var s = "## Abstract\n\n"
s += "This paper"
if (title)
{
s += ", titled '" + title + ",'"
}
if (accept)
{
s += " finally "
}
else
{
s += " one more time "
}
s += "addresses the problem of " + field + " in " + area + ". The approach "
var the_approach = "the approach"
if (novel) {
s += "is novel, and "
the_approach = "it"
}
s += "is well-described in the paper. "
if (incremental && !accept) {
s += "Yet, " + the_approach + " is only incremental with respect to the state of the art. "
if (anon && author)
{
s += "This especially holds when compared against the latest works of " + author + ". "
}
}
if (accept)
{
s += "The approach shows most excellent results across the board. This is a revolution in the making. "
}
else
{
if (subjects.length == 0 || parseInt(subjects) == 0)
{
s += "The approach is shown to work only in theory; but "
}
if (parseInt(subjects) == 1)
{
s += "The approach is evaluated on one subject; and "
}
if (parseInt(subjects) > 1)
{
s += "The approach is evaluated on " + subjects + " subjects; and "
}
s += "despite its obvious limitations, it is shown to have potential under specific circumstances. "
if (parseInt(working) >= parseInt(subjects))
{
s += "In particular, its results consistently improve over the competition in the small set of experiments conducted. "
}
else if (parseInt(working) > 0)
{
s += "At least, this holds for a subset of " + working + " subjects. "
}
}
if (accept) {
s += "The paper is lucidly and elegantly written. It should become a mandatory reading in all science-oriented curricula."
}
else {
if (layout && grammar && typos)
{
s += "Finally, the paper is well-written. "
}
}
s += "\n\n"
return s
}
function all_decisions()
{
return any_approach_set() && any_evaluation_set() && any_limitations_set() && any_reproducibility_set() && any_presentation_set()
}
function any_title_set()
{
var title = document.getElementById("title").value
var area = document.getElementById("area").value
var field = document.getElementById("field").value
return title || area || field
}
function any_author_set()
{
var author = document.getElementById("author").value
var institution = document.getElementById("institution").value
var anon_i = document.getElementById("anon").indeterminate
var rebuttal_i = document.getElementById("rebuttal").indeterminate
return author || institution || !anon_i || !rebuttal_i
}
function any_decision()
{
return any_title_set() || any_author_set() || any_approach_set() || any_evaluation_set() || any_limitations_set() || any_reproducibility_set() || any_presentation_set()
}
function review_summary(accept)
{
var title = document.getElementById("title").value
var area = document.getElementById("area").value
var field = document.getElementById("field").value
var author = document.getElementById("author").value
var anon = document.getElementById("anon").checked
var novel = document.getElementById("novel").checked
var incremental = document.getElementById("incremental").checked
var subjects = document.getElementById("subjects").value
var working = document.getElementById("working").value
var layout = document.getElementById("layout").checked
var grammar = document.getElementById("grammar").checked
var typos = document.getElementById("typos").checked
if (!all_decisions())
{
return ""
}
if (!field) {
field = "research"
}
if (!area) {
area = "the field"
}
var s = "## Summary\n\nPoints in favor:\n"
if (accept)
{
s += "(+) Revolutionary approach to " + field + " in " + area + "\n"
s += "(+) Earth-shattering results\n"
s += "(+) Beautifully written\n"