-
Notifications
You must be signed in to change notification settings - Fork 381
/
amphtml-update.py
executable file
·1097 lines (894 loc) · 41.6 KB
/
amphtml-update.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
This script is used to generate the 'class-amp-allowed-tags-generated.php'
file that is used by the class AMP_Tag_And_Attribute_Sanitizer.
A bash script, amphtml-update.sh, is provided to automatically run this script. To run the bash script, type:
`bash amphtml-update.sh`
from within a Linux environment such as VVV.
See the Updating Allowed Tags and Attributes section of the Engineering Guidelines
https://github.com/ampproject/amp-wp/wiki/Engineering-Guidelines#updating-allowed-tags-and-attributes.
Then have fun sanitizing your AMP posts!
"""
import glob
import logging
import os
import re
import subprocess
import sys
import tempfile
import collections
import json
import google
from collections.abc import Sequence
seen_spec_names = set()
# Note: This is a temporary measure while waiting for resolution of <https://github.com/ampproject/amphtml/issues/36749#issuecomment-996160595>.
latest_extensions_file_path = os.path.join( os.path.realpath( os.path.dirname(__file__) ), 'latest-extension-versions.json' )
def Die(msg):
print >> sys.stderr, msg
sys.exit(1)
def SetupOutDir(out_dir):
"""Sets up a clean output directory.
Args:
out_dir: directory name of the output directory. Must not have slashes,
dots, etc.
"""
logging.info('entering ...')
assert re.match(r'^[a-zA-Z_\-0-9]+$', out_dir), 'bad out_dir: %s' % out_dir
if os.path.exists(out_dir):
subprocess.check_call(['rm', '-rf', out_dir])
os.mkdir(out_dir)
logging.info('... done')
def GenValidatorPb2Py(validator_directory, out_dir):
"""Calls the proto compiler to generate validator_pb2.py.
Args:
out_dir: directory name of the output directory. Must not have slashes,
dots, etc.
"""
logging.info('entering ...')
assert re.match(r'^[a-zA-Z_\-0-9]+$', out_dir), 'bad out_dir: %s' % out_dir
subprocess.check_call(
['protoc', os.path.join(validator_directory, 'validator.proto'), '--proto_path=%s' % validator_directory, '--python_out=%s' % out_dir])
open('%s/__init__.py' % out_dir, 'w').close()
logging.info('... done')
def GenValidatorProtoascii(validator_directory, out_dir):
"""Assembles the validator protoascii file from the main and extensions.
Args:
out_dir: directory name of the output directory. Must not have slashes,
dots, etc.
"""
logging.info('entering ...')
assert re.match(r'^[a-zA-Z_\-0-9]+$', out_dir), 'bad out_dir: %s' % out_dir
protoascii_segments = [open(os.path.join(validator_directory, 'validator-main.protoascii')).read()]
protoascii_segments.append(open(os.path.join(validator_directory, 'validator-css.protoascii')).read())
protoascii_segments.append(open(os.path.join(validator_directory, 'validator-svg.protoascii')).read())
extensions = glob.glob(os.path.join(validator_directory, 'extensions/*/validator-*.protoascii'))
# In the Github project, the extensions are located in a sibling directory
# to the validator rather than a child directory.
if not extensions:
extensions = glob.glob(os.path.join(validator_directory, '../extensions/*/validator-*.protoascii'))
extensions.sort()
for extension in extensions:
protoascii_segments.append(open(extension).read())
f = open('%s/validator.protoascii' % out_dir, 'w')
f.write(''.join(protoascii_segments))
f.close()
logging.info('... done')
def GeneratePHP(repo_directory, out_dir):
"""Generates PHP for WordPress AMP plugin to consume.
Args:
validator_directory: directory for where the validator is located, inside the amphtml repo.
out_dir: directory name of the output directory
"""
allowed_tags, attr_lists, descendant_lists, reference_points, versions = ParseRules(repo_directory, out_dir)
expected_spec_names = (
b'style amp-custom',
b'style[amp-keyframes]',
)
for expected_spec_name in expected_spec_names:
if expected_spec_name not in seen_spec_names:
raise Exception( 'Missing spec: %s' % expected_spec_name )
#Generate the output
out = []
GenerateHeaderPHP(out)
GenerateSpecVersionPHP(out, versions)
GenerateDescendantListsPHP(out, descendant_lists)
GenerateAllowedTagsPHP(out, allowed_tags)
GenerateLayoutAttributesPHP(out, attr_lists)
GenerateGlobalAttributesPHP(out, attr_lists)
GenerateReferencePointsPHP(out, reference_points)
GenerateFooterPHP(out)
# join out array into a single string and remove unneeded whitespace
output = re.sub("\\(\\s*\\)", "()", '\n'.join(out))
# replace 'True' with true and 'False' with false
output = re.sub("'True'", "true", output)
output = re.sub("'False'", "false", output)
# Write the php file to STDOUT.
print( output )
def GenerateHeaderPHP(out):
# Output the file's header
out.append('<?php')
out.append('/**')
out.append(' * Generated by %s - do not edit.' % os.path.basename(__file__))
out.append(' *')
out.append(' * This is a list of HTML tags and attributes that are allowed by the')
out.append(' * AMP specification. Note that tag names have been converted to lowercase.')
out.append(' *')
out.append(' * Note: This file only contains tags that are relevant to the `body` of')
out.append(' * an AMP page. To include additional elements modify the variable')
out.append(' * `mandatory_parent_denylist` in the amp_wp_build.py script.')
out.append(' *')
out.append(' * phpcs:ignoreFile')
out.append(' *')
out.append(' * @internal')
out.append(' */')
out.append('class AMP_Allowed_Tags_Generated {')
out.append('')
def GenerateSpecVersionPHP(out, versions):
# Output the version of the spec file and matching validator version
if versions['spec_file_revision']:
out.append('\tprivate static $spec_file_revision = %d;' % versions['spec_file_revision'])
if versions['min_validator_revision_required']:
out.append('\tprivate static $minimum_validator_revision_required = %d;' % versions['min_validator_revision_required'])
def GenerateDescendantListsPHP(out, descendant_lists):
out.append('')
out.append('\tprivate static $descendant_tag_lists = %s;' % Phpize( descendant_lists, 1 ).lstrip() )
def GenerateAllowedTagsPHP(out, allowed_tags):
# Output the allowed tags dictionary along with each tag's allowed attributes
out.append('')
out.append('\tprivate static $allowed_tags = %s;' % Phpize( allowed_tags, 1 ).lstrip() )
def GenerateLayoutAttributesPHP(out, attr_lists):
# Output the attribute list allowed for layouts.
out.append('')
out.append('\tprivate static $layout_allowed_attrs = %s;' % Phpize( attr_lists[b'$AMP_LAYOUT_ATTRS'], 1 ).lstrip() )
out.append('')
def GenerateGlobalAttributesPHP(out, attr_lists):
# Output the globally allowed attribute list.
out.append('')
out.append('\tprivate static $globally_allowed_attrs = %s;' % Phpize( attr_lists[b'$GLOBAL_ATTRS'], 1 ).lstrip() )
out.append('')
def GenerateReferencePointsPHP(out, reference_points):
# Output the reference points.
out.append('')
out.append('\tprivate static $reference_points = %s;' % Phpize( reference_points, 1 ).lstrip() )
out.append('')
def GenerateFooterPHP(out):
# Output the footer.
out.append('''
/**
* Get allowed tags.
*
* @since 0.5
* @return array Allowed tags.
*/
public static function get_allowed_tags() {
return self::$allowed_tags;
}
/**
* Get extension specs.
*
* @since 1.5
* @internal
* @return array Extension specs, keyed by extension name.
*/
public static function get_extension_specs() {
static $extension_specs = [];
if ( ! empty( $extension_specs ) ) {
return $extension_specs;
}
foreach ( self::get_allowed_tag( 'script' ) as $script_spec ) {
if ( isset( $script_spec[ AMP_Rule_Spec::TAG_SPEC ]['extension_spec'] ) ) {
$extension_specs[ $script_spec[ AMP_Rule_Spec::TAG_SPEC ]['extension_spec']['name'] ] = $script_spec[ AMP_Rule_Spec::TAG_SPEC ]['extension_spec'];
}
}
return $extension_specs;
}
/**
* Get allowed tag.
*
* Get the rules for a single tag so that the entire data structure needn't be passed around.
*
* @since 0.7
* @param string $node_name Tag name.
* @return array|null Allowed tag, or null if the tag does not exist.
*/
public static function get_allowed_tag( $node_name ) {
if ( isset( self::$allowed_tags[ $node_name ] ) ) {
return self::$allowed_tags[ $node_name ];
}
return null;
}
/**
* Get descendant tag lists.
*
* @since 1.1
* @return array Descendant tags list.
*/
public static function get_descendant_tag_lists() {
return self::$descendant_tag_lists;
}
/**
* Get allowed descendant tag list for a tag.
*
* Get the descendant rules for a single tag so that the entire data structure needn't be passed around.
*
* @since 1.1
* @param string $name Name for the descendants list.
* @return array|bool Allowed tags list, or false if there are no restrictions.
*/
public static function get_descendant_tag_list( $name ) {
if ( isset( self::$descendant_tag_lists[ $name ] ) ) {
return self::$descendant_tag_lists[ $name ];
}
return false;
}
/**
* Get reference point spec.
*
* @since 1.0
* @param string $tag_spec_name Tag spec name.
* @return array|null Reference point spec, or null if does not exist.
*/
public static function get_reference_point_spec( $tag_spec_name ) {
if ( isset( self::$reference_points[ $tag_spec_name ] ) ) {
return self::$reference_points[ $tag_spec_name ];
}
return null;
}
/**
* Get list of globally-allowed attributes.
*
* @since 0.5
* @return array Allowed tag.
*/
public static function get_allowed_attributes() {
return self::$globally_allowed_attrs;
}
/**
* Get layout attributes.
*
* @since 0.5
* @return array Allowed tag.
*/
public static function get_layout_attributes() {
return self::$layout_allowed_attrs;
}''')
out.append('')
out.append('}')
out.append('')
def ParseRules(repo_directory, out_dir):
# These imports happen late, within this method because they don't necessarily
# exist when the module starts running, and the ones that probably do
# are checked by CheckPrereqs.
# pylint: disable=g-import-not-at-top
from google.protobuf import text_format
from google.protobuf import json_format
from google.protobuf import descriptor
sys.path.append(os.getcwd())
from amp_validator_dist import validator_pb2
allowed_tags = {}
attr_lists = {}
descendant_lists = {}
reference_points = {}
versions = {}
specfile='%s/validator.protoascii' % out_dir
# Merge specfile with message buffers.
rules = validator_pb2.ValidatorRules()
text_format.Merge(open(specfile).read(), rules)
# Record the version of this specfile and the corresponding validator version.
if rules.HasField('spec_file_revision'):
versions['spec_file_revision'] = rules.spec_file_revision
if rules.HasField('min_validator_revision_required'):
versions['min_validator_revision_required'] = rules.min_validator_revision_required
# Build a dictionary of the named attribute lists that are used by multiple tags.
for (field_desc, field_val) in rules.ListFields():
if 'attr_lists' == field_desc.name:
for attr_spec in field_val:
attr_lists[UnicodeEscape(attr_spec.name)] = GetAttrs(attr_spec.attrs)
# Build a dictionary of allowed tags and an associated list of their allowed
# attributes, values and other criteria.
# Don't include tags that have a mandatory parent with one of these tag names
# since we're only concerned with using this tag list to validate the HTML
# of the DOM
mandatory_parent_denylist = [
'$ROOT',
'!DOCTYPE',
]
manual_versioning = {
'amp-stream-gallery': ['0.1'],
'amp-wordpress-embed': ['1.0'],
}
bento_spec_names = {}
# Extension to include `IMG` as allowed descendant.
allow_img_as_descendant = [
'amp-story-player-allowed-descendants',
'amp-mega-menu-allowed-descendants',
]
for (field_desc, field_val) in rules.ListFields():
if 'tags' == field_desc.name:
for tag_spec in field_val:
# Ignore tags that are outside of the body
if tag_spec.HasField('mandatory_parent') and tag_spec.mandatory_parent in mandatory_parent_denylist and tag_spec.tag_name != 'HTML':
continue
# Ignore deprecated tags (except for amp-sidebar in amp-story for now).
if tag_spec.HasField('deprecation') and 'AMP-SIDEBAR' != tag_spec.tag_name:
continue
# Handle the special $REFERENCE_POINT tag
if '$REFERENCE_POINT' == tag_spec.tag_name:
gotten_tag_spec = GetTagSpec(tag_spec, attr_lists)
if gotten_tag_spec is not None:
reference_points[ tag_spec.spec_name ] = gotten_tag_spec
continue
# If we made it here, then start adding the tag_spec
tag_name = tag_spec.tag_name.lower()
if tag_name not in allowed_tags:
tag_list = []
else:
tag_list = allowed_tags[tag_name]
if 'SCRIPT' == tag_spec.tag_name:
gotten_tag_spec = GetTagSpec(tag_spec, attr_lists)
if gotten_tag_spec is not None:
if 'extension_spec' in gotten_tag_spec['tag_spec'] and 'bento_supported_version' in gotten_tag_spec['tag_spec']['extension_spec']:
if '' != tag_spec.spec_name:
# In case some other extension requires bento extension, we need to know the spec name of the bento extension.
bento_spec_names[ gotten_tag_spec['tag_spec']['extension_spec']['name'] ] = tag_spec.spec_name
# Remove bento spec version from the list of versions.
version = gotten_tag_spec['tag_spec']['extension_spec']['version']
bento_version = gotten_tag_spec['tag_spec']['extension_spec']['bento_supported_version']
for _version in bento_version:
if _version in version:
version.remove( _version )
gotten_tag_spec['tag_spec']['extension_spec'].pop('bento_supported_version', None)
gotten_tag_spec['tag_spec']['extension_spec']['version'] = version
if len( version ) == 0 and gotten_tag_spec['tag_spec']['extension_spec']['name'] in manual_versioning.keys():
gotten_tag_spec['tag_spec']['extension_spec']['version'] = manual_versioning[ gotten_tag_spec['tag_spec']['extension_spec']['name'] ]
tag_list.append(gotten_tag_spec)
allowed_tags[tag_name] = tag_list
continue
gotten_tag_spec = GetTagSpec(tag_spec, attr_lists)
if gotten_tag_spec is not None:
tag_list.append(gotten_tag_spec)
allowed_tags[tag_name] = tag_list
elif 'descendant_tag_list' == field_desc.name:
for _list in field_val:
descendant_lists[_list.name] = []
for val in _list.tag:
# Skip tags specific to transformed AMP.
if 'I-AMPHTML-SIZER' == val:
continue
# The img tag is currently exclusively to transformed AMP, except few cases.
if 'IMG' == val and _list.name not in allow_img_as_descendant:
continue
descendant_lists[_list.name].append( val.lower() )
# Remove any tags that requires bento_spec_names.keys()
for tag_name, tag_list in allowed_tags.items():
for tag in tag_list:
if 'requires_extension' not in tag['tag_spec'] or 'requires_condition' not in tag['tag_spec']:
continue
requires_condition = tag['tag_spec']['requires_condition'][0]
requires_extension = tag['tag_spec']['requires_extension'][0]
if requires_extension not in bento_spec_names:
continue
if bento_spec_names[requires_extension] == requires_condition:
tag_list.remove( tag )
# Separate extension scripts from non-extension scripts and gather the versions
extension_scripts = collections.defaultdict(list)
extension_specs_by_satisfies = dict()
script_tags = []
for script_tag in allowed_tags['script']:
if 'extension_spec' in script_tag['tag_spec']:
extension = script_tag['tag_spec']['extension_spec']['name']
extension_scripts[extension].append(script_tag)
if 'satisfies_condition' in script_tag['tag_spec']:
satisfies = script_tag['tag_spec']['satisfies_condition']
else:
satisfies = extension
if satisfies in extension_specs_by_satisfies:
raise Exception( 'Duplicate extension script that satisfies %s.' % satisfies )
extension_specs_by_satisfies[satisfies] = script_tag['tag_spec']['extension_spec']
# These component lack an explicit requirement on a specific extension script:
# - amp-selector
# - amp-accordion
# - amp-soundcloud
# - amp-brightcove
# - amp-video
# - amp-video-iframe
# - amp-vimeo
# - amp-twitter
# - amp-instagram
# - amp-lightbox
# - amp-facebook
# - amp-youtube
# - amp-social-share
# - amp-fit-text
# So use the one with the latest version as a fallback.
if 'latest' in script_tag['tag_spec']['extension_spec']['version']:
extension_specs_by_satisfies[extension] = script_tag['tag_spec']['extension_spec']
else:
script_tags.append(script_tag)
# Amend the allowed_tags to supply the required versions for each component.
for tag_name, tags in allowed_tags.items():
for tag in tags:
tag['tag_spec'].pop('satisfies_condition', None) # We don't need it anymore.
tag['tag_spec'].pop('requires_condition', None) # We don't need it anymore.
requires = tag['tag_spec'].pop('requires', [])
if 'requires_extension' not in tag['tag_spec']:
continue
requires_extension_versions = {}
for required_extension in tag['tag_spec']['requires_extension']:
required_versions = []
for require in requires:
if require in extension_specs_by_satisfies:
if required_extension != extension_specs_by_satisfies[require]['name']:
raise Exception('Expected satisfied to be for the %s extension' % required_extension)
required_versions = extension_specs_by_satisfies[require]['version']
break
if len( required_versions ) == 0:
if required_extension in extension_specs_by_satisfies:
if required_extension != extension_specs_by_satisfies[required_extension]['name']:
raise Exception('Expected satisfied to be for the %s extension' % required_extension)
required_versions = extension_specs_by_satisfies[required_extension]['version']
if len( required_versions ) == 0:
raise Exception('Unable to obtain any versions for %s' % required_extension)
requires_extension_versions[required_extension] = filter( lambda ver: ver != 'latest', required_versions )
tag['tag_spec']['requires_extension'] = requires_extension_versions
extensions = json.load( open( os.path.join( repo_directory, 'build-system/compile/bundles.config.extensions.json' ) ) )
latest_versions = json.load( open( latest_extensions_file_path ) )
extensions_versions = dict()
for extension in extensions:
if '-impl' in extension['name'] or '-polyfill' in extension['name']:
continue
name = extension['name']
is_bento = name.startswith( 'bento-' )
if is_bento:
name = name.replace( 'bento-', 'amp-' )
if name not in extensions_versions:
extensions_versions[ name ] = {
'versions': [],
'latest': None,
}
if type(extension['version']) == list:
extensions_versions[ name ]['versions'].extend( extension['version'] )
else:
extensions_versions[ name ]['versions'].append( extension['version'] )
if name not in latest_versions:
raise Exception( 'There is no latest version for ' + name + ' so please add it to bin/latest-extension-version.json' )
extensions_versions[ name ]['latest'] = latest_versions[ name ]
if is_bento or ( 'options' in extension and ( ( 'bento' in extension['options'] and extension['options']['bento'] ) or ( 'wrapper' in extension['options'] and extension['options']['wrapper'] == 'bento' ) ) ):
extensions_versions[ name ]['bento'] = {
'version': extension['version'],
'has_css': extension['options'].get( 'hasCss', False ),
}
# Merge extension scripts (e.g. Bento and non-Bento) into one script per extension.
for extension_name in sorted(extension_scripts):
if extension_name not in extensions_versions:
raise Exception( 'There is a script for an unknown extension: ' + extension_name );
extension_script_list = extension_scripts[extension_name]
validator_versions = set(extension_script_list[0]['tag_spec']['extension_spec']['version'])
for extension_script in extension_script_list[1:]:
validator_versions.update(extension_script['tag_spec']['extension_spec']['version'])
if 'latest' in validator_versions:
validator_versions.remove('latest')
bundle_versions = set( extensions_versions[extension_name]['versions'] )
if not validator_versions.issubset( bundle_versions ):
logging.info( 'Validator versions are not a subset of bundle versions: ' + extension_name )
if 'bento' in extensions_versions[extension_name] and extensions_versions[extension_name]['bento']['version'] not in validator_versions:
logging.info( 'Skipping bento for ' + extension_name + ' since version ' + extensions_versions[extension_name]['bento']['version'] + ' is not yet valid' )
del extensions_versions[extension_name]['bento']
# Verify bento_supported_version matches the version info in the bundle file.
if 'bento_supported_version' in extension_script_list[0]['tag_spec']['extension_spec']:
if 'bento' not in extensions_versions[extension_name]:
logging.info( 'Warning: bento_supported_version found but bento not meta not obtained for ' + extension_name )
elif extension_script_list[0]['tag_spec']['extension_spec']['bento_supported_version'][0] != extensions_versions[extension_name]['bento']['version']:
logging.info( 'Warning: bento_supported_version does not match the bento meta version for ' + extension_name )
# Remove redundant information.
del extension_script_list[0]['tag_spec']['extension_spec']['bento_supported_version']
validator_versions = sorted( validator_versions, key=lambda version: list(map(int, version.split('.') )) )
extension_script_list[0]['tag_spec']['extension_spec']['version'] = validator_versions
if 'bento' in extensions_versions[extension_name] and extensions_versions[extension_name]['bento']['version'] in validator_versions:
extension_script_list[0]['tag_spec']['extension_spec']['bento'] = extensions_versions[extension_name]['bento']
extension_script_list[0]['tag_spec']['extension_spec']['latest'] = extensions_versions[extension_name]['latest']
extension_script_list[0]['tag_spec']['extension_spec'].pop('version_name', None)
# Remove the spec name since we've potentially merged multiple scripts, thus it does not refer to one.
extension_script_list[0]['tag_spec'].pop('spec_name', None)
script_tags.append(extension_script_list[0])
allowed_tags['script'] = script_tags
# Now that Bento information is in hand, re-decorate specs with require_extension to indicate which
for tag_name, tags in allowed_tags.items():
tags_bento_status = []
for tag in tags:
if 'requires_extension' not in tag['tag_spec']:
continue
# Determine the Bento availability of all the required extensions.
tag_extensions_with_bento = {}
for extension, extension_versions in tag['tag_spec']['requires_extension'].items():
if extension in extensions_versions and 'bento' in extensions_versions[extension] and extensions_versions[extension]['bento']['version'] in extension_versions:
tag_extensions_with_bento[ extension ] = True
else:
tag_extensions_with_bento[ extension ] = False
# Mark that this tag is for Bento since all its required extensions have Bento available.
if len( tag_extensions_with_bento ) > 0 and False not in tag_extensions_with_bento.values():
tags_bento_status.append( True )
tag['tag_spec']['bento'] = True
else:
tags_bento_status.append( False )
# Now that the ones with Bento have been identified, add flags to tag specs when there are different versions specifically for Bento:
for tag in tags:
if 'requires_extension' not in tag['tag_spec']:
continue
if False not in tags_bento_status:
# Clear the Bento flag if _all_ of the components are for Bento.
tag['tag_spec'].pop( 'bento', None )
elif True in tags_bento_status and 'bento' not in tag['tag_spec']:
# Otherwise, if _some_ of the components were exclusively for Bento, flag the others as being _not_ for Bento specifically.
tag['tag_spec']['bento'] = False
# Now convert requires_versions back into a list of extensions rather than an extension/versions mapping.
tag['tag_spec']['requires_extension'] = sorted( tag['tag_spec']['requires_extension'].keys() )
return allowed_tags, attr_lists, descendant_lists, reference_points, versions
def GetTagSpec(tag_spec, attr_lists):
tag_dict = GetTagRules(tag_spec)
if tag_dict is None:
return None
attr_dict = {}
# First add attributes from any attribute lists to this tag.
for (tag_field_desc, tag_field_val) in tag_spec.ListFields():
if 'attr_lists' == tag_field_desc.name:
for attr_list in tag_field_val:
attr_dict.update(attr_lists[UnicodeEscape(attr_list)])
# Then merge the spec-specific attributes on top to override any list definitions.
attr_dict.update(GetAttrs(tag_spec.attrs))
tag_spec_dict = {'tag_spec':tag_dict, 'attr_spec_list':attr_dict}
if tag_spec.HasField('cdata'):
cdata_dict = {}
for (field_descriptor, field_value) in tag_spec.cdata.ListFields():
if isinstance(field_value, (str, bytes, bool, int)):
cdata_dict[ field_descriptor.name ] = field_value
elif isinstance( field_value, google.protobuf.pyext._message.RepeatedCompositeContainer ):
cdata_dict[ field_descriptor.name ] = []
for value in field_value:
entry = {}
for (key,val) in value.ListFields():
entry[ key.name ] = val
cdata_dict[ field_descriptor.name ].append( entry )
elif hasattr( field_value, '_values' ):
cdata_dict[ field_descriptor.name ] = {}
for _value in field_value._values:
for (key,val) in _value.ListFields():
cdata_dict[ field_descriptor.name ][ key.name ] = val
elif 'css_spec' == field_descriptor.name:
css_spec = {}
css_spec['allowed_at_rules'] = []
for at_rule_spec in field_value.at_rule_spec:
if '$DEFAULT' == at_rule_spec.name:
continue
css_spec['allowed_at_rules'].append( at_rule_spec.name )
for css_spec_field_name in ( 'allowed_declarations', 'declaration', 'font_url_spec', 'image_url_spec', 'validate_keyframes' ):
if not hasattr( field_value, css_spec_field_name ):
continue
css_spec_field_value = getattr( field_value, css_spec_field_name )
if isinstance(css_spec_field_value, (list, Sequence, google.protobuf.internal.containers.RepeatedScalarFieldContainer, google.protobuf.pyext._message.RepeatedScalarContainer)):
css_spec[ css_spec_field_name ] = [ val for val in css_spec_field_value ]
elif hasattr( css_spec_field_value, 'ListFields' ):
css_spec[ css_spec_field_name ] = {}
for (css_spec_field_item_descriptor, css_spec_field_item_value) in getattr( field_value, css_spec_field_name ).ListFields():
if isinstance(css_spec_field_item_value, (list, Sequence, google.protobuf.internal.containers.RepeatedScalarFieldContainer, google.protobuf.pyext._message.RepeatedScalarContainer)):
css_spec[ css_spec_field_name ][ css_spec_field_item_descriptor.name ] = [ val for val in css_spec_field_item_value ]
else:
css_spec[ css_spec_field_name ][ css_spec_field_item_descriptor.name ] = css_spec_field_item_value
else:
css_spec[ css_spec_field_name ] = css_spec_field_value
cdata_dict['css_spec'] = css_spec
if len( cdata_dict ) > 0:
if 'disallowed_cdata_regex' in cdata_dict:
for entry in cdata_dict['disallowed_cdata_regex']:
if 'error_message' not in entry:
raise Exception( 'Missing error_message for disallowed_cdata_regex.' );
if entry['error_message'] not in ( 'contents', 'html comments', 'CSS i-amphtml- name prefix' ):
raise Exception( 'Unexpected error_message "%s" for disallowed_cdata_regex.' % entry['error_message'] );
entry['regex'] = EscapeRegex( entry['regex'] )
tag_spec_dict['cdata'] = cdata_dict
if 'spec_name' not in tag_spec_dict['tag_spec']:
if 'extension_spec' in tag_spec_dict['tag_spec']:
# CUSTOM_ELEMENT=1 (default), CUSTOM_TEMPLATE=2
extension_type = tag_spec_dict['tag_spec']['extension_spec'].get('extension_type', 1)
spec_name = 'script [%s=%s]' % ( 'custom-element' if 1 == extension_type else 'custom-template', tag_spec_dict['tag_spec']['extension_spec']['name'].lower() )
else:
spec_name = tag_spec.tag_name.lower()
else:
spec_name = tag_spec_dict['tag_spec']['spec_name']
seen_spec_names.add( spec_name )
return tag_spec_dict
def GetTagRules(tag_spec):
tag_rules = {}
if hasattr(tag_spec, 'also_requires_tag') and tag_spec.also_requires_tag:
also_requires_tag_list = []
for also_requires_tag in tag_spec.also_requires_tag:
also_requires_tag_list.append(UnicodeEscape(also_requires_tag))
tag_rules['also_requires_tag'] = also_requires_tag_list
requires_extension_list = set()
if hasattr(tag_spec, 'requires_extension') and len( tag_spec.requires_extension ) != 0:
for requires_extension in tag_spec.requires_extension:
requires_extension_list.add(requires_extension)
requires_condition_list = set()
if hasattr(tag_spec, 'requires_condition') and len( tag_spec.requires_condition ) != 0:
for requires_condition in tag_spec.requires_condition:
requires_condition_list.add(requires_condition)
if hasattr(tag_spec, 'requires') and len( tag_spec.requires ) != 0:
tag_rules['requires'] = [ requires for requires in tag_spec.requires ]
if hasattr(tag_spec, 'also_requires_tag_warning') and len( tag_spec.also_requires_tag_warning ) != 0:
for also_requires_tag_warning in tag_spec.also_requires_tag_warning:
matches = re.search( r'(amp-\S+) extension( \.js)? script', also_requires_tag_warning )
if not matches:
raise Exception( 'Unexpected also_requires_tag_warning format: ' + also_requires_tag_warning )
requires_extension_list.add(matches.group(1))
if len( requires_extension_list ) > 0:
tag_rules['requires_extension'] = list( requires_extension_list )
if len( requires_condition_list ) > 0:
tag_rules['requires_condition'] = list( requires_condition_list )
if hasattr(tag_spec, 'reference_points') and len( tag_spec.reference_points ) != 0:
tag_reference_points = {}
for reference_point_spec in tag_spec.reference_points:
tag_reference_points[ reference_point_spec.tag_spec_name ] = {
"mandatory": reference_point_spec.mandatory,
"unique": reference_point_spec.unique
}
if len( tag_reference_points ) > 0:
tag_rules['reference_points'] = tag_reference_points
if tag_spec.disallowed_ancestor:
disallowed_ancestor_list = []
for disallowed_ancestor in tag_spec.disallowed_ancestor:
disallowed_ancestor_list.append(UnicodeEscape(disallowed_ancestor).lower())
tag_rules['disallowed_ancestor'] = disallowed_ancestor_list
if tag_spec.html_format:
has_amp_format = False
for html_format in tag_spec.html_format:
if 1 == html_format:
has_amp_format = True
if not has_amp_format:
return None
# Ignore transformed AMP for now.
if tag_spec.enabled_by and 'transformed' in tag_spec.enabled_by:
return None
# Ignore amp-custom-length-check because the AMP plugin will indicate how close they are to the limit.
# TODO: Remove the AMP4EMAIL check once this change is released: <https://github.com/ampproject/amphtml/pull/25246>.
if tag_spec.HasField('spec_name') and ( tag_spec.spec_name == 'style amp-custom-length-check' or 'AMP4EMAIL' in tag_spec.spec_name ):
return None
if tag_spec.HasField('extension_spec'):
# See https://github.com/ampproject/amphtml/blob/e37f50d/validator/validator.proto#L430-L454
ERROR = 1
NONE = 3
extension_spec = {
'requires_usage': 1 # (ERROR=1)
}
for field in tag_spec.extension_spec.ListFields():
if isinstance(field[1], (list, google.protobuf.internal.containers.RepeatedScalarFieldContainer, google.protobuf.pyext._message.RepeatedScalarContainer)):
extension_spec[ field[0].name ] = []
for val in field[1]:
extension_spec[ field[0].name ].append( val )
else:
extension_spec[ field[0].name ] = field[1]
# Normalize ERROR and GRANDFATHERED as true, since we control which scripts are added (or removed) from the output.
extension_spec['requires_usage'] = ( extension_spec['requires_usage'] != 3 ) # NONE=3
if 'version' not in extension_spec:
raise Exception( 'Missing required version field' )
if 'name' not in extension_spec:
raise Exception( 'Missing required name field' )
# Unused since amp_filter_script_loader_tag() and \AMP_Tag_And_Attribute_Sanitizer::get_rule_spec_list_to_validate() just hard-codes the check for amp-mustache.
if 'extension_type' in extension_spec:
del extension_spec['extension_type']
if 'deprecated_version' in extension_spec:
del extension_spec['deprecated_version']
if 'deprecated_allow_duplicates' in extension_spec:
del extension_spec['deprecated_allow_duplicates']
tag_rules['extension_spec'] = extension_spec
if tag_spec.HasField('mandatory'):
tag_rules['mandatory'] = tag_spec.mandatory
if tag_spec.HasField('mandatory_alternatives'):
tag_rules['mandatory_alternatives'] = UnicodeEscape(tag_spec.mandatory_alternatives)
if tag_spec.HasField('mandatory_ancestor'):
tag_rules['mandatory_ancestor'] = UnicodeEscape(tag_spec.mandatory_ancestor).lower()
if tag_spec.HasField('mandatory_ancestor_suggested_alternative'):
tag_rules['mandatory_ancestor_suggested_alternative'] = UnicodeEscape(tag_spec.mandatory_ancestor_suggested_alternative).lower()
if tag_spec.HasField('mandatory_parent'):
tag_rules['mandatory_parent'] = UnicodeEscape(tag_spec.mandatory_parent).lower()
if tag_spec.HasField('spec_name'):
tag_rules['spec_name'] = UnicodeEscape(tag_spec.spec_name)
if hasattr(tag_spec, 'satisfies_condition') and len( tag_spec.satisfies_condition ) > 0:
if len( tag_spec.satisfies_condition ) > 1:
raise Exception('More than expected was satisfied')
tag_rules['satisfies_condition'] = tag_spec.satisfies_condition[0]
if tag_spec.HasField('spec_url'):
tag_rules['spec_url'] = UnicodeEscape(tag_spec.spec_url)
if tag_spec.HasField('unique'):
tag_rules['unique'] = tag_spec.unique
if tag_spec.HasField('unique_warning'):
tag_rules['unique_warning'] = tag_spec.unique_warning
if tag_spec.HasField('siblings_disallowed'):
tag_rules['siblings_disallowed'] = tag_spec.siblings_disallowed
if tag_spec.HasField('child_tags'):
child_tags = collections.defaultdict( lambda: [] )
for field in tag_spec.child_tags.ListFields():
if isinstance(field[1], (int)):
child_tags[ field[0].name ] = field[1]
elif isinstance(field[1], (list, google.protobuf.internal.containers.RepeatedScalarFieldContainer, google.protobuf.pyext._message.RepeatedScalarContainer)):
for val in field[1]:
child_tags[ field[0].name ].append( val.lower() )
tag_rules['child_tags'] = child_tags
if tag_spec.HasField('descendant_tag_list'):
tag_rules['descendant_tag_list'] = tag_spec.descendant_tag_list
if tag_spec.HasField('amp_layout'):
amp_layout = {}
for field in tag_spec.amp_layout.ListFields():
if 'supported_layouts' == field[0].name:
amp_layout['supported_layouts'] = [ val for val in field[1] ]
else:
amp_layout[ field[0].name ] = field[1]
tag_rules['amp_layout'] = amp_layout
for mandatory_of_constraint in ['mandatory_anyof', 'mandatory_oneof']:
mandatory_of_spec = GetMandatoryOf(tag_spec.attrs, mandatory_of_constraint)
if mandatory_of_spec:
tag_rules[ mandatory_of_constraint ] = mandatory_of_spec
return tag_rules
def GetAttrs(attrs):
attr_dict = {}
for attr_spec in attrs:
value_dict = GetValues(attr_spec)
if value_dict is not None:
# Skip rules for dev mode attributes since the AMP plugin will allow them to pass through.
# See <https://github.com/ampproject/amphtml/pull/27174#issuecomment-601391161> for how the rules are
# defined in a way that they can never be satisfied, and thus to make the attribute never allowed.
# This runs contrary to the needs of the AMP plugin, as the internal sanitizers are built to ignore them.
if 'data-ampdevmode' == attr_spec.name:
continue
# Normalize bracketed amp-bind attribute syntax to data-amp-bind-* syntax.
name = attr_spec.name
if name[0] == '[':
name = 'data-amp-bind-' + name.strip( '[]' )
# Add attribute name and alternative_names
attr_dict[UnicodeEscape(name)] = value_dict
return attr_dict
def GetValues(attr_spec):
value_dict = {}
# Ignore transformed AMP for now.
if 'transformed' in attr_spec.enabled_by:
return None
# Add alternative names
if attr_spec.alternative_names:
alt_names_list = []
for alternative_name in attr_spec.alternative_names:
alt_names_list.append(UnicodeEscape(alternative_name))
value_dict['alternative_names'] = alt_names_list
# Add disallowed value regex
if attr_spec.HasField('disallowed_value_regex'):
value_dict['disallowed_value_regex'] = EscapeRegex( attr_spec.disallowed_value_regex )
# dispatch_key is an int
if attr_spec.HasField('dispatch_key'):
value_dict['dispatch_key'] = attr_spec.dispatch_key
# mandatory is a boolean
if attr_spec.HasField('mandatory'):
value_dict['mandatory'] = attr_spec.mandatory
# Add allowed value
if attr_spec.value:
value_dict['value'] = list( attr_spec.value )
# value_casei
if attr_spec.value_casei:
value_dict['value_casei'] = list( attr_spec.value_casei )
# value_regex
if attr_spec.HasField('value_regex'):
value_dict['value_regex'] = EscapeRegex( attr_spec.value_regex )
# value_regex_casei
if attr_spec.HasField('value_regex_casei'):
value_dict['value_regex_casei'] = EscapeRegex( attr_spec.value_regex_casei )
#value_properties is a dictionary of dictionaries
if attr_spec.HasField('value_properties'):
value_properties_dict = {}
for (value_properties_key, value_properties_val) in attr_spec.value_properties.ListFields():
for value_property in value_properties_val:
property_dict = {}
# print 'value_property.name: %s' % value_property.name
for (key,val) in value_property.ListFields():
if val != value_property.name:
if isinstance(val, str):
val = UnicodeEscape(val)
property_dict[UnicodeEscape(key.name)] = val
value_properties_dict[UnicodeEscape(value_property.name)] = property_dict
value_dict['value_properties'] = value_properties_dict
# value_url is a dictionary
if attr_spec.HasField('value_url'):
value_url_dict = {}
for (value_url_key, value_url_val) in attr_spec.value_url.ListFields():
if isinstance(value_url_val, (list, Sequence, google.protobuf.internal.containers.RepeatedScalarFieldContainer, google.protobuf.pyext._message.RepeatedScalarContainer)):
value_url_val_val = []
for val in value_url_val:
value_url_val_val.append(UnicodeEscape(val))
else:
value_url_val_val = value_url_val
value_url_dict[value_url_key.name] = value_url_val_val
value_dict['value_url'] = value_url_dict
if hasattr(attr_spec, 'requires_extension') and len( attr_spec.requires_extension ) != 0:
requires_extension_list = []
for requires_extension in attr_spec.requires_extension:
requires_extension_list.append(requires_extension)
value_dict['requires_extension'] = requires_extension_list
return value_dict
def UnicodeEscape(string):
"""Helper function which escapes unicode characters.