Add a combined source without #line directives into the dist package

RELEASES.rst

@@ -1155,6 +1155,9 @@ Planned
 * Allow debugger detached callback to call duk_debugger_attach(), previously
   this clobbered some internal state (GH-399)
 
+* Add a combined duktape.c without #line directives into the dist package,
+  as it is a useful alternative in some environments (GH-363)
+
 * Fix "debugger" statement line number off-by-one so that the debugger now
   correctly pauses on the debugger statement rather than after it (GH-347)
 

dist-files/README.rst

@@ -61,6 +61,10 @@ This distributable contains:
 * ``src/``: main Duktape library in a "single source file" format (duktape.c,
   duktape.h, and duk_config.h).
 
+* ``src-noline/``: contains a variant of ``src/duktape.c`` with no ``#line``
+  directives which is preferable for some users.  See discussion in
+  https://github.com/svaarala/duktape/pull/363.
+
 * ``src-separate/``: main Duktape library in multiple files format.
 
 * ``config/``: genconfig utility for creating duk_config.h configuration

util/combine_src.py

@@ -47,6 +47,8 @@
 import os
 import sys
 import re
+import json
+import optparse
 
 re_extinc = re.compile(r'^#include <(.*?)>.*$')
 re_intinc = re.compile(r'^#include \"(duk.*?)\".*$')  # accept duktape.h too
@@ -131,8 +133,13 @@ def processDeclarations(f):
 		elif line.data.startswith('extern int') or line.data.startswith('extern void '):
 			line.data = 'static ' + line.data[7:]  # replace extern with static
 
-def createCombined(files, extinc, intinc, duk_version, git_commit, git_describe, git_branch, license_file, authors_file):
+def createCombined(files, extinc, intinc, duk_version, git_commit, git_describe, git_branch, license_file, authors_file, line_directives):
 	res = []
+	line_map = []   # indicate combined source lines where uncombined file/line would change
+	metadata = {
+		'line_map': line_map
+	}
+
 
 	emit_state = [ None, None ]  # curr_filename, curr_lineno
 
@@ -175,7 +182,11 @@ def emit(line):
 			emit_state[1] += 1
 		else:
 			if line.filename != emit_state[0] or line.lineno != emit_state[1]:
-				res.append('#line %d "%s"' % (line.lineno, line.filename))
+				line_map.append({ 'original_file': line.filename,
+				                  'original_line': line.lineno,
+				                  'combined_line': len(res) + 1 })
+				if line_directives:
+					res.append('#line %d "%s"' % (line.lineno, line.filename))
 			res.append(line.data)
 			emit_state[0] = line.filename
 			emit_state[1] = line.lineno + 1
@@ -233,25 +244,38 @@ def processHeader(f_hdr):
 				incname = m.group(1)
 				emit('/* include removed: %s */' % incname)
 
-	return '\n'.join(res) + '\n'
+	return '\n'.join(res) + '\n', metadata
 
 def main():
+	parser = optparse.OptionParser()
+	parser.add_option('--source-dir', dest='source_dir', help='Source directory')
+	parser.add_option('--output-source', dest='output_source', help='Output source filename')
+	parser.add_option('--output-metadata', dest='output_metadata', help='Output metadata filename')
+	parser.add_option('--duk-version', type='int', dest='duk_version', help='Duktape version integer (e.g. 10203 for 1.2.3)')
+	parser.add_option('--git-commit', dest='git_commit', help='Git commit hash')
+	parser.add_option('--git-describe', dest='git_describe', help='Git describe')
+	parser.add_option('--git-branch', dest='git_branch', help='Git branch')
+	parser.add_option('--license-file', dest='license_file', help='License file to embed')
+	parser.add_option('--authors-file', dest='authors_file', help='Authors file to embed')
+	parser.add_option('--line-directives', dest='line_directives', action='store_true', default=False, help='Use #line directives in combined source')
+	(opts, args) = parser.parse_args()
+
+	assert(opts.source_dir)
+	assert(opts.output_source)
+	assert(opts.output_metadata)
+	assert(opts.duk_version)
+	assert(opts.git_commit)
+	assert(opts.git_describe)
+	assert(opts.git_branch)
+	assert(opts.license_file)
+	assert(opts.authors_file)
+
 	if not os.path.exists('LICENSE.txt'):
 		raise Exception('CWD must be Duktape checkout top')
 
-	outname = sys.argv[2]
-	assert(outname)
-
-	duk_version = int(sys.argv[3])
-	git_commit = sys.argv[4]
-	git_describe = sys.argv[5]
-	git_branch = sys.argv[6]
-	license_file = sys.argv[7]
-	authors_file = sys.argv[8]
-
 	print 'Read input files'
 	files = []
-	filelist = os.listdir(sys.argv[1])
+	filelist = os.listdir(opts.source_dir)
 	filelist.sort()  # for consistency
 	handpick = [ 'duk_strings.c',
 	             'duk_debug_macros.c',
@@ -273,7 +297,7 @@ def main():
 	for fn in filelist:
 		if os.path.splitext(fn)[1] not in [ '.c', '.h' ]:
 			continue
-		res = read(os.path.join(sys.argv[1], fn))
+		res = read(os.path.join(opts.source_dir, fn))
 		files.append(res)
 	print '%d files read' % len(files)
 
@@ -300,12 +324,17 @@ def main():
 		pass
 
 	print 'Output final file'
-	final = createCombined(files, extinc, intinc, duk_version, git_commit, git_describe, git_branch, license_file, authors_file)
-	f = open(outname, 'wb')
-	f.write(final)
-	f.close()
-
-	print 'Wrote %d bytes to %s' % (len(final), outname)
+	combined_source, metadata = \
+	    createCombined(files, extinc, intinc, opts.duk_version,
+	                   opts.git_commit, opts.git_describe, opts.git_branch,
+	                   opts.license_file, opts.authors_file,
+	                   opts.line_directives)
+	with open(opts.output_source, 'wb') as f:
+		f.write(combined_source)
+	with open(opts.output_metadata, 'wb') as f:
+		f.write(json.dumps(metadata, indent=4))
+
+	print 'Wrote %d bytes to %s' % (len(combined_source), opts.output_source)
 
 if __name__ == '__main__':
 	main()

util/make_dist.sh

@@ -37,6 +37,8 @@ ENTRYPWD=`pwd`
 DIST=`pwd`/dist
 DISTSRCSEP=$DIST/src-separate
 DISTSRCCOM=$DIST/src
+DISTSRCNOL=$DIST/src-noline  # src-noline/duktape.c is same as src/duktape.c but without line directives
+                             # https://github.com/svaarala/duktape/pull/363
 
 # DUK_VERSION is grepped from duk_api_public.h.in: it is needed for the
 # public API and we want to avoid defining it in two places.
@@ -61,6 +63,7 @@ rm -rf $DIST
 mkdir $DIST
 mkdir $DIST/src-separate
 mkdir $DIST/src
+mkdir $DIST/src-noline
 mkdir $DIST/config
 mkdir $DIST/extras
 mkdir $DIST/polyfills
@@ -449,6 +452,7 @@ python config/genconfig.py --metadata config --output $DIST/duk_config.h.tmp \
 	--git-commit "$GIT_COMMIT" --git-describe "$GIT_DESCRIBE" --git-branch "$GIT_BRANCH" \
 	autodetect-header-legacy
 cp $DIST/duk_config.h.tmp $DISTSRCCOM/duk_config.h
+cp $DIST/duk_config.h.tmp $DISTSRCNOL/duk_config.h
 cp $DIST/duk_config.h.tmp $DISTSRCSEP/duk_config.h
 #cp $DIST/duk_config.h.tmp $DIST/config/duk_config.h-autodetect
 
@@ -509,6 +513,7 @@ cat src/duktape.h.in | sed -e '
 # keep the line so line numbers match between the two variant headers
 cat $DISTSRCCOM/duktape.h | sed -e 's/^#define DUK_SINGLE_FILE$//' \
 	> $DISTSRCSEP/duktape.h
+cp $DISTSRCSEP/duktape.h $DISTSRCNOL/duktape.h
 
 # Initjs code: built-in Ecmascript code snippets which are evaluated when
 # a new global context is created.  There are multiple minifiers, closure
@@ -818,11 +823,32 @@ rm $DISTSRCSEP/caseconv.txt
 # these files into their repository, the result should be deterministic and
 # diffable.  Also, it must retain __FILE__/__LINE__ behavior through
 # preprocessor directives.  Whitespace and comments can be stripped as long
-# as the other requirements are met.
-
-python util/combine_src.py $DISTSRCSEP $DISTSRCCOM/duktape.c \
-	"$DUK_VERSION" "$GIT_COMMIT" "$GIT_DESCRIBE" "$GIT_BRANCH" \
-	$DIST/LICENSE.txt.tmp $DIST/AUTHORS.rst.tmp
+# as the other requirements are met.  For some users it's preferable *not*
+# to use #line directives in the combined source, so a separate variant is
+# created for that, see: https://github.com/svaarala/duktape/pull/363.
+
+python util/combine_src.py \
+	--source-dir $DISTSRCSEP \
+	--output-source $DISTSRCCOM/duktape.c \
+	--output-metadata $DISTSRCCOM/metadata.json \
+	--duk-version "$DUK_VERSION" \
+	--git-commit "$GIT_COMMIT" \
+	--git-describe "$GIT_DESCRIBE" \
+	--git-branch "$GIT_BRANCH" \
+	--license-file $DIST/LICENSE.txt.tmp \
+	--authors-file $DIST/AUTHORS.rst.tmp \
+	--line-directives
+
+python util/combine_src.py \
+	--source-dir $DISTSRCSEP \
+	--output-source $DISTSRCNOL/duktape.c \
+	--output-metadata $DISTSRCNOL/metadata.json \
+	--duk-version "$DUK_VERSION" \
+	--git-commit "$GIT_COMMIT" \
+	--git-describe "$GIT_DESCRIBE" \
+	--git-branch "$GIT_BRANCH" \
+	--license-file $DIST/LICENSE.txt.tmp \
+	--authors-file $DIST/AUTHORS.rst.tmp
 
 # Clean up temp files
 rm $DIST/*.tmp

util/resolve_combined_lineno.py

@@ -0,0 +1,26 @@
+#!/usr/bin/python
+#
+#  Resolve a line number in the combined source into an uncombined file/line
+#  using a dist/src/metadata.json file.
+#
+#  Usage: $ python resolve_combined_lineno.py dist/src/metadata.json 12345
+#
+
+import os
+import sys
+import json
+
+def main():
+	with open(sys.argv[1], 'rb') as f:
+		metadata = json.loads(f.read())
+	lineno = int(sys.argv[2])
+
+	for e in reversed(metadata['line_map']):
+		if lineno >= e['combined_line']:
+			orig_lineno = e['original_line'] + (lineno - e['combined_line'])
+			print('%s:%d -> %s:%d' % ('duktape.c', lineno,
+			                          e['original_file'], orig_lineno))
+			break
+
+if __name__ == '__main__':
+	main()