Closes Issue 17861: Autogenerate Include/opcode.h from opcode.py.

It includes required changes in Makefile.pre.in and configure.ac
among other files.

7 files changed, 239 insertions, 141 deletions

diff --git a/.hgtouch b/.hgtouch
index 7e3a5e7370..00efbefa74 100644
--- a/.hgtouch
+++ b/.hgtouch
@@ -4,6 +4,8 @@
 
 Python/importlib.h: Lib/importlib/_bootstrap.py Modules/_freeze_importlib.c
 
+Include/opcode.h:  Lib/opcode.py Tools/scripts/generate_opcode_h.py
+
 Include/Python-ast.h: Parser/Python.asdl Parser/asdl.py Parser/asdl_c.py
 Python/Python-ast.c: Include/Python-ast.h
 
diff --git a/Include/opcode.h b/Include/opcode.h
index 9e5f0bf5d1..0638b5468c 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -1,3 +1,4 @@
+/* Auto-generated by Tools/scripts/generate_opcode_h.py */
 #ifndef Py_OPCODE_H
 #define Py_OPCODE_H
 #ifdef __cplusplus
@@ -5,144 +6,111 @@ extern "C" {
 #endif
 
 
-/* Instruction opcodes for compiled code */
-
-#define POP_TOP         1
-#define ROT_TWO         2
-#define ROT_THREE       3
-#define DUP_TOP         4
-#define DUP_TOP_TWO     5
-#define NOP             9
-
-#define UNARY_POSITIVE  10
-#define UNARY_NEGATIVE  11
-#define UNARY_NOT       12
-
-#define UNARY_INVERT    15
-
-#define BINARY_MATRIX_MULTIPLY 16
-#define INPLACE_MATRIX_MULTIPLY 17
-
-#define BINARY_POWER    19
-
-#define BINARY_MULTIPLY 20
-
-#define BINARY_MODULO   22
-#define BINARY_ADD      23
-#define BINARY_SUBTRACT 24
-#define BINARY_SUBSCR   25
-#define BINARY_FLOOR_DIVIDE 26
-#define BINARY_TRUE_DIVIDE 27
-#define INPLACE_FLOOR_DIVIDE 28
-#define INPLACE_TRUE_DIVIDE 29
-
-#define STORE_MAP       54
-#define INPLACE_ADD     55
-#define INPLACE_SUBTRACT        56
-#define INPLACE_MULTIPLY        57
-
-#define INPLACE_MODULO  59
-#define STORE_SUBSCR    60
-#define DELETE_SUBSCR   61
-
-#define BINARY_LSHIFT   62
-#define BINARY_RSHIFT   63
-#define BINARY_AND      64
-#define BINARY_XOR      65
-#define BINARY_OR       66
-#define INPLACE_POWER   67
-#define GET_ITER        68
-#define PRINT_EXPR      70
-#define LOAD_BUILD_CLASS 71
-#define YIELD_FROM      72
-
-#define INPLACE_LSHIFT  75
-#define INPLACE_RSHIFT  76
-#define INPLACE_AND     77
-#define INPLACE_XOR     78
-#define INPLACE_OR      79
-#define BREAK_LOOP      80
-#define WITH_CLEANUP    81
-
-#define RETURN_VALUE    83
-#define IMPORT_STAR     84
-
-#define YIELD_VALUE     86
-#define POP_BLOCK       87
-#define END_FINALLY     88
-#define POP_EXCEPT      89
-
-#define HAVE_ARGUMENT   90      /* Opcodes from here have an argument: */
-
-#define STORE_NAME      90      /* Index in name list */
-#define DELETE_NAME     91      /* "" */
-#define UNPACK_SEQUENCE 92      /* Number of sequence items */
-#define FOR_ITER        93
-#define UNPACK_EX       94      /* Num items before variable part +
-                                   (Num items after variable part << 8) */
-
-#define STORE_ATTR      95      /* Index in name list */
-#define DELETE_ATTR     96      /* "" */
-#define STORE_GLOBAL    97      /* "" */
-#define DELETE_GLOBAL   98      /* "" */
-
-#define LOAD_CONST      100     /* Index in const list */
-#define LOAD_NAME       101     /* Index in name list */
-#define BUILD_TUPLE     102     /* Number of tuple items */
-#define BUILD_LIST      103     /* Number of list items */
-#define BUILD_SET       104     /* Number of set items */
-#define BUILD_MAP       105     /* Always zero for now */
-#define LOAD_ATTR       106     /* Index in name list */
-#define COMPARE_OP      107     /* Comparison operator */
-#define IMPORT_NAME     108     /* Index in name list */
-#define IMPORT_FROM     109     /* Index in name list */
-
-#define JUMP_FORWARD    110     /* Number of bytes to skip */
-#define JUMP_IF_FALSE_OR_POP 111        /* Target byte offset from beginning of code */
-#define JUMP_IF_TRUE_OR_POP 112 /* "" */
-#define JUMP_ABSOLUTE   113     /* "" */
-#define POP_JUMP_IF_FALSE 114   /* "" */
-#define POP_JUMP_IF_TRUE 115    /* "" */
-
-#define LOAD_GLOBAL     116     /* Index in name list */
-
-#define CONTINUE_LOOP   119     /* Start of loop (absolute) */
-#define SETUP_LOOP      120     /* Target address (relative) */
-#define SETUP_EXCEPT    121     /* "" */
-#define SETUP_FINALLY   122     /* "" */
-
-#define LOAD_FAST       124     /* Local variable number */
-#define STORE_FAST      125     /* Local variable number */
-#define DELETE_FAST     126     /* Local variable number */
-
-#define RAISE_VARARGS   130     /* Number of raise arguments (1, 2 or 3) */
-/* CALL_FUNCTION_XXX opcodes defined below depend on this definition */
-#define CALL_FUNCTION   131     /* #args + (#kwargs<<8) */
-#define MAKE_FUNCTION   132     /* #defaults + #kwdefaults<<8 + #annotations<<16 */
-#define BUILD_SLICE     133     /* Number of items */
-
-#define MAKE_CLOSURE    134     /* same as MAKE_FUNCTION */
-#define LOAD_CLOSURE    135     /* Load free variable from closure */
-#define LOAD_DEREF      136     /* Load and dereference from closure cell */ 
-#define STORE_DEREF     137     /* Store into cell */ 
-#define DELETE_DEREF    138     /* Delete closure cell */ 
-
-/* The next 3 opcodes must be contiguous and satisfy
-   (CALL_FUNCTION_VAR - CALL_FUNCTION) & 3 == 1  */
-#define CALL_FUNCTION_VAR          140  /* #args + (#kwargs<<8) */
-#define CALL_FUNCTION_KW           141  /* #args + (#kwargs<<8) */
-#define CALL_FUNCTION_VAR_KW       142  /* #args + (#kwargs<<8) */
-
-#define SETUP_WITH 143
-
-/* Support for opargs more than 16 bits long */
-#define EXTENDED_ARG  144
-
-#define LIST_APPEND     145
-#define SET_ADD         146
-#define MAP_ADD         147
-
-#define LOAD_CLASSDEREF   148
+    /* Instruction opcodes for compiled code */
+#define POP_TOP             	1  
+#define ROT_TWO             	2  
+#define ROT_THREE           	3  
+#define DUP_TOP             	4  
+#define DUP_TOP_TWO         	5  
+#define NOP                 	9  
+#define UNARY_POSITIVE      	10 
+#define UNARY_NEGATIVE      	11 
+#define UNARY_NOT           	12 
+#define UNARY_INVERT        	15 
+#define BINARY_MATRIX_MULTIPLY	16 
+#define INPLACE_MATRIX_MULTIPLY	17 
+#define BINARY_POWER        	19 
+#define BINARY_MULTIPLY     	20 
+#define BINARY_MODULO       	22 
+#define BINARY_ADD          	23 
+#define BINARY_SUBTRACT     	24 
+#define BINARY_SUBSCR       	25 
+#define BINARY_FLOOR_DIVIDE 	26 
+#define BINARY_TRUE_DIVIDE  	27 
+#define INPLACE_FLOOR_DIVIDE	28 
+#define INPLACE_TRUE_DIVIDE 	29 
+#define STORE_MAP           	54 
+#define INPLACE_ADD         	55 
+#define INPLACE_SUBTRACT    	56 
+#define INPLACE_MULTIPLY    	57 
+#define INPLACE_MODULO      	59 
+#define STORE_SUBSCR        	60 
+#define DELETE_SUBSCR       	61 
+#define BINARY_LSHIFT       	62 
+#define BINARY_RSHIFT       	63 
+#define BINARY_AND          	64 
+#define BINARY_XOR          	65 
+#define BINARY_OR           	66 
+#define INPLACE_POWER       	67 
+#define GET_ITER            	68 
+#define PRINT_EXPR          	70 
+#define LOAD_BUILD_CLASS    	71 
+#define YIELD_FROM          	72 
+#define INPLACE_LSHIFT      	75 
+#define INPLACE_RSHIFT      	76 
+#define INPLACE_AND         	77 
+#define INPLACE_XOR         	78 
+#define INPLACE_OR          	79 
+#define BREAK_LOOP          	80 
+#define WITH_CLEANUP        	81 
+#define RETURN_VALUE        	83 
+#define IMPORT_STAR         	84 
+#define YIELD_VALUE         	86 
+#define POP_BLOCK           	87 
+#define END_FINALLY         	88 
+#define POP_EXCEPT          	89 
+#define HAVE_ARGUMENT       	90 
+#define STORE_NAME          	90 
+#define DELETE_NAME         	91 
+#define UNPACK_SEQUENCE     	92 
+#define FOR_ITER            	93 
+#define UNPACK_EX           	94 
+#define STORE_ATTR          	95 
+#define DELETE_ATTR         	96 
+#define STORE_GLOBAL        	97 
+#define DELETE_GLOBAL       	98 
+#define LOAD_CONST          	100
+#define LOAD_NAME           	101
+#define BUILD_TUPLE         	102
+#define BUILD_LIST          	103
+#define BUILD_SET           	104
+#define BUILD_MAP           	105
+#define LOAD_ATTR           	106
+#define COMPARE_OP          	107
+#define IMPORT_NAME         	108
+#define IMPORT_FROM         	109
+#define JUMP_FORWARD        	110
+#define JUMP_IF_FALSE_OR_POP	111
+#define JUMP_IF_TRUE_OR_POP 	112
+#define JUMP_ABSOLUTE       	113
+#define POP_JUMP_IF_FALSE   	114
+#define POP_JUMP_IF_TRUE    	115
+#define LOAD_GLOBAL         	116
+#define CONTINUE_LOOP       	119
+#define SETUP_LOOP          	120
+#define SETUP_EXCEPT        	121
+#define SETUP_FINALLY       	122
+#define LOAD_FAST           	124
+#define STORE_FAST          	125
+#define DELETE_FAST         	126
+#define RAISE_VARARGS       	130
+#define CALL_FUNCTION       	131
+#define MAKE_FUNCTION       	132
+#define BUILD_SLICE         	133
+#define MAKE_CLOSURE        	134
+#define LOAD_CLOSURE        	135
+#define LOAD_DEREF          	136
+#define STORE_DEREF         	137
+#define DELETE_DEREF        	138
+#define CALL_FUNCTION_VAR   	140
+#define CALL_FUNCTION_KW    	141
+#define CALL_FUNCTION_VAR_KW	142
+#define SETUP_WITH          	143
+#define EXTENDED_ARG        	144
+#define LIST_APPEND         	145
+#define SET_ADD             	146
+#define MAP_ADD             	147
+#define LOAD_CLASSDEREF     	148
 
 /* EXCEPT_HANDLER is a special, implicit block type which is created when
    entering an except handler. It is not an opcode but we define it here
@@ -151,8 +119,9 @@ extern "C" {
 #define EXCEPT_HANDLER 257
 
 
-enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE, PyCmp_GT=Py_GT, PyCmp_GE=Py_GE,
-             PyCmp_IN, PyCmp_NOT_IN, PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD};
+enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE,
+                PyCmp_GT=Py_GT, PyCmp_GE=Py_GE, PyCmp_IN, PyCmp_NOT_IN,
+                PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD};
 
 #define HAS_ARG(op) ((op) >= HAVE_ARGUMENT)
 
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 5af61e3d9c..fbec3b7a68 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -323,6 +323,13 @@ PGENSRCS=	$(PSRCS) $(PGSRCS)
 PGENOBJS=	$(POBJS) $(PGOBJS)
 
 ##########################################################################
+# opcode.h generation
+OPCODE_H_DIR= 	Include
+OPCODE_H_SCRIPT= Tools/scripts/generate_opcode_h.py
+OPCODE_H=	$(srcdir)/$(OPCODE_H_DIR)/opcode.h
+OPCODE_H_GEN=	@OPCODEHGEN@  $(OPCODE_H_SCRIPT) Lib/ $(OPCODE_H)
+#
+##########################################################################
 # AST
 AST_H_DIR=	Include
 AST_H=		$(AST_H_DIR)/Python-ast.h
@@ -760,6 +767,9 @@ $(AST_C): $(AST_H) $(AST_ASDL) $(ASDLGEN_FILES)
 	$(MKDIR_P) $(AST_C_DIR)
 	$(ASDLGEN) -c $(AST_C_DIR) $(AST_ASDL)
 
+$(OPCODE_H): $(srcdir)/Lib/opcode.py $(OPCODE_H_SCRIPT)
+	$(OPCODE_H_GEN)
+
 Python/compile.o Python/symtable.o Python/ast.o: $(GRAMMAR_H) $(AST_H)
 
 Python/getplatform.o: $(srcdir)/Python/getplatform.c
@@ -871,7 +881,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/node.h \
 		$(srcdir)/Include/object.h \
 		$(srcdir)/Include/objimpl.h \
-		$(srcdir)/Include/opcode.h \
+		$(OPCODE_H) \
 		$(srcdir)/Include/osdefs.h \
 		$(srcdir)/Include/patchlevel.h \
 		$(srcdir)/Include/pgen.h \
diff --git a/Misc/NEWS b/Misc/NEWS
index da77694ade..d820400435 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -223,6 +223,9 @@ IDLE
 Build
 -----
 
+- Issue #17861: Tools/scripts/generate_opcode_h.py automatically regenerates
+  Include/opcode.h from Lib/opcode.py if the later gets any change.
+
 - Issue #20644: OS X installer build support for documentation build changes
   in 3.4.1: assume externally supplied sphinx-build is available in /usr/bin.
 
diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py
new file mode 100644
index 0000000000..b315491cb1
--- /dev/null
+++ b/Tools/scripts/generate_opcode_h.py
@@ -0,0 +1,53 @@
+# This script generates the opcode.h header file.
+
+import sys
+if len(sys.argv) > 0:
+    sys.path.insert(0, sys.argv[1])
+# Importing module from our given src directory.
+import opcode
+header = """/* Auto-generated by Tools/scripts/generate_opcode_h.py */
+#ifndef Py_OPCODE_H
+#define Py_OPCODE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+    /* Instruction opcodes for compiled code */
+"""
+
+footer = """
+/* EXCEPT_HANDLER is a special, implicit block type which is created when
+   entering an except handler. It is not an opcode but we define it here
+   as we want it to be available to both frameobject.c and ceval.c, while
+   remaining private.*/
+#define EXCEPT_HANDLER 257
+
+
+enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE,
+                PyCmp_GT=Py_GT, PyCmp_GE=Py_GE, PyCmp_IN, PyCmp_NOT_IN,
+                PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD};
+
+#define HAS_ARG(op) ((op) >= HAVE_ARGUMENT)
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_OPCODE_H */
+"""
+
+
+def main(outfile='Include/opcode.h'):
+    with open(outfile, 'w') as fobj:
+        fobj.write(header)
+        for name in opcode.opname:
+            if name in opcode.opmap:
+                fobj.write("#define %-20s\t%-3s\n" % (name, opcode.opmap[name]))
+            if name == 'POP_EXCEPT': # Special entry for HAVE_ARGUMENT
+                fobj.write("#define %-20s\t%-3d\n" %
+                            ('HAVE_ARGUMENT', opcode.HAVE_ARGUMENT))
+        fobj.write(footer)
+
+
+if __name__ == '__main__':
+    main(sys.argv[2])
diff --git a/configure b/configure
index d762b7160d..7b0c6a9294 100755
--- a/configure
+++ b/configure
@@ -670,6 +670,7 @@ MKDIR_P
 INSTALL_DATA
 INSTALL_SCRIPT
 INSTALL_PROGRAM
+OPCODEHGEN
 PYTHON
 ASDLGEN
 ac_ct_READELF
@@ -6048,6 +6049,57 @@ else
 fi
 
 
+for ac_prog in python$PACKAGE_VERSION python3 python
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_PYTHON+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  if test -n "$PYTHON"; then
+  ac_cv_prog_PYTHON="$PYTHON" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_PYTHON="$ac_prog"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+fi
+fi
+PYTHON=$ac_cv_prog_PYTHON
+if test -n "$PYTHON"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON" >&5
+$as_echo "$PYTHON" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+  test -n "$PYTHON" && break
+done
+test -n "$PYTHON" || PYTHON="not-found"
+
+if test "$PYTHON" = not-found; then
+    OPCODEHGEN="@echo python: $PYTHON! cannot run Tools/scripts/generate_opcode_h.py"
+else
+    OPCODEHGEN="$PYTHON"
+fi
+
+
+
 case $MACHDEP in
 bsdos*|hp*|HP*)
 	# install -d does not work on BSDI or HP-UX
diff --git a/configure.ac b/configure.ac
index abade0f669..457819b0a3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1036,6 +1036,15 @@ else
     ASDLGEN="$PYTHON"
 fi
 
+AC_SUBST(OPCODEHGEN)
+AC_CHECK_PROGS(PYTHON, python$PACKAGE_VERSION python3 python, not-found)
+if test "$PYTHON" = not-found; then
+    OPCODEHGEN="@echo python: $PYTHON! cannot run Tools/scripts/generate_opcode_h.py"
+else
+    OPCODEHGEN="$PYTHON"
+fi
+
+
 
 case $MACHDEP in
 bsdos*|hp*|HP*)