gdb: add extension language print_insn hook

This commit is setup for the next commit.

In the next commit I will add a Python API to intercept the print_insn
calls within GDB, each print_insn call is responsible for
disassembling, and printing one instruction.  After the next commit it
will be possible for a user to write Python code that either wraps
around the existing disassembler, or even, in extreme situations,
entirely replaces the existing disassembler.

This commit does not add any new Python API.

What this commit does is put the extension language framework in place
for a print_insn hook.  There's a new callback added to 'struct
extension_language_ops', which is then filled in with nullptr for Python
and Guile.

Finally, in the disassembler, the code is restructured so that the new
extension language function ext_lang_print_insn is called before we
delegate to gdbarch_print_insn.

After this, the next commit can focus entirely on providing a Python
implementation of the new print_insn callback.

There should be no user visible change after this commit.

gdb/disasm.c

@@ -851,6 +851,29 @@ gdb_disassemble_info::~gdb_disassemble_info ()
   disassemble_free_target (&m_di);
 }
 
+/* Wrapper around calling gdbarch_print_insn.  This function takes care of
+   first calling the extension language hooks for print_insn, and, if none
+   of the extension languages can print this instruction, calls
+   gdbarch_print_insn to do the work.
+
+   GDBARCH is the architecture to disassemble in, VMA is the address of the
+   instruction being disassembled, and INFO is the libopcodes disassembler
+   related information.  */
+
+static int
+gdb_print_insn_1 (struct gdbarch *gdbarch, CORE_ADDR vma,
+		  struct disassemble_info *info)
+{
+  /* Call into the extension languages to do the disassembly.  */
+  gdb::optional<int> length = ext_lang_print_insn (gdbarch, vma, info);
+  if (length.has_value ())
+    return *length;
+
+  /* No extension language wanted to do the disassembly, so do it
+     manually.  */
+  return gdbarch_print_insn (gdbarch, vma, info);
+}
+
 /* See disasm.h.  */
 
 bool gdb_disassembler::use_ext_lang_colorization_p = true;
@@ -864,7 +887,7 @@ gdb_disassembler::print_insn (CORE_ADDR memaddr,
   m_err_memaddr.reset ();
   m_buffer.clear ();
 
-  int length = gdbarch_print_insn (arch (), memaddr, &m_di);
+  int length = gdb_print_insn_1 (arch (), memaddr, &m_di);
 
   /* If we have successfully disassembled an instruction, styling is on, we
      think that the extension language might be able to perform styling for
@@ -899,7 +922,7 @@ gdb_disassembler::print_insn (CORE_ADDR memaddr,
 	  gdb_assert (!m_buffer.term_out ());
 	  m_buffer.~string_file ();
 	  new (&m_buffer) string_file (true);
-	  length = gdbarch_print_insn (arch (), memaddr, &m_di);
+	  length = gdb_print_insn_1 (arch (), memaddr, &m_di);
 	  gdb_assert (length > 0);
 	}
     }
@@ -1054,7 +1077,7 @@ gdb_buffered_insn_length (struct gdbarch *gdbarch,
   gdb_buffered_insn_length_init_dis (gdbarch, &di, insn, max_len, addr,
 				     &disassembler_options_holder);
 
-  int result = gdbarch_print_insn (gdbarch, addr, &di);
+  int result = gdb_print_insn_1 (gdbarch, addr, &di);
   disassemble_free_target (&di);
   return result;
 }

gdb/extension-priv.h

@@ -263,6 +263,21 @@ struct extension_language_ops
      contents, or an empty optional.  */
   gdb::optional<std::string> (*colorize_disasm) (const std::string &content,
 						 gdbarch *gdbarch);
+
+  /* Print a single instruction from ADDRESS in architecture GDBARCH.  INFO
+     is the standard libopcodes disassembler_info structure.  Bytes for the
+     instruction being printed should be read using INFO->read_memory_func
+     as the actual instruction bytes might be in a buffer.
+
+     Use INFO->fprintf_func to print the results of the disassembly, and
+     return the length of the instruction.
+
+     If no instruction can be disassembled then return an empty value and
+     other extension languages will get a chance to perform the
+     disassembly.  */
+  gdb::optional<int> (*print_insn) (struct gdbarch *gdbarch,
+				    CORE_ADDR address,
+				    struct disassemble_info *info);
 };
 
 /* State necessary to restore a signal handler to its previous value.  */

gdb/extension.c

@@ -924,6 +924,26 @@ ext_lang_colorize_disasm (const std::string &content, gdbarch *gdbarch)
   return result;
 }
 
+/* See extension.h.  */
+
+gdb::optional<int>
+ext_lang_print_insn (struct gdbarch *gdbarch, CORE_ADDR address,
+		     struct disassemble_info *info)
+{
+  for (const struct extension_language_defn *extlang : extension_languages)
+    {
+      if (extlang->ops == nullptr
+	  || extlang->ops->print_insn == nullptr)
+	continue;
+      gdb::optional<int> length
+	= extlang->ops->print_insn (gdbarch, address, info);
+      if (length.has_value ())
+	return length;
+    }
+
+  return {};
+}
+
 /* Called via an observer before gdb prints its prompt.
    Iterate over the extension languages giving them a chance to
    change the prompt.  The first one to change the prompt wins,

gdb/extension.h

@@ -327,6 +327,16 @@ extern gdb::optional<std::string> ext_lang_colorize
 extern gdb::optional<std::string> ext_lang_colorize_disasm
   (const std::string &content, gdbarch *gdbarch);
 
+/* Calls extension_language_ops::print_insn for each extension language,
+   returning the result from the first extension language that returns a
+   non-empty result (any further extension languages are not then called).
+
+   All arguments are forwarded to extension_language_ops::print_insn, see
+   that function for a full description.  */
+
+extern gdb::optional<int> ext_lang_print_insn
+  (struct gdbarch *gdbarch, CORE_ADDR address, struct disassemble_info *info);
+
 #if GDB_SELF_TEST
 namespace selftests {
 extern void (*hook_set_active_ext_lang) ();

gdb/guile/guile.c

@@ -130,8 +130,12 @@ static const struct extension_language_ops guile_extension_ops =
   gdbscm_breakpoint_has_cond,
   gdbscm_breakpoint_cond_says_stop,
 
-  NULL, /* gdbscm_check_quit_flag, */
   NULL, /* gdbscm_set_quit_flag, */
+  NULL, /* gdbscm_check_quit_flag, */
+  NULL, /* gdbscm_before_prompt, */
+  NULL, /* gdbscm_get_matching_xmethod_workers */
+  NULL, /* gdbscm_colorize */
+  NULL, /* gdbscm_print_insn */
 };
 #endif
 

gdb/python/python.c

@@ -166,6 +166,8 @@ static const struct extension_language_ops python_extension_ops =
   gdbpy_colorize,
 
   gdbpy_colorize_disasm,
+
+  NULL, /* gdbpy_print_insn, */
 };
 
 #endif /* HAVE_PYTHON */