analyzer work on issues with flex-generated lexers [PR103546]

PR analyzer/103546 tracks various false positives seen on
flex-generated lexers.

Whilst investigating them, I noticed an ICE with
-fanalyzer-call-summaries due to attempting to store sm-state
for an UNKNOWN svalue, which this patch fixes.

This patch also provides known_function implementations of all of the
external functions called by the lexer, reducing the number of false
positives.

The patch doesn't eliminate all false positives, but adds integration
tests to try to establish a baseline from which the remaining false
positives can be fixed.

gcc/analyzer/ChangeLog:
	PR analyzer/103546
	* analyzer.h (register_known_file_functions): New decl.
	* program-state.cc (sm_state_map::replay_call_summary): Rejct
	attempts to store sm-state for caller_sval that can't have
	associated state.
	* region-model-impl-calls.cc (register_known_functions): Call
	register_known_file_functions.
	* sm-fd.cc (class kf_isatty): New.
	(register_known_fd_functions): Register it.
	* sm-file.cc (class kf_ferror): New.
	(class kf_fileno): New.
	(class kf_getc): New.
	(register_known_file_functions): New.

gcc/ChangeLog:
	PR analyzer/103546
	* doc/invoke.texi (Static Analyzer Options): Add isatty, ferror,
	fileno, and getc to the list of functions known to the analyzer.

gcc/testsuite/ChangeLog:
	PR analyzer/103546
	* gcc.dg/analyzer/ferror-1.c: New test.
	* gcc.dg/analyzer/fileno-1.c: New test.
	* gcc.dg/analyzer/flex-with-call-summaries.c: New test.
	* gcc.dg/analyzer/flex-without-call-summaries.c: New test.
	* gcc.dg/analyzer/getc-1.c: New test.
	* gcc.dg/analyzer/isatty-1.c: New test.

Signed-off-by: David Malcolm <dmalcolm@redhat.com>

gcc/analyzer/analyzer.h

@@ -259,6 +259,7 @@ public:
 
 extern void register_known_functions (known_function_manager &mgr);
 extern void register_known_fd_functions (known_function_manager &kfm);
+extern void register_known_file_functions (known_function_manager &kfm);
 extern void register_varargs_builtins (known_function_manager &kfm);
 
 /* Passed by pointer to PLUGIN_ANALYZER_INIT callbacks.  */

gcc/analyzer/program-state.cc

@@ -821,6 +821,8 @@ sm_state_map::replay_call_summary (call_summary_replay &r,
       const svalue *caller_sval = r.convert_svalue_from_summary (summary_sval);
       if (!caller_sval)
 	continue;
+      if (!caller_sval->can_have_associated_state_p ())
+	continue;
       const svalue *summary_origin = kv.second.m_origin;
       const svalue *caller_origin
 	= (summary_origin

gcc/analyzer/region-model-impl-calls.cc

@@ -1662,6 +1662,7 @@ register_known_functions (known_function_manager &kfm)
     kfm.add ("putenv", make_unique<kf_putenv> ());
 
     register_known_fd_functions (kfm);
+    register_known_file_functions (kfm);
   }
 
   /* glibc functions.  */

gcc/analyzer/sm-fd.cc

@@ -2487,6 +2487,84 @@ public:
   }
 };
 
+/* Handler for "isatty"".
+   See e.g. https://man7.org/linux/man-pages/man3/isatty.3.html  */
+
+class kf_isatty : public known_function
+{
+  class outcome_of_isatty : public succeed_or_fail_call_info
+  {
+  public:
+    outcome_of_isatty (const call_details &cd, bool success)
+    : succeed_or_fail_call_info (cd, success)
+    {}
+
+    bool update_model (region_model *model,
+		       const exploded_edge *,
+		       region_model_context *ctxt) const final override
+    {
+      const call_details cd (get_call_details (model, ctxt));
+
+      if (m_success)
+	{
+	  /* Return 1.  */
+	  model->update_for_int_cst_return (cd, 1, true);
+	}
+      else
+	{
+	  /* Return 0; set errno.  */
+	  model->update_for_int_cst_return (cd, 0, true);
+	  model->set_errno (cd);
+	}
+
+      return feasible_p (cd, ctxt);
+    }
+
+  private:
+    bool feasible_p (const call_details &cd,
+		     region_model_context *ctxt) const
+    {
+      if (m_success)
+	{
+	  /* Can't be "success" on a closed/invalid fd.  */
+	  sm_state_map *smap;
+	  const fd_state_machine *fd_sm;
+	  std::unique_ptr<sm_context> sm_ctxt;
+	  if (!get_fd_state (ctxt, &smap, &fd_sm, NULL, &sm_ctxt))
+	    return true;
+	  const extrinsic_state *ext_state = ctxt->get_ext_state ();
+	  if (!ext_state)
+	    return true;
+
+	  const svalue *fd_sval = cd.get_arg_svalue (0);
+	  state_machine::state_t old_state
+	    = sm_ctxt->get_state (cd.get_call_stmt (), fd_sval);
+
+	  if (fd_sm->is_closed_fd_p (old_state)
+	      || old_state == fd_sm->m_invalid)
+	    return false;
+	}
+      return true;
+    }
+  }; // class outcome_of_isatty
+
+public:
+  bool matches_call_types_p (const call_details &cd) const final override
+  {
+    return cd.num_args () == 1;
+  }
+
+  void impl_call_post (const call_details &cd) const final override
+  {
+    if (cd.get_ctxt ())
+      {
+	cd.get_ctxt ()->bifurcate (make_unique<outcome_of_isatty> (cd, false));
+	cd.get_ctxt ()->bifurcate (make_unique<outcome_of_isatty> (cd, true));
+	cd.get_ctxt ()->terminate_path ();
+      }
+  }
+};
+
 /* Handler for calls to "pipe" and "pipe2".
    See e.g. https://www.man7.org/linux/man-pages/man2/pipe.2.html  */
 
@@ -2582,6 +2660,7 @@ register_known_fd_functions (known_function_manager &kfm)
   kfm.add ("accept", make_unique<kf_accept> ());
   kfm.add ("bind", make_unique<kf_bind> ());
   kfm.add ("connect", make_unique<kf_connect> ());
+  kfm.add ("isatty", make_unique<kf_isatty> ());
   kfm.add ("listen", make_unique<kf_listen> ());
   kfm.add ("pipe", make_unique<kf_pipe> (1));
   kfm.add ("pipe2", make_unique<kf_pipe> (2));

gcc/analyzer/sm-file.cc

@@ -489,6 +489,59 @@ make_fileptr_state_machine (logger *logger)
   return new fileptr_state_machine (logger);
 }
 
+/* Handler for "ferror"".  */
+
+class kf_ferror : public known_function
+{
+public:
+  bool matches_call_types_p (const call_details &cd) const final override
+  {
+    return (cd.num_args () == 1
+	    && cd.arg_is_pointer_p (0));
+  }
+
+  /* No side effects.  */
+};
+
+/* Handler for "fileno"".  */
+
+class kf_fileno : public known_function
+{
+public:
+  bool matches_call_types_p (const call_details &cd) const final override
+  {
+    return (cd.num_args () == 1
+	    && cd.arg_is_pointer_p (0));
+  }
+
+  /* No side effects.  */
+};
+
+/* Handler for "getc"".  */
+
+class kf_getc : public known_function
+{
+public:
+  bool matches_call_types_p (const call_details &cd) const final override
+  {
+    return (cd.num_args () == 1
+	    && cd.arg_is_pointer_p (0));
+  }
+
+  /* No side effects.  */
+};
+
+/* Populate KFM with instances of known functions relating to
+   stdio streams.  */
+
+void
+register_known_file_functions (known_function_manager &kfm)
+{
+  kfm.add ("ferror", make_unique<kf_ferror> ());
+  kfm.add ("fileno", make_unique<kf_fileno> ());
+  kfm.add ("getc", make_unique<kf_getc> ());
+}
+
 #if CHECKING_P
 
 namespace selftest {

gcc/doc/invoke.texi

@@ -10774,6 +10774,7 @@ of the following functions for working with file descriptors:
 @item @code{close}
 @item @code{creat}
 @item @code{dup}, @code{dup2} and @code{dup3}
+@item @code{isatty}
 @item @code{pipe}, and @code{pipe2}
 @item @code{read}
 @item @code{write}
@@ -10794,9 +10795,12 @@ of the following functions for working with @code{<stdio.h>} streams:
 @code{__builtin_vprintf}
 @item @code{fopen}
 @item @code{fclose}
+@item @code{ferror}
 @item @code{fgets}
 @item @code{fgets_unlocked}
+@item @code{fileno}
 @item @code{fread}
+@item @code{getc}
 @item @code{getchar}
 @item @code{fprintf}
 @item @code{printf}

gcc/testsuite/gcc.dg/analyzer/ferror-1.c

@@ -0,0 +1,6 @@
+#include <stdio.h>
+
+int test_pass_through (FILE *stream)
+{
+  return ferror (stream);
+}

gcc/testsuite/gcc.dg/analyzer/fileno-1.c

@@ -0,0 +1,6 @@
+#include <stdio.h>
+
+int test_pass_through (FILE *stream)
+{
+  return fileno (stream);
+}

gcc/testsuite/gcc.dg/analyzer/getc-1.c

@@ -0,0 +1,6 @@
+#include <stdio.h>
+
+int test_pass_through (FILE *stream)
+{
+  return getc (stream);
+}

gcc/testsuite/gcc.dg/analyzer/isatty-1.c

@@ -0,0 +1,56 @@
+#include <errno.h>
+#include "analyzer-decls.h"
+
+extern int isatty(int fd);
+extern int close(int fd);
+
+int test_pass_through (int fd)
+{
+  return isatty (fd);
+}
+
+void test_merging (int fd)
+{
+  isatty (fd);
+  __analyzer_dump_exploded_nodes (0); /* { dg-warning "1 processed enode" } */
+}
+
+int test_outcomes (int fd)
+{
+  errno = 0;
+  int result = isatty (fd);
+  switch (result)
+    {
+    default:
+      __analyzer_dump_path (); /* { dg-bogus "path" } */
+      break;
+    case 0:
+      __analyzer_dump_path (); /* { dg-message "path" } */
+      __analyzer_eval (errno > 0); /* { dg-warning "TRUE" } */
+      break;
+    case 1:
+      __analyzer_dump_path (); /* { dg-message "path" } */
+      __analyzer_eval (errno == 0); /* { dg-warning "TRUE" } */
+      break;
+    }
+  return result;
+}
+
+int test_isatty_on_invalid_fd (void)
+{
+  errno = 0;
+  int result = isatty (-1);
+  __analyzer_eval (result == 0); /* { dg-warning "TRUE" } */
+  __analyzer_eval (errno > 0); /* { dg-warning "TRUE" } */
+  return result;
+}
+
+int test_isatty_on_closed_fd (int fd)
+{
+  close (fd);
+  errno = 0;
+  int result = isatty (fd);
+  __analyzer_eval (result == 0); /* { dg-warning "TRUE" } */
+  __analyzer_eval (errno > 0); /* { dg-warning "TRUE" } */
+  return result;
+}